--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd">
+<html >
+<head><title></title>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta name="generator" content="TeX4ht (https://tug.org/tex4ht/)">
+<meta name="originator" content="TeX4ht (https://tug.org/tex4ht/)">
+<!-- html,2 -->
+<meta name="src" content="tutorial.tex">
+<link rel="stylesheet" type="text/css" href="tutorial.css">
+</head><body
+>
+<div class="center"
+>
+<!--l. 34--><p class="noindent" >
+<!--l. 35--><p class="noindent" ><span
+class="bchb7t-x-x-172">Systemtap tutorial</span></div>
+<!--l. 38--><p class="noindent" > <div class="minipage">Frank Ch. Eigler <span
+class="cmtt-9"><fche@redhat.com> </span><br
+class="newline" /> November 19,
+2021</div>
+<div class="tableofcontents">
+<span class="likesectionToc" ><a
+href="tutorialli1.html#x2-1000" id="QQ2-2-1">Contents</a></span>
+<br /><span class="sectionToc" >1 <a
+href="tutorialse1.html#x3-20001" id="QQ2-3-2">Introduction</a></span>
+<br /><span class="sectionToc" >2 <a
+href="tutorialse2.html#x4-30002" id="QQ2-4-5">Tracing</a></span>
+<br /> <span class="subsectionToc" >2.1 <a
+href="tutorialse2.html#x4-40002.1" id="QQ2-4-6">Where to probe</a></span>
+<br /> <span class="subsectionToc" >2.2 <a
+href="tutorialse2.html#x4-50002.2" id="QQ2-4-7">What to print</a></span>
+<br /> <span class="subsectionToc" >2.3 <a
+href="tutorialse2.html#x4-60002.3" id="QQ2-4-9">Exercises</a></span>
+<br /><span class="sectionToc" >3 <a
+href="tutorialse3.html#x6-70003" id="QQ2-6-10">Analysis</a></span>
+<br /> <span class="subsectionToc" >3.1 <a
+href="tutorialse3.html#x6-80003.1" id="QQ2-6-11">Basic constructs</a></span>
+<br /> <span class="subsectionToc" >3.2 <a
+href="tutorialse3.html#x6-90003.2" id="QQ2-6-13">Target variables</a></span>
+<br /> <span class="subsectionToc" >3.3 <a
+href="tutorialse3.html#x6-100003.3" id="QQ2-6-15">Functions</a></span>
+<br /> <span class="subsectionToc" >3.4 <a
+href="tutorialse3.html#x6-110003.4" id="QQ2-6-17">Arrays</a></span>
+<br /> <span class="subsectionToc" >3.5 <a
+href="tutorialse3.html#x6-120003.5" id="QQ2-6-18">Aggregates</a></span>
+<br /> <span class="subsectionToc" >3.6 <a
+href="tutorialse3.html#x6-130003.6" id="QQ2-6-19">Safety</a></span>
+<br /> <span class="subsectionToc" >3.7 <a
+href="tutorialse3.html#x6-140003.7" id="QQ2-6-20">Exercises</a></span>
+<br /><span class="sectionToc" >4 <a
+href="tutorialse4.html#x10-150004" id="QQ2-10-21">Tapsets</a></span>
+<br /> <span class="subsectionToc" >4.1 <a
+href="tutorialse4.html#x10-160004.1" id="QQ2-10-22">Automatic selection</a></span>
+<br /> <span class="subsectionToc" >4.2 <a
+href="tutorialse4.html#x10-170004.2" id="QQ2-10-25">Probe point aliases</a></span>
+<br /> <span class="subsectionToc" >4.3 <a
+href="tutorialse4.html#x10-180004.3" id="QQ2-10-27">Embedded C</a></span>
+<br /> <span class="subsectionToc" >4.4 <a
+href="tutorialse4.html#x10-190004.4" id="QQ2-10-29">Naming conventions</a></span>
+<br /> <span class="subsectionToc" >4.5 <a
+href="tutorialse4.html#x10-200004.5" id="QQ2-10-30">Exercises</a></span>
+<br /><span class="sectionToc" >5 <a
+href="tutorialse5.html#x12-210005" id="QQ2-12-31">Further information</a></span>
+<br /><span class="sectionToc" >A <a
+href="tutorialse6.html#x13-22000A" id="QQ2-13-32">Errors</a></span>
+<br /> <span class="subsectionToc" >A.1 <a
+href="tutorialse6.html#x13-23000A.1" id="QQ2-13-33">Parse errors</a></span>
+
+<br /> <span class="subsectionToc" >A.2 <a
+href="tutorialse6.html#x13-24000A.2" id="QQ2-13-34">Type errors</a></span>
+<br /> <span class="subsectionToc" >A.3 <a
+href="tutorialse6.html#x13-25000A.3" id="QQ2-13-35">Symbol errors</a></span>
+<br /> <span class="subsectionToc" >A.4 <a
+href="tutorialse6.html#x13-26000A.4" id="QQ2-13-36">Probing errors </a></span>
+<br /> <span class="subsectionToc" >A.5 <a
+href="tutorialse6.html#x13-27000A.5" id="QQ2-13-37">Runtime errors</a></span>
+<br /><span class="sectionToc" >B <a
+href="tutorialse7.html#x14-28000B" id="QQ2-14-38">Acknowledgments</a></span>
+</div>
+
+
+
+
+
+
+
+</body></html>
+
+
+
--- /dev/null
+
+/* start css.sty */
+.cmmi-10{font-style: italic;}
+.bchb7t-x-x-172{font-size:172%;font-weight: bold;}
+.bchb7t-x-x-172{font-weight: bold;}
+.cmtt-10{font-family: monospace,monospace;}
+.cmtt-10{font-family: monospace,monospace;}
+.cmtt-10{font-family: monospace,monospace;}
+.cmtt-10{font-family: monospace,monospace;}
+.cmtt-9{font-size:90%;font-family: monospace,monospace;}
+.cmtt-9{font-family: monospace,monospace;}
+.cmtt-9{font-family: monospace,monospace;}
+.cmtt-9{font-family: monospace,monospace;}
+.bchb7t-{font-weight: bold;}
+.bchb7t-{font-weight: bold;}
+.bchri7t-{font-style: oblique;}
+.bchri7t-{font-style: oblique;}
+.bchr7t-x-x-120{font-size:120%;}
+.bchr7t-x-x-90{font-size:90%;}
+.bchr7t-x-x-80{font-size:80%;}
+.cmtt-8{font-size:80%;font-family: monospace,monospace;}
+.cmtt-8{font-family: monospace,monospace;}
+.cmtt-8{font-family: monospace,monospace;}
+.cmtt-8{font-family: monospace,monospace;}
+.bchri7t-x-x-120{font-size:120%;font-style: oblique;}
+.bchri7t-x-x-120{font-style: oblique;}
+.cmtt-12{font-size:120%;font-family: monospace,monospace;}
+.cmtt-12{font-family: monospace,monospace;}
+.cmtt-12{font-family: monospace,monospace;}
+.cmtt-12{font-family: monospace,monospace;}
+.bchbi7t-{font-style: oblique;}
+p{margin-top:0;margin-bottom:0}
+p.indent{text-indent:0;}
+p + p{margin-top:1em;}
+p + div, p + pre {margin-top:1em;}
+div + p, pre + p {margin-top:1em;}
+@media print {div.crosslinks {visibility:hidden;}}
+a img { border-top: 0; border-left: 0; border-right: 0; }
+center { margin-top:1em; margin-bottom:1em; }
+td center { margin-top:0em; margin-bottom:0em; }
+.Canvas { position:relative; }
+img.math{vertical-align:middle;}
+li p.indent { text-indent: 0em }
+li p:first-child{ margin-top:0em; }
+li p:last-child, li div:last-child { margin-bottom:0.5em; }
+li p~ul:last-child, li p~ol:last-child{ margin-bottom:0.5em; }
+.enumerate1 {list-style-type:decimal;}
+.enumerate2 {list-style-type:lower-alpha;}
+.enumerate3 {list-style-type:lower-roman;}
+.enumerate4 {list-style-type:upper-alpha;}
+div.newtheorem { margin-bottom: 2em; margin-top: 2em;}
+.obeylines-h,.obeylines-v {white-space: nowrap; }
+div.obeylines-v p { margin-top:0; margin-bottom:0; }
+.overline{ text-decoration:overline; }
+.overline img{ border-top: 1px solid black; }
+td.displaylines {text-align:center; white-space:nowrap;}
+.centerline {text-align:center;}
+.rightline {text-align:right;}
+pre.verbatim {font-family: monospace,monospace; text-align:left; clear:both; }
+.fbox {padding-left:3.0pt; padding-right:3.0pt; text-indent:0pt; border:solid black 0.4pt; }
+div.fbox {display:table}
+div.center div.fbox {text-align:center; clear:both; padding-left:3.0pt; padding-right:3.0pt; text-indent:0pt; border:solid black 0.4pt; }
+div.minipage{width:100%;}
+div.center, div.center div.center {text-align: center; margin-left:1em; margin-right:1em;}
+div.center div {text-align: left;}
+div.flushright, div.flushright div.flushright {text-align: right;}
+div.flushright div {text-align: left;}
+div.flushleft {text-align: left;}
+.underline{ text-decoration:underline; }
+.underline img{ border-bottom: 1px solid black; margin-bottom:1pt; }
+.framebox-c, .framebox-l, .framebox-r { padding-left:3.0pt; padding-right:3.0pt; text-indent:0pt; border:solid black 0.4pt; }
+.framebox-c {text-align:center;}
+.framebox-l {text-align:left;}
+.framebox-r {text-align:right;}
+span.thank-mark{ vertical-align: super }
+span.footnote-mark sup.textsuperscript, span.footnote-mark a sup.textsuperscript{ font-size:80%; }
+div.tabular, div.center div.tabular {text-align: center; margin-top:0.5em; margin-bottom:0.5em; }
+table.tabular td p{margin-top:0em;}
+table.tabular {margin-left: auto; margin-right: auto;}
+td p:first-child{ margin-top:0em; }
+td p:last-child{ margin-bottom:0em; }
+div.td00{ margin-left:0pt; margin-right:0pt; }
+div.td01{ margin-left:0pt; margin-right:5pt; }
+div.td10{ margin-left:5pt; margin-right:0pt; }
+div.td11{ margin-left:5pt; margin-right:5pt; }
+table[rules] {border-left:solid black 0.4pt; border-right:solid black 0.4pt; }
+td.td00{ padding-left:0pt; padding-right:0pt; }
+td.td01{ padding-left:0pt; padding-right:5pt; }
+td.td10{ padding-left:5pt; padding-right:0pt; }
+td.td11{ padding-left:5pt; padding-right:5pt; }
+table[rules] {border-left:solid black 0.4pt; border-right:solid black 0.4pt; }
+.hline hr, .cline hr{ height : 0px; margin:0px; }
+.hline td, .cline td{ padding: 0; }
+.hline hr, .cline hr{border:none;border-top:1px solid black;}
+.tabbing-right {text-align:right;}
+div.float, div.figure {margin-left: auto; margin-right: auto;}
+div.float img {text-align:center;}
+div.figure img {text-align:center;}
+.marginpar,.reversemarginpar {width:20%; float:right; text-align:left; margin-left:auto; margin-top:0.5em; font-size:85%; text-decoration:underline;}
+.marginpar p,.reversemarginpar p{margin-top:0.4em; margin-bottom:0.4em;}
+.reversemarginpar{float:left;}
+table.equation {width:100%;}
+.equation td{text-align:center; }
+td.equation { margin-top:1em; margin-bottom:1em; }
+td.equation-label { width:5%; text-align:center; }
+td.eqnarray4 { width:5%; white-space: normal; }
+td.eqnarray2 { width:5%; }
+table.eqnarray-star, table.eqnarray {width:100%;}
+div.eqnarray{text-align:center;}
+div.array {text-align:center;}
+div.pmatrix {text-align:center;}
+table.pmatrix {width:100%;}
+span.pmatrix img{vertical-align:middle;}
+div.pmatrix {text-align:center;}
+table.pmatrix {width:100%;}
+span.bar-css {text-decoration:overline;}
+img.cdots{vertical-align:middle;}
+.partToc a, .partToc, .likepartToc a, .likepartToc {line-height: 200%; font-weight:bold; font-size:110%;}
+.index-item, .index-subitem, .index-subsubitem {display:block}
+div.caption {text-indent:-2em; margin-left:3em; margin-right:1em; text-align:left;}
+div.caption span.id{font-weight: bold; white-space: nowrap; }
+h1.partHead{text-align: center}
+p.bibitem { text-indent: -2em; margin-left: 2em; margin-top:0.6em; margin-bottom:0.6em; }
+p.bibitem-p { text-indent: 0em; margin-left: 2em; margin-top:0.6em; margin-bottom:0.6em; }
+.paragraphHead, .likeparagraphHead { margin-top:2em; font-weight: bold;}
+.subparagraphHead, .likesubparagraphHead { font-weight: bold;}
+.quote {margin-bottom:0.25em; margin-top:0.25em; margin-left:1em; margin-right:1em; text-align:justify;}
+.verse{white-space:nowrap; margin-left:2em}
+div.maketitle {text-align:center;}
+h2.titleHead{text-align:center;}
+div.maketitle{ margin-bottom: 2em; }
+div.author, div.date {text-align:center;}
+div.thanks{text-align:left; margin-left:10%; font-size:85%; font-style:italic; }
+div.author{white-space: nowrap;}
+.quotation {margin-bottom:0.25em; margin-top:0.25em; margin-left:1em; }
+.abstract p {margin-left:5%; margin-right:5%;}
+div.abstract {width:100%;}
+.subsectionToc, .likesubsectionToc {margin-left:2em;}
+.subsubsectionToc, .likesubsubsectionToc {margin-left:4em;}
+.rotatebox{display: inline-block;}
+P.fancyvrb {white-space: nowrap; margin:0em;}
+.lstlisting .label{margin-right:0.5em; }
+div.lstlisting{font-family: monospace,monospace; white-space: nowrap; margin-top:0.5em; margin-bottom:0.5em; }
+div.lstinputlisting{ font-family: monospace,monospace; white-space: nowrap; }
+.lstinputlisting .label{margin-right:0.5em;}
+.ovalbox { padding-left:3pt; padding-right:3pt; border:solid thin; }
+.Ovalbox-thick { padding-left:3pt; padding-right:3pt; border:solid thick; }
+.shadowbox { padding-left:3pt; padding-right:3pt; border:solid thin; border-right:solid thick; border-bottom:solid thick; }
+.doublebox { padding-left:3pt; padding-right:3pt; border-style:double; border:solid thick; }
+/* end css.sty */
+
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd">
+<html >
+<head><title></title>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta name="generator" content="TeX4ht (https://tug.org/tex4ht/)">
+<meta name="originator" content="TeX4ht (https://tug.org/tex4ht/)">
+<!-- html,2 -->
+<meta name="src" content="tutorial.tex">
+<link rel="stylesheet" type="text/css" href="tutorial.css">
+</head><body
+>
+<div class="center"
+>
+<!--l. 34--><p class="noindent" >
+<!--l. 35--><p class="noindent" ><span
+class="bchb7t-x-x-172">Systemtap tutorial</span></div>
+<!--l. 38--><p class="noindent" > <div class="minipage">Frank Ch. Eigler <span
+class="cmtt-9"><fche@redhat.com> </span><br
+class="newline" /> November 19,
+2021</div>
+<div class="tableofcontents">
+<span class="likesectionToc" ><a
+href="tutorialli1.html#x2-1000" id="QQ2-2-1">Contents</a></span>
+<br /><span class="sectionToc" >1 <a
+href="tutorialse1.html#x3-20001" id="QQ2-3-2">Introduction</a></span>
+<br /><span class="sectionToc" >2 <a
+href="tutorialse2.html#x4-30002" id="QQ2-4-5">Tracing</a></span>
+<br /> <span class="subsectionToc" >2.1 <a
+href="tutorialse2.html#x4-40002.1" id="QQ2-4-6">Where to probe</a></span>
+<br /> <span class="subsectionToc" >2.2 <a
+href="tutorialse2.html#x4-50002.2" id="QQ2-4-7">What to print</a></span>
+<br /> <span class="subsectionToc" >2.3 <a
+href="tutorialse2.html#x4-60002.3" id="QQ2-4-9">Exercises</a></span>
+<br /><span class="sectionToc" >3 <a
+href="tutorialse3.html#x6-70003" id="QQ2-6-10">Analysis</a></span>
+<br /> <span class="subsectionToc" >3.1 <a
+href="tutorialse3.html#x6-80003.1" id="QQ2-6-11">Basic constructs</a></span>
+<br /> <span class="subsectionToc" >3.2 <a
+href="tutorialse3.html#x6-90003.2" id="QQ2-6-13">Target variables</a></span>
+<br /> <span class="subsectionToc" >3.3 <a
+href="tutorialse3.html#x6-100003.3" id="QQ2-6-15">Functions</a></span>
+<br /> <span class="subsectionToc" >3.4 <a
+href="tutorialse3.html#x6-110003.4" id="QQ2-6-17">Arrays</a></span>
+<br /> <span class="subsectionToc" >3.5 <a
+href="tutorialse3.html#x6-120003.5" id="QQ2-6-18">Aggregates</a></span>
+<br /> <span class="subsectionToc" >3.6 <a
+href="tutorialse3.html#x6-130003.6" id="QQ2-6-19">Safety</a></span>
+<br /> <span class="subsectionToc" >3.7 <a
+href="tutorialse3.html#x6-140003.7" id="QQ2-6-20">Exercises</a></span>
+<br /><span class="sectionToc" >4 <a
+href="tutorialse4.html#x10-150004" id="QQ2-10-21">Tapsets</a></span>
+<br /> <span class="subsectionToc" >4.1 <a
+href="tutorialse4.html#x10-160004.1" id="QQ2-10-22">Automatic selection</a></span>
+<br /> <span class="subsectionToc" >4.2 <a
+href="tutorialse4.html#x10-170004.2" id="QQ2-10-25">Probe point aliases</a></span>
+<br /> <span class="subsectionToc" >4.3 <a
+href="tutorialse4.html#x10-180004.3" id="QQ2-10-27">Embedded C</a></span>
+<br /> <span class="subsectionToc" >4.4 <a
+href="tutorialse4.html#x10-190004.4" id="QQ2-10-29">Naming conventions</a></span>
+<br /> <span class="subsectionToc" >4.5 <a
+href="tutorialse4.html#x10-200004.5" id="QQ2-10-30">Exercises</a></span>
+<br /><span class="sectionToc" >5 <a
+href="tutorialse5.html#x12-210005" id="QQ2-12-31">Further information</a></span>
+<br /><span class="sectionToc" >A <a
+href="tutorialse6.html#x13-22000A" id="QQ2-13-32">Errors</a></span>
+<br /> <span class="subsectionToc" >A.1 <a
+href="tutorialse6.html#x13-23000A.1" id="QQ2-13-33">Parse errors</a></span>
+
+<br /> <span class="subsectionToc" >A.2 <a
+href="tutorialse6.html#x13-24000A.2" id="QQ2-13-34">Type errors</a></span>
+<br /> <span class="subsectionToc" >A.3 <a
+href="tutorialse6.html#x13-25000A.3" id="QQ2-13-35">Symbol errors</a></span>
+<br /> <span class="subsectionToc" >A.4 <a
+href="tutorialse6.html#x13-26000A.4" id="QQ2-13-36">Probing errors </a></span>
+<br /> <span class="subsectionToc" >A.5 <a
+href="tutorialse6.html#x13-27000A.5" id="QQ2-13-37">Runtime errors</a></span>
+<br /><span class="sectionToc" >B <a
+href="tutorialse7.html#x14-28000B" id="QQ2-14-38">Acknowledgments</a></span>
+</div>
+
+
+
+
+
+
+
+</body></html>
+
+
+
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd">
+<html >
+<head><title></title>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta name="generator" content="TeX4ht (https://tug.org/tex4ht/)">
+<meta name="originator" content="TeX4ht (https://tug.org/tex4ht/)">
+<!-- html,2 -->
+<meta name="src" content="tutorial.tex">
+<link rel="stylesheet" type="text/css" href="tutorial.css">
+</head><body
+>
+ <div class="footnote-text">
+ <!--l. 963--><p class="noindent" ><span class="footnote-mark"><a
+ id="fn5x0"> <sup class="textsuperscript">5</sup></a></span><span
+class="bchr7t-x-x-80">This is only necessary if the types cannot be inferred from other sources, such as the call sites.</span></div>
+
+</body></html>
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd">
+<html >
+<head><title></title>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta name="generator" content="TeX4ht (https://tug.org/tex4ht/)">
+<meta name="originator" content="TeX4ht (https://tug.org/tex4ht/)">
+<!-- html,2 -->
+<meta name="src" content="tutorial.tex">
+<link rel="stylesheet" type="text/css" href="tutorial.css">
+</head><body
+>
+ <div class="footnote-text">
+ <!--l. 189--><p class="noindent" ><span class="footnote-mark"><a
+ id="fn1x0"> <sup class="textsuperscript">1</sup></a></span><span
+class="bchr7t-x-x-80">Without the </span><span
+class="cmtt-8">.call </span><span
+class="bchr7t-x-x-80">qualifier, inlined function instances are also probed, but they have no corresponding </span><span
+class="cmtt-8">.return</span><span
+class="bchr7t-x-x-80">.</span></div>
+
+</body></html>
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd">
+<html >
+<head><title></title>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta name="generator" content="TeX4ht (https://tug.org/tex4ht/)">
+<meta name="originator" content="TeX4ht (https://tug.org/tex4ht/)">
+<!-- html,2 -->
+<meta name="src" content="tutorial.tex">
+<link rel="stylesheet" type="text/css" href="tutorial.css">
+</head><body
+>
+ <div class="footnote-text">
+ <!--l. 360--><p class="noindent" ><span class="footnote-mark"><a
+ id="fn2x0"> <sup class="textsuperscript">2</sup></a></span><span
+class="bchr7t-x-x-80">Use them between consecutive expressions that place unary </span><span
+class="cmtt-8">+</span><span
+class="bchr7t-x-x-80">,</span><span
+class="cmtt-8">- </span><span
+class="bchr7t-x-x-80">or mixed pre/post </span><span
+class="cmtt-8">++</span><span
+class="bchr7t-x-x-80">,</span><span
+class="cmtt-8">-- </span><span
+class="bchr7t-x-x-80">in an ambiguous manner.</span></div>
+
+</body></html>
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd">
+<html >
+<head><title></title>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta name="generator" content="TeX4ht (https://tug.org/tex4ht/)">
+<meta name="originator" content="TeX4ht (https://tug.org/tex4ht/)">
+<!-- html,2 -->
+<meta name="src" content="tutorial.tex">
+<link rel="stylesheet" type="text/css" href="tutorial.css">
+</head><body
+>
+ <div class="footnote-text">
+ <!--l. 667--><p class="noindent" ><span class="footnote-mark"><a
+ id="fn3x0"> <sup class="textsuperscript">3</sup></a></span><span
+class="bchr7t-x-x-80">We anticipate support for indexing and looping using </span><span
+class="cmtt-8">foreach </span><span
+class="bchr7t-x-x-80">shortly.</span></div>
+
+</body></html>
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd">
+<html >
+<head><title></title>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta name="generator" content="TeX4ht (https://tug.org/tex4ht/)">
+<meta name="originator" content="TeX4ht (https://tug.org/tex4ht/)">
+<!-- html,2 -->
+<meta name="src" content="tutorial.tex">
+<link rel="stylesheet" type="text/css" href="tutorial.css">
+</head><body
+>
+ <div class="footnote-text">
+ <!--l. 718--><p class="noindent" ><span class="footnote-mark"><a
+ id="fn4x0"> <sup class="textsuperscript">4</sup></a></span><span
+class="bchr7t-x-x-80">See </span><span
+class="cmtt-8">http://sourceware.org/bugzilla</span></div>
+</body></html>
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd">
+<html >
+<head><title>Contents</title>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta name="generator" content="TeX4ht (https://tug.org/tex4ht/)">
+<meta name="originator" content="TeX4ht (https://tug.org/tex4ht/)">
+<!-- html,2 -->
+<meta name="src" content="tutorial.tex">
+<link rel="stylesheet" type="text/css" href="tutorial.css">
+</head><body
+>
+<!--l. 44--><div class="crosslinks"><p class="noindent">[<a
+href="tutorialse1.html" >next</a>] [<a
+href="#tailtutorialli1.html">tail</a>] [<a
+href="tutorial.html#tutorialli1.html" >up</a>] </p></div>
+<h3 class="likesectionHead"><a
+ id="x2-1000"></a>Contents</h3>
+<div class="tableofcontents">
+<span class="sectionToc" >1 <a
+href="tutorialse1.html#x3-20001">Introduction</a></span>
+<br /><span class="sectionToc" >2 <a
+href="tutorialse2.html#x4-30002">Tracing</a></span>
+<br /> <span class="subsectionToc" >2.1 <a
+href="tutorialse2.html#x4-40002.1">Where to probe</a></span>
+<br /> <span class="subsectionToc" >2.2 <a
+href="tutorialse2.html#x4-50002.2">What to print</a></span>
+<br /> <span class="subsectionToc" >2.3 <a
+href="tutorialse2.html#x4-60002.3">Exercises</a></span>
+<br /><span class="sectionToc" >3 <a
+href="tutorialse3.html#x6-70003">Analysis</a></span>
+<br /> <span class="subsectionToc" >3.1 <a
+href="tutorialse3.html#x6-80003.1">Basic constructs</a></span>
+<br /> <span class="subsectionToc" >3.2 <a
+href="tutorialse3.html#x6-90003.2">Target variables</a></span>
+<br /> <span class="subsectionToc" >3.3 <a
+href="tutorialse3.html#x6-100003.3">Functions</a></span>
+<br /> <span class="subsectionToc" >3.4 <a
+href="tutorialse3.html#x6-110003.4">Arrays</a></span>
+<br /> <span class="subsectionToc" >3.5 <a
+href="tutorialse3.html#x6-120003.5">Aggregates</a></span>
+<br /> <span class="subsectionToc" >3.6 <a
+href="tutorialse3.html#x6-130003.6">Safety</a></span>
+<br /> <span class="subsectionToc" >3.7 <a
+href="tutorialse3.html#x6-140003.7">Exercises</a></span>
+<br /><span class="sectionToc" >4 <a
+href="tutorialse4.html#x10-150004">Tapsets</a></span>
+<br /> <span class="subsectionToc" >4.1 <a
+href="tutorialse4.html#x10-160004.1">Automatic selection</a></span>
+<br /> <span class="subsectionToc" >4.2 <a
+href="tutorialse4.html#x10-170004.2">Probe point aliases</a></span>
+<br /> <span class="subsectionToc" >4.3 <a
+href="tutorialse4.html#x10-180004.3">Embedded C</a></span>
+<br /> <span class="subsectionToc" >4.4 <a
+href="tutorialse4.html#x10-190004.4">Naming conventions</a></span>
+<br /> <span class="subsectionToc" >4.5 <a
+href="tutorialse4.html#x10-200004.5">Exercises</a></span>
+<br /><span class="sectionToc" >5 <a
+href="tutorialse5.html#x12-210005">Further information</a></span>
+<br /><span class="sectionToc" >A <a
+href="tutorialse6.html#x13-22000A">Errors</a></span>
+<br /> <span class="subsectionToc" >A.1 <a
+href="tutorialse6.html#x13-23000A.1">Parse errors</a></span>
+<br /> <span class="subsectionToc" >A.2 <a
+href="tutorialse6.html#x13-24000A.2">Type errors</a></span>
+<br /> <span class="subsectionToc" >A.3 <a
+href="tutorialse6.html#x13-25000A.3">Symbol errors</a></span>
+
+<br /> <span class="subsectionToc" >A.4 <a
+href="tutorialse6.html#x13-26000A.4">Probing errors </a></span>
+<br /> <span class="subsectionToc" >A.5 <a
+href="tutorialse6.html#x13-27000A.5">Runtime errors</a></span>
+<br /><span class="sectionToc" >B <a
+href="tutorialse7.html#x14-28000B">Acknowledgments</a></span>
+</div>
+
+<!--l. 46--><div class="crosslinks"><p class="noindent">[<a
+href="tutorialse1.html" >next</a>] [<a
+href="tutorialli1.html" >front</a>] [<a
+href="tutorial.html#tutorialli1.html" >up</a>] </p></div>
+<!--l. 46--><p class="noindent" ><a
+ id="tailtutorialli1.html"></a>
+</body></html>
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd">
+<html >
+<head><title>Introduction</title>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta name="generator" content="TeX4ht (https://tug.org/tex4ht/)">
+<meta name="originator" content="TeX4ht (https://tug.org/tex4ht/)">
+<!-- html,2 -->
+<meta name="src" content="tutorial.tex">
+<link rel="stylesheet" type="text/css" href="tutorial.css">
+</head><body
+>
+<!--l. 46--><div class="crosslinks"><p class="noindent">[<a
+href="tutorialse2.html" >next</a>] [<a
+href="tutorialli1.html" >prev</a>] [<a
+href="tutorialli1.html#tailtutorialli1.html" >prev-tail</a>] [<a
+href="#tailtutorialse1.html">tail</a>] [<a
+href="tutorial.html#tutorialse1.html" >up</a>] </p></div>
+<h3 class="sectionHead"><span class="titlemark">1 </span> <a
+ id="x3-20001"></a>Introduction</h3>
+<!--l. 48--><p class="noindent" >Systemtap is a tool that allows developers and administrators to write and reuse simple scripts to deeply examine the
+activities of a live Linux system. Data may be extracted, filtered, and summarized quickly and safely, to enable
+diagnoses of complex performance or functional problems.
+<!--l. 54--><p class="noindent" ><span
+class="bchb7t-">NOTE: </span>This tutorial does not describe every feature available in systemtap. Please see the individual <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">stap</span></span></span>
+manual pages for the most up-to-date information. These may be available installed on your system, or at
+<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">http://sourceware.org/systemtap/man/</span></span></span>.
+<!--l. 61--><p class="noindent" >The essential idea behind a systemtap script is to name <span
+class="bchri7t-">events</span>, and to give them <span
+class="bchri7t-">handlers</span>. Whenever a specified event
+occurs, the Linux kernel runs the handler as if it were a quick subroutine, then resumes. There are several kind of
+events, such as entering or exiting a function, a timer expiring, or the entire systemtap session starting or stopping. A
+handler is a series of script language statements that specify the work to be done whenever the event occurs. This
+work normally includes extracting data from the event context, storing them into internal variables, or printing
+results.
+<!--l. 79--><p class="noindent" >Systemtap works by translating the script to C, running the system C compiler to create a kernel module from that.
+When the module is loaded, it activates all the probed events by hooking into the kernel. Then, as
+events occur on any processor, the compiled handlers run. Eventually, the session stops, the hooks are
+disconnected, and the module removed. This entire process is driven from a single command-line program,
+<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">stap</span></span></span>.
+<!--l. 87--><p class="noindent" ><hr class="figure"><div class="figure"
+>
+
+<a
+ id="x3-20011"></a>
+
+<div class="center"
+>
+<!--l. 88--><p class="noindent" >
+<div class="fbox"><div class="minipage"><pre class="verbatim" id="verbatim-1">
+# cat hello-world.stp
+probe begin
+{
+  print ("hello world\n")
+  exit ()
+}
+# stap hello-world.stp
+hello world
+</pre>
+<!--l. 99--><p class="nopar" > </div></div>
+</div>
+<br /> <div class="caption"
+><span class="id">Figure 1: </span><span
+class="content">A systemtap smoke test.</span></div><!--tex4ht:label?: x3-20011 -->
+
+<!--l. 103--><p class="noindent" ></div><hr class="endfigure">
+<!--l. 105--><p class="noindent" >This paper assumes that you have installed systemtap and its prerequisite kernel development tools and debugging
+data, so that you can run the scripts such as the simple one in Figure <a
+href="#x3-20001">1<!--tex4ht:ref: fig:hello-world --></a>. Log on as <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">root</span></span></span>, or even better,
+login as a user that is a member of <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">stapdev</span></span></span> group or as a user authorized to <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">sudo</span></span></span>, before running
+systemtap.
+<!--l. 112--><p class="noindent" ><hr class="figure"><div class="figure"
+>
+
+<a
+ id="x3-20022"></a>
+
+<div class="center"
+>
+<!--l. 113--><p class="noindent" >
+<div class="fbox"><div class="minipage"><pre class="verbatim" id="verbatim-2">
+# cat strace-open.stp
+probe syscall.open
+{
+  printf ("%s(%d) open (%s)\n", execname(), pid(), argstr)
+}
+probe timer.ms(4000) # after 4 seconds
+{
+  exit ()
+}
+# stap strace-open.stp
+vmware-guestd(2206) open ("/etc/redhat-release", O_RDONLY)
+hald(2360) open ("/dev/hdc", O_RDONLY|O_EXCL|O_NONBLOCK)
+hald(2360) open ("/dev/hdc", O_RDONLY|O_EXCL|O_NONBLOCK)
+hald(2360) open ("/dev/hdc", O_RDONLY|O_EXCL|O_NONBLOCK)
+df(3433) open ("/etc/ld.so.cache", O_RDONLY)
+df(3433) open ("/lib/tls/libc.so.6", O_RDONLY)
+df(3433) open ("/etc/mtab", O_RDONLY)
+hald(2360) open ("/dev/hdc", O_RDONLY|O_EXCL|O_NONBLOCK)
+</pre>
+<!--l. 134--><p class="nopar" > </div></div>
+</div>
+<br /> <div class="caption"
+><span class="id">Figure 2: </span><span
+class="content">A taste of systemtap: a system-wide <span
+class="cmtt-10">strace</span>, just for the <span
+class="cmtt-10">open </span>system call.</span></div><!--tex4ht:label?: x3-20022 -->
+
+<!--l. 139--><p class="noindent" ></div><hr class="endfigure">
+
+<!--l. 142--><div class="crosslinks"><p class="noindent">[<a
+href="tutorialse2.html" >next</a>] [<a
+href="tutorialli1.html" >prev</a>] [<a
+href="tutorialli1.html#tailtutorialli1.html" >prev-tail</a>] [<a
+href="tutorialse1.html" >front</a>] [<a
+href="tutorial.html#tutorialse1.html" >up</a>] </p></div>
+<!--l. 142--><p class="noindent" ><a
+ id="tailtutorialse1.html"></a>
+</body></html>
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd">
+<html >
+<head><title>Tracing</title>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta name="generator" content="TeX4ht (https://tug.org/tex4ht/)">
+<meta name="originator" content="TeX4ht (https://tug.org/tex4ht/)">
+<!-- html,2 -->
+<meta name="src" content="tutorial.tex">
+<link rel="stylesheet" type="text/css" href="tutorial.css">
+</head><body
+>
+<!--l. 142--><div class="crosslinks"><p class="noindent">[<a
+href="tutorialse4.html" >next</a>] [<a
+href="tutorialse1.html" >prev</a>] [<a
+href="tutorialse1.html#tailtutorialse1.html" >prev-tail</a>] [<a
+href="#tailtutorialse2.html">tail</a>] [<a
+href="tutorial.html#tutorialse2.html" >up</a>] </p></div>
+<h3 class="sectionHead"><span class="titlemark">2 </span> <a
+ id="x4-30002"></a>Tracing</h3>
+<!--l. 144--><p class="noindent" >The simplest kind of probe is simply to <span
+class="bchri7t-">trace </span>an event. This is the effect of inserting strategically located <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">print</span></span></span>
+statements into a program. This is often the first step of problem solving: explore by seeing a history of what has
+happened.
+<!--l. 150--><p class="noindent" >This style of instrumentation is the simplest. It just asks systemtap to print something at each event. To express this
+in the script language, you need to say where to probe and what to print there.
+<!--l. 154--><p class="noindent" >
+<h4 class="subsectionHead"><span class="titlemark">2.1 </span> <a
+ id="x4-40002.1"></a>Where to probe</h4>
+<!--l. 156--><p class="noindent" >Systemtap supports a number of built-in events. The library of scripts that comes with systemtap, each called a
+“tapset”, may define additional ones defined in terms of the built-in family. <span
+class="bchb7t-">See the </span><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">stapprobes</span></span></span> <span
+class="bchb7t-">man page for</span>
+<span
+class="bchb7t-">details on these and many other probe point families. </span>All these events are named using a unified syntax with
+dot-separated parameterized identifiers:
+<div class="tabular"> <table id="TBL-1" class="tabular"
+cellspacing="0" cellpadding="0"
+><colgroup id="TBL-1-1g"><col
+id="TBL-1-1"><col
+id="TBL-1-2"></colgroup><tr
+ style="vertical-align:baseline;" id="TBL-1-1-"><td style="white-space:nowrap; text-align:right;" id="TBL-1-1-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">begin</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-1-1-2"
+class="td11">The startup of the systemtap session. </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-1-2-"><td style="white-space:nowrap; text-align:right;" id="TBL-1-2-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">end</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-1-2-2"
+class="td11">The end of the systemtap session. </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-1-3-"><td style="white-space:nowrap; text-align:right;" id="TBL-1-3-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">kernel.function("sys_open")</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-1-3-2"
+class="td11">The entry to the function named <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">sys_open</span></span></span> in the kernel.</td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-1-4-"><td style="white-space:nowrap; text-align:right;" id="TBL-1-4-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">syscall.close.return</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-1-4-2"
+class="td11">The return from the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">close</span></span></span> system call. </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-1-5-"><td style="white-space:nowrap; text-align:right;" id="TBL-1-5-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">module("ext3").statement(0xdeadbeef)</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-1-5-2"
+class="td11">The addressed instruction in the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">ext3</span></span></span> filesystem driver. </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-1-6-"><td style="white-space:nowrap; text-align:right;" id="TBL-1-6-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">timer.ms(200)</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-1-6-2"
+class="td11">A timer that fires every 200 milliseconds. </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-1-7-"><td style="white-space:nowrap; text-align:right;" id="TBL-1-7-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">timer.profile</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-1-7-2"
+class="td11">A timer that fires periodically on every CPU. </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-1-8-"><td style="white-space:nowrap; text-align:right;" id="TBL-1-8-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">perf.hw.cache_misses</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-1-8-2"
+class="td11">A particular number of CPU cache misses have occurred.</td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-1-9-"><td style="white-space:nowrap; text-align:right;" id="TBL-1-9-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">procfs("status").read</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-1-9-2"
+class="td11">A process trying to read a synthetic file. </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-1-10-"><td style="white-space:nowrap; text-align:right;" id="TBL-1-10-1"
+class="td11"><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">process("a.out").statement("*@main.c:200")</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-1-10-2"
+class="td11">Line 200 of the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">a.out</span></span></span> program. </td>
+</tr></table></div>
+<!--l. 181--><p class="noindent" >Let’s say that you would like to trace all function entries and exits in a source file, say <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">net/socket.c</span></span></span> in the
+kernel. The <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">kernel.function</span></span></span> probe point lets you express that easily, since systemtap examines the
+kernel’s debugging information to relate object code to source code. It works like a debugger: if you can
+name or place it, you can probe it. Use <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">kernel.function("*@net/socket.c").call</span></span></span> for the function
+entries<span class="footnote-mark"><a
+href="tutorial5.html#fn1x0"><sup class="textsuperscript">1</sup></a></span><a
+ id="x4-4001f1"></a>,
+and <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">kernel.function("*@net/socket.c").return</span></span></span> for matching exits. Note the use of wildcards in the function
+name part, and the subsequent <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">@FILENAME</span></span></span> part. You can also put wildcards into the file name, and even add a colon
+(<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">:</span></span></span>) and a line number, if you want to restrict the search that precisely. Since systemtap will put a separate probe in
+every place that matches a probe point, a few wildcards can expand to hundreds or thousands of probes, so be
+careful what you ask for.
+<!--l. 202--><p class="noindent" >Once you identify the probe points, the skeleton of the systemtap script appears. The <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">probe</span></span></span> keyword introduces a
+
+probe point, or a comma-separated list of them. The following <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">{</span></span></span> and <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">}</span></span></span> braces enclose the handler for all listed probe
+points.
+
+<pre class="verbatim" id="verbatim-3">
+probe kernel.function("*@net/socket.c") { }
+probe kernel.function("*@net/socket.c").return { }
+</pre>
+<!--l. 209--><p class="nopar" > You can run this script as is, though with empty handlers there will be no output. Put the two lines into a new file.
+Run <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">stap</span><span
+class="cmtt-10"> -v</span><span
+class="cmtt-10"> FILE</span></span></span>. Terminate it any time with <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">^C</span></span></span>. (The <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">-v</span></span></span> option tells systemtap to print more verbose messages
+during its processing. Try the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">-h</span></span></span> option to see more options.)
+<h4 class="subsectionHead"><span class="titlemark">2.2 </span> <a
+ id="x4-50002.2"></a>What to print</h4>
+<!--l. 218--><p class="noindent" >Since you are interested in each function that was entered and exited, a line should be printed for each,
+containing the function name. In order to make that list easy to read, systemtap should indent the lines
+so that functions called by other traced functions are nested deeper. To tell each single process apart
+from any others that may be running concurrently, systemtap should also print the process ID in the
+line.
+<!--l. 226--><p class="noindent" >Systemtap provides a variety of such contextual data, ready for formatting. They usually appear as function calls
+within the handler, like you already saw in Figure <a
+href="tutorialse1.html#x3-20001">1<!--tex4ht:ref: fig:strace-open --></a>. <span
+class="bchb7t-">See the </span><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">function::*</span></span></span> <span
+class="bchb7t-">man pages for those functions and</span>
+<span
+class="bchb7t-">more defined in the tapset library</span>, but here’s a sampling:
+<div class="tabular"> <table id="TBL-2" class="tabular"
+cellspacing="0" cellpadding="0"
+><colgroup id="TBL-2-1g"><col
+id="TBL-2-1"><col
+id="TBL-2-2"></colgroup><tr
+ style="vertical-align:baseline;" id="TBL-2-1-"><td style="white-space:nowrap; text-align:right;" id="TBL-2-1-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">tid()</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-2-1-2"
+class="td11">The id of the current thread. </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-2-2-"><td style="white-space:nowrap; text-align:right;" id="TBL-2-2-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">pid()</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-2-2-2"
+class="td11">The process (task group) id of the current thread. </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-2-3-"><td style="white-space:nowrap; text-align:right;" id="TBL-2-3-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">uid()</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-2-3-2"
+class="td11">The id of the current user. </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-2-4-"><td style="white-space:nowrap; text-align:right;" id="TBL-2-4-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">execname()</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-2-4-2"
+class="td11">The name of the current process. </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-2-5-"><td style="white-space:nowrap; text-align:right;" id="TBL-2-5-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">cpu()</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-2-5-2"
+class="td11">The current cpu number. </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-2-6-"><td style="white-space:nowrap; text-align:right;" id="TBL-2-6-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">gettimeofday_s()</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-2-6-2"
+class="td11">Number of seconds since epoch. </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-2-7-"><td style="white-space:nowrap; text-align:right;" id="TBL-2-7-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">get_cycles()</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-2-7-2"
+class="td11">Snapshot of hardware cycle counter. </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-2-8-"><td style="white-space:nowrap; text-align:right;" id="TBL-2-8-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">pp()</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-2-8-2"
+class="td11">A string describing the probe point being currently handled. </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-2-9-"><td style="white-space:nowrap; text-align:right;" id="TBL-2-9-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">ppfunc()</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-2-9-2"
+class="td11">If known, the the function name in which this probe was placed. </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-2-10-"><td style="white-space:nowrap; text-align:right;" id="TBL-2-10-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">$$vars</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-2-10-2"
+class="td11">If available, a pretty-printed listing of all local variables in scope.</td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-2-11-"><td style="white-space:nowrap; text-align:right;" id="TBL-2-11-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">print_backtrace()</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-2-11-2"
+class="td11">If possible, print a kernel backtrace. </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-2-12-"><td style="white-space:nowrap; text-align:right;" id="TBL-2-12-1"
+class="td11"><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">print_ubacktrace()</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-2-12-2"
+class="td11">If possible, print a user-space backtrace. </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-2-13-"><td style="white-space:nowrap; text-align:right;" id="TBL-2-13-1"
+class="td11"> </td>
+</tr></table></div>
+<!--l. 249--><p class="noindent" >The values returned may be strings or numbers. The <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">print()</span></span></span> built-in function accepts either as its sole argument. Or,
+you can use the C-style <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">printf()</span></span></span> built-in, whose formatting argument may include <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">%s</span></span></span> for a string, <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">%d</span></span></span> for a number.
+<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">printf</span></span></span> and other functions take comma-separated arguments. Don’t forget a <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">"\n"</span></span></span> at the end. There exist more
+printing / formatting functions too.
+<!--l. 257--><p class="noindent" >A particularly handy function in the tapset library is <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">thread_indent</span></span></span>. Given an indentation delta parameter, it stores
+internally an indentation counter for each thread (<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">tid()</span></span></span>), and returns a string with some generic trace data plus an
+appropriate number of indentation spaces. That generic data includes a timestamp (number of microseconds since
+the initial indentation for the thread), a process name and the thread id itself. It therefore gives an idea not only
+about what functions were called, but who called them, and how long they took. Figure <a
+href="#x4-50013">3<!--tex4ht:ref: fig:socket-trace --></a> shows the finished script.
+It lacks a call to the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">exit()</span></span></span> function, so you need to interrupt it with <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">^C</span></span></span> when you want the tracing to
+stop.
+<!--l. 269--><p class="noindent" ><hr class="figure"><div class="figure"
+>
+
+<a
+ id="x4-50013"></a>
+
+<div class="center"
+>
+<!--l. 270--><p class="noindent" >
+<div class="fbox"><div class="minipage"><pre class="verbatim" id="verbatim-4">
+# cat socket-trace.stp
+probe kernel.function("*@net/socket.c").call {
+  printf ("%s -> %s\n", thread_indent(1), ppfunc())
+}
+probe kernel.function("*@net/socket.c").return {
+  printf ("%s <- %s\n", thread_indent(-1), ppfunc())
+}
+# stap socket-trace.stp
+     0 hald(2632): -> sock_poll
+    28 hald(2632): <- sock_poll
+[...]
+     0 ftp(7223): -> sys_socketcall
+  1159 ftp(7223):  -> sys_socket
+  2173 ftp(7223):   -> __sock_create
+  2286 ftp(7223):    -> sock_alloc_inode
+  2737 ftp(7223):    <- sock_alloc_inode
+  3349 ftp(7223):    -> sock_alloc
+  3389 ftp(7223):    <- sock_alloc
+  3417 ftp(7223):   <- __sock_create
+  4117 ftp(7223):   -> sock_create
+  4160 ftp(7223):   <- sock_create
+  4301 ftp(7223):   -> sock_map_fd
+  4644 ftp(7223):    -> sock_map_file
+  4699 ftp(7223):    <- sock_map_file
+  4715 ftp(7223):   <- sock_map_fd
+  4732 ftp(7223):  <- sys_socket
+  4775 ftp(7223): <- sys_socketcall
+[...]
+</pre>
+<!--l. 301--><p class="nopar" > </div></div>
+</div>
+<br /> <div class="caption"
+><span class="id">Figure 3: </span><span
+class="content">Tracing and timing functions in <span
+class="cmtt-10">net/sockets.c</span>.</span></div><!--tex4ht:label?: x4-50013 -->
+
+<!--l. 305--><p class="noindent" ></div><hr class="endfigure">
+<h4 class="subsectionHead"><span class="titlemark">2.3 </span> <a
+ id="x4-60002.3"></a>Exercises</h4>
+<!--l. 309--><p class="noindent" >
+ <ol class="enumerate1" >
+ <li
+ class="enumerate" id="x4-6002x1">Use the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">-L</span></span></span> option to systemtap to list all the kernel functions named with the word “nit” in them.
+ </li>
+ <li
+ class="enumerate" id="x4-6004x2">Trace some system calls (use <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">syscall.NAME</span></span></span> and <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">.return</span></span></span> probe points), with the same <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">thread_indent</span></span></span>
+ probe handler as in Figure <a
+href="#x4-50013">3<!--tex4ht:ref: fig:socket-trace --></a>. Print parameters using <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">$$parms</span></span></span> and <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">$$return</span></span></span>. Interpret the results.
+ </li>
+ <li
+ class="enumerate" id="x4-6006x3">Change figure <a
+href="#x4-50013">3<!--tex4ht:ref: fig:socket-trace --></a> by removing the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">.call</span></span></span> modifier from the first probe. Note how function entry and
+ function return now don’t match anymore. This is because now the first probe will match both normal
+ function entry and inlined functions. Try putting the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">.call</span></span></span> modifier back and add another probe just
+ for <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">probe</span><span
+class="cmtt-10"> kernel.function("*@net/socket.c").inline</span></span></span> What <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">printf</span></span></span> statement can you come up
+ with in the probe handler to show the inlined function entries nicely in between the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">.call</span></span></span> and <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">.return</span></span></span>
+ thread indented output?
+ </li></ol>
+
+<!--l. 330--><div class="crosslinks"><p class="noindent">[<a
+href="tutorialse4.html" >next</a>] [<a
+href="tutorialse1.html" >prev</a>] [<a
+href="tutorialse1.html#tailtutorialse1.html" >prev-tail</a>] [<a
+href="tutorialse2.html" >front</a>] [<a
+href="tutorial.html#tutorialse2.html" >up</a>] </p></div>
+<!--l. 330--><p class="noindent" ><a
+ id="tailtutorialse2.html"></a>
+</body></html>
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd">
+<html >
+<head><title>Analysis</title>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta name="generator" content="TeX4ht (https://tug.org/tex4ht/)">
+<meta name="originator" content="TeX4ht (https://tug.org/tex4ht/)">
+<!-- html,2 -->
+<meta name="src" content="tutorial.tex">
+<link rel="stylesheet" type="text/css" href="tutorial.css">
+</head><body
+>
+<!--l. 330--><div class="crosslinks"><p class="noindent">[<a
+href="tutorialse2.html" >prev</a>] [<a
+href="tutorialse2.html#tailtutorialse2.html" >prev-tail</a>] [<a
+href="#tailtutorialse3.html">tail</a>] [<a
+href="tutorial.html#tutorialse4.html" >up</a>] </p></div>
+<h3 class="sectionHead"><span class="titlemark">3 </span> <a
+ id="x6-70003"></a>Analysis</h3>
+<!--l. 332--><p class="noindent" >Pages of generic tracing text may give you enough information for exploring a system. With systemtap, it is possible
+to analyze that data, to filter, aggregate, transform, and summarize it. Different probes can work together to share
+data. Probe handlers can use a rich set of control constructs to describe algorithms, with a syntax taken roughly from
+<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">awk</span></span></span>. With these tools, systemtap scripts can focus on a specific question and provide a compact response: no <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">grep</span></span></span>
+needed.
+<!--l. 342--><p class="noindent" >
+<h4 class="subsectionHead"><span class="titlemark">3.1 </span> <a
+ id="x6-80003.1"></a>Basic constructs</h4>
+<!--l. 344--><p class="noindent" >Most systemtap scripts include conditionals, to limit tracing or other logic to those processes or users or <span
+class="bchri7t-">whatever </span>of
+interest. The syntax is simple:
+<div class="tabular"> <table id="TBL-3" class="tabular"
+cellspacing="0" cellpadding="0"
+><colgroup id="TBL-3-1g"><col
+id="TBL-3-1"><col
+id="TBL-3-2"></colgroup><tr
+ style="vertical-align:baseline;" id="TBL-3-1-"><td style="white-space:nowrap; text-align:right;" id="TBL-3-1-1"
+class="td11"><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">if</span><span
+class="cmtt-10"> (</span></span></span><span
+class="bchri7t-">EXPR</span><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">)</span></span></span> <span
+class="bchri7t-">STATEMENT </span>[<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">else</span></span></span> <span
+class="bchri7t-">STATEMENT</span><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">]</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-3-1-2"
+class="td11">if/else statement</td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-3-2-"><td style="white-space:nowrap; text-align:right;" id="TBL-3-2-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">while</span><span
+class="cmtt-10"> (</span></span></span><span
+class="bchri7t-">EXPR</span><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">)</span></span></span> <span
+class="bchri7t-">STATEMENT</span></td><td style="white-space:nowrap; text-align:left;" id="TBL-3-2-2"
+class="td11">while loop </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-3-3-"><td style="white-space:nowrap; text-align:right;" id="TBL-3-3-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">for</span><span
+class="cmtt-10"> (</span></span></span><span
+class="bchri7t-">A</span><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">;</span></span></span> <span
+class="bchri7t-">B</span><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">;</span></span></span> <span
+class="bchri7t-">C</span><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">)</span></span></span> <span
+class="bchri7t-">STATEMENT</span></td><td style="white-space:nowrap; text-align:left;" id="TBL-3-3-2"
+class="td11">for loop </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-3-4-"><td style="white-space:nowrap; text-align:right;" id="TBL-3-4-1"
+class="td11"> </td>
+</tr></table></div>
+<!--l. 354--><p class="noindent" >Scripts may use <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">break</span></span></span>/<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">continue</span></span></span> as in C. Probe handlers can return early using <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">next</span></span></span> as in <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">awk</span></span></span>. Blocks of statements are
+enclosed in <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">{</span></span></span> and <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">}</span></span></span>. In systemtap, the semicolon (<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">;</span></span></span>) is accepted as a null statement rather than as a statement terminator, so
+is only rarely<span class="footnote-mark"><a
+href="tutorial7.html#fn2x0"><sup class="textsuperscript">2</sup></a></span><a
+ id="x6-8001f2"></a>
+necessary. Shell-style (<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">#</span></span></span>), C-style (<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">/*</span><span
+class="cmtt-10"> */</span></span></span>), and C++-style (<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">//</span></span></span>) comments are all accepted.
+<!--l. 364--><p class="noindent" >Expressions look like C or <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">awk</span></span></span>, and support the usual operators, precedences, and numeric literals. Strings are
+treated as atomic values rather than arrays of characters. String concatenation is done with the dot (<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">"a"</span><span
+class="cmtt-10"> .</span><span
+class="cmtt-10"> "b"</span></span></span>).
+Some examples:
+<div class="tabular"> <table id="TBL-4" class="tabular"
+cellspacing="0" cellpadding="0"
+><colgroup id="TBL-4-1g"><col
+id="TBL-4-1"><col
+id="TBL-4-2"></colgroup><tr
+ style="vertical-align:baseline;" id="TBL-4-1-"><td style="white-space:nowrap; text-align:right;" id="TBL-4-1-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">(uid()</span><span
+class="cmtt-10"> ></span><span
+class="cmtt-10"> 100)</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-4-1-2"
+class="td11">probably an ordinary user </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-4-2-"><td style="white-space:nowrap; text-align:right;" id="TBL-4-2-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">(execname()</span><span
+class="cmtt-10"> ==</span><span
+class="cmtt-10"> "sed")</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-4-2-2"
+class="td11">current process is sed </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-4-3-"><td style="white-space:nowrap; text-align:right;" id="TBL-4-3-1"
+class="td11"><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">(cpu()</span><span
+class="cmtt-10"> ==</span><span
+class="cmtt-10"> 0</span><span
+class="cmtt-10"> &&</span><span
+class="cmtt-10"> gettimeofday_s()</span><span
+class="cmtt-10"> ></span><span
+class="cmtt-10"> 1140498000)</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-4-3-2"
+class="td11">after Feb. 21, 2006, on CPU 0</td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-4-4-"><td style="white-space:nowrap; text-align:right;" id="TBL-4-4-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">"hello"</span><span
+class="cmtt-10"> .</span><span
+class="cmtt-10"> "</span><span
+class="cmtt-10"> "</span><span
+class="cmtt-10"> .</span><span
+class="cmtt-10"> "world"</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-4-4-2"
+class="td11">a string in three easy pieces </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-4-5-"><td style="white-space:nowrap; text-align:right;" id="TBL-4-5-1"
+class="td11"> </td>
+</tr></table></div>
+<!--l. 376--><p class="noindent" >Variables may be used as well. Just pick a name, assign to it, and use it in expressions. They are automatically
+initialized and declared. The type of each identifier – string vs. number – is automatically inferred by systemtap from
+the kinds of operators and literals used on it. Any inconsistencies will be reported as errors. Conversion between
+string and number types is done through explicit function calls.
+
+<div class="tabular"> <table id="TBL-5" class="tabular"
+cellspacing="0" cellpadding="0"
+><colgroup id="TBL-5-1g"><col
+id="TBL-5-1"><col
+id="TBL-5-2"></colgroup><tr
+ style="vertical-align:baseline;" id="TBL-5-1-"><td style="white-space:nowrap; text-align:right;" id="TBL-5-1-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">foo</span><span
+class="cmtt-10"> =</span><span
+class="cmtt-10"> gettimeofday_s()</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-5-1-2"
+class="td11">foo is a number </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-5-2-"><td style="white-space:nowrap; text-align:right;" id="TBL-5-2-1"
+class="td11"><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">bar</span><span
+class="cmtt-10"> =</span><span
+class="cmtt-10"> "/usr/bin/"</span><span
+class="cmtt-10"> .</span><span
+class="cmtt-10"> execname()</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-5-2-2"
+class="td11">bar is a string </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-5-3-"><td style="white-space:nowrap; text-align:right;" id="TBL-5-3-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">c++</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-5-3-2"
+class="td11">c is a number </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-5-4-"><td style="white-space:nowrap; text-align:right;" id="TBL-5-4-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">s</span><span
+class="cmtt-10"> =</span><span
+class="cmtt-10"> sprint(2345)</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-5-4-2"
+class="td11">s becomes the string ”2345”</td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-5-5-"><td style="white-space:nowrap; text-align:right;" id="TBL-5-5-1"
+class="td11"> </td>
+</tr></table></div>
+<!--l. 399--><p class="noindent" >By default, variables are local to the probe they are used in. That is, they are initialized, used, and
+disposed of at each probe handler invocation. To share variables between probes, declare them global
+anywhere in the script. Because of possible concurrency (multiple probe handlers running on different
+CPUs), each global variable used by a probe is automatically read- or write-locked while the handler is
+running.
+<!--l. 412--><p class="noindent" ><hr class="figure"><div class="figure"
+>
+
+<a
+ id="x6-80024"></a>
+
+<div class="center"
+>
+<!--l. 413--><p class="noindent" >
+<div class="fbox"><div class="minipage"><pre class="verbatim" id="verbatim-5">
+# cat timer-jiffies.stp
+global count_jiffies, count_ms
+probe timer.jiffies(100) { count_jiffies ++ }
+probe timer.ms(100) { count_ms ++ }
+probe timer.ms(12345)
+{
+  hz=(1000*count_jiffies) / count_ms
+  printf ("jiffies:ms ratio %d:%d => CONFIG_HZ=%d\n",
+    count_jiffies, count_ms, hz)
+  exit ()
+}
+# stap timer-jiffies.stp
+jiffies:ms ratio 30:123 => CONFIG_HZ=243
+</pre>
+<!--l. 429--><p class="nopar" > </div></div>
+</div>
+<br /> <div class="caption"
+><span class="id">Figure 4: </span><span
+class="content">Experimentally measuring <span
+class="cmtt-10">CONFIG</span><span
+class="cmtt-10">_HZ</span>.</span></div><!--tex4ht:label?: x6-80024 -->
+
+<!--l. 433--><p class="noindent" ></div><hr class="endfigure">
+<h4 class="subsectionHead"><span class="titlemark">3.2 </span> <a
+ id="x6-90003.2"></a>Target variables</h4>
+<!--l. 437--><p class="noindent" >A class of special “target variables” allow access to the probe point context. In a symbolic debugger, when you’re
+stopped at a breakpoint, you can print values from the program’s context. In systemtap scripts, for those probe points
+that match with specific executable point (rather than an asynchronous event like a timer), you can do the
+same.
+<!--l. 446--><p class="noindent" >In addition, you can take their address (the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">&</span></span></span> operator), pretty-print structures (the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">$</span></span></span> and <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">$$</span></span></span> suffix), pretty-print
+multiple variables in scope (the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">$$vars</span></span></span> and related variables), or cast pointers to their types (the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">@cast</span></span></span>
+operator), or test their existence / resolvability (the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">@defined</span></span></span> operator). Read about these in the manual
+pages.
+<!--l. 453--><p class="noindent" >To know which variables are likely to be available, you will need to be familiar with the kernel source you are
+probing. In addition, you will need to check that the compiler has not optimized those values into
+unreachable nonexistence. You can use <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">stap</span><span
+class="cmtt-10"> -L</span><span
+class="cmtt-10"> PROBEPOINT</span></span></span> to enumerate the variables available
+there.
+<!--l. 459--><p class="noindent" >Let’s say that you are trying to trace filesystem reads/writes to a particular device/inode. From your knowledge of
+the kernel, you know that two functions of interest could be <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">vfs_read</span></span></span> and <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">vfs_write</span></span></span>. Each takes a <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">struct</span><span
+class="cmtt-10"> file</span><span
+class="cmtt-10"> *</span></span></span>
+argument, inside there is either a <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">struct</span><span
+class="cmtt-10"> dentry</span><span
+class="cmtt-10"> *</span></span></span> or <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">struct</span><span
+class="cmtt-10"> path</span><span
+class="cmtt-10"> *</span></span></span> which has a <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">struct</span><span
+class="cmtt-10"> dentry</span><span
+class="cmtt-10"> *</span></span></span>. The
+<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">struct</span><span
+class="cmtt-10"> dentry</span><span
+class="cmtt-10"> *</span></span></span> contains a <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">struct</span><span
+class="cmtt-10"> inode</span><span
+class="cmtt-10"> *</span></span></span>, and so on. Systemtap allows limited dereferencing of such
+pointer chains. Two functions, <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">user_string</span></span></span> and <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">kernel_string</span></span></span>, can copy <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">char</span><span
+class="cmtt-10"> *</span></span></span> target variables into
+systemtap strings. Figure <a
+href="#x6-90015">5<!--tex4ht:ref: fig:inode-watch --></a> demonstrates one way to monitor a particular file (identified by device number
+and inode number). The script selects the appropriate variants of <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">dev_nr</span></span></span> and<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">inode_nr</span></span></span> based on the
+kernel version. This example also demonstrates passing numeric command-line arguments (<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">$1</span></span></span> etc.) into
+scripts.
+<!--l. 478--><p class="noindent" ><hr class="figure"><div class="figure"
+>
+
+<a
+ id="x6-90015"></a>
+
+<div class="center"
+>
+<!--l. 479--><p class="noindent" >
+<div class="fbox"><div class="minipage"><pre class="verbatim" id="verbatim-6">
+# cat inode-watch.stp
+probe kernel.function ("vfs_write"),
+      kernel.function ("vfs_read")
+{
+  if (@defined($file->f_path->dentry)) {
+    dev_nr = $file->f_path->dentry->d_inode->i_sb->s_dev
+    inode_nr = $file->f_path->dentry->d_inode->i_ino
+  } else {
+    dev_nr = $file->f_dentry->d_inode->i_sb->s_dev
+    inode_nr = $file->f_dentry->d_inode->i_ino
+  }
+  if (dev_nr == ($1 << 20 | $2) # major/minor device
+      && inode_nr == $3)
+    printf ("%s(%d) %s 0x%x/%u\n",
+      execname(), pid(), ppfunc(), dev_nr, inode_nr)
+}
+# stat -c "%D %i" /etc/crontab
+fd03 133099
+# stap inode-watch.stp 0xfd 3 133099
+more(30789) vfs_read 0xfd00003/133099
+more(30789) vfs_read 0xfd00003/133099
+</pre>
+<!--l. 503--><p class="nopar" > </div></div>
+</div>
+<br /> <div class="caption"
+><span class="id">Figure 5: </span><span
+class="content">Watching for reads/writes to a particular file.</span></div><!--tex4ht:label?: x6-90015 -->
+
+<!--l. 508--><p class="noindent" ></div><hr class="endfigure">
+<h4 class="subsectionHead"><span class="titlemark">3.3 </span> <a
+ id="x6-100003.3"></a>Functions</h4>
+<!--l. 512--><p class="noindent" >Functions are conveniently packaged reusable software: it would be a shame to have to duplicate a complex
+condition expression or logging directive in every placed it’s used. So, systemtap lets you define functions of your
+own. Like global variables, systemtap functions may be defined anywhere in the script. They may take any number of
+string or numeric arguments (by value), and may return a single string or number. The parameter types are inferred
+as for ordinary variables, and must be consistent throughout the program. Local and global script variables are
+available, but target variables are <span
+class="bchri7t-">not</span>. That’s because there is no specific debugging-level context associated with a
+function.
+<!--l. 527--><p class="noindent" >A function is defined with the keyword <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">function</span></span></span> followed by a name. Then comes a comma-separated formal
+argument list (just a list of variable names). The <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">{</span><span
+class="cmtt-10"> }</span></span></span>-enclosed body consists of any list of statements, including
+expressions that call functions. Recursion is possible, up to a nesting depth limit. Figure <a
+href="#x6-100016">6<!--tex4ht:ref: fig:functions --></a> displays function
+syntax.
+<!--l. 535--><p class="noindent" ><hr class="figure"><div class="figure"
+>
+
+<a
+ id="x6-100016"></a>
+
+<div class="center"
+>
+<!--l. 536--><p class="noindent" >
+<div class="fbox"><div class="minipage"><pre class="verbatim" id="verbatim-7">
+# Red Hat convention; see /etc/login.defs UID_MIN
+function system_uid_p (u) { return u < 500 }
+# kernel device number assembly macro
+function makedev (major,minor) { return major << 20 | minor }
+function trace_common ()
+{
+  printf("%d %s(%d)", gettimeofday_s(), execname(), pid())
+  # no return value necessary
+}
+function fibonacci (i)
+{
+  if (i < 1) return 0
+  else if (i < 2) return 1
+  else return fibonacci(i-1) + fibonacci(i-2)
+}
+</pre>
+<!--l. 556--><p class="nopar" > </div></div>
+</div>
+<br /> <div class="caption"
+><span class="id">Figure 6: </span><span
+class="content">Some functions of dubious utility.</span></div><!--tex4ht:label?: x6-100016 -->
+
+<!--l. 560--><p class="noindent" ></div><hr class="endfigure">
+<h4 class="subsectionHead"><span class="titlemark">3.4 </span> <a
+ id="x6-110003.4"></a>Arrays</h4>
+<!--l. 564--><p class="noindent" >Often, probes will want to share data that cannot be represented as a simple scalar value. Much data is naturally
+tabular in nature, indexed by some tuple of thread numbers, processor ids, names, time, and so on. Systemtap offers
+associative arrays for this purpose. These arrays are implemented as hash tables with a maximum size that is fixed at
+startup. Because they are too large to be created dynamically for individual probes handler runs, they must be
+declared as global.
+<div class="tabular"> <table id="TBL-6" class="tabular"
+cellspacing="0" cellpadding="0"
+><colgroup id="TBL-6-1g"><col
+id="TBL-6-1"><col
+id="TBL-6-2"></colgroup><tr
+ style="vertical-align:baseline;" id="TBL-6-1-"><td style="white-space:nowrap; text-align:right;" id="TBL-6-1-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">global</span><span
+class="cmtt-10"> a</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-6-1-2"
+class="td11">declare global scalar or array variable </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-6-2-"><td style="white-space:nowrap; text-align:right;" id="TBL-6-2-1"
+class="td11"><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">global</span><span
+class="cmtt-10"> b[400]</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-6-2-2"
+class="td11">declare array, reserving space for up to 400 tuples</td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-6-3-"><td style="white-space:nowrap; text-align:right;" id="TBL-6-3-1"
+class="td11"> </td>
+</tr></table></div>
+<!--l. 581--><p class="noindent" >The basic operations for arrays are setting and looking up elements. These are expressed in <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">awk</span></span></span> syntax: the array
+name followed by an opening <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">[</span></span></span> bracket, a comma-separated list of index expressions, and a closing <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">]</span></span></span> bracket.
+Each index expression may be string or numeric, as long as it is consistently typed throughout the
+script.
+<div class="tabular"> <table id="TBL-7" class="tabular"
+cellspacing="0" cellpadding="0"
+><colgroup id="TBL-7-1g"><col
+id="TBL-7-1"><col
+id="TBL-7-2"></colgroup><tr
+ style="vertical-align:baseline;" id="TBL-7-1-"><td style="white-space:nowrap; text-align:right;" id="TBL-7-1-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">foo</span><span
+class="cmtt-10"> [4,"hello"]</span><span
+class="cmtt-10"> ++</span><span
+class="cmtt-10"> </span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-7-1-2"
+class="td11">increment the named array slot</td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-7-2-"><td style="white-space:nowrap; text-align:right;" id="TBL-7-2-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">processusage</span><span
+class="cmtt-10"> [uid(),execname()]</span><span
+class="cmtt-10"> ++</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-7-2-2"
+class="td11">update a statistic </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-7-3-"><td style="white-space:nowrap; text-align:right;" id="TBL-7-3-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">times</span><span
+class="cmtt-10"> [tid()]</span><span
+class="cmtt-10"> =</span><span
+class="cmtt-10"> get_cycles()</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-7-3-2"
+class="td11">set a timestamp reference point</td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-7-4-"><td style="white-space:nowrap; text-align:right;" id="TBL-7-4-1"
+class="td11"><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">delta</span><span
+class="cmtt-10"> =</span><span
+class="cmtt-10"> get_cycles()</span><span
+class="cmtt-10"> -</span><span
+class="cmtt-10"> times</span><span
+class="cmtt-10"> [tid()]</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-7-4-2"
+class="td11">compute a timestamp delta </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-7-5-"><td style="white-space:nowrap; text-align:right;" id="TBL-7-5-1"
+class="td11"> </td>
+</tr></table></div>
+<!--l. 597--><p class="noindent" >Array elements that have not been set <span
+class="bchri7t-">may </span>be fetched, and return a dummy null value (zero or an empty string) as
+appropriate. However, assigning a null value does not delete the element: an explicit <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">delete</span></span></span> statement is required.
+Systemtap provides syntactic sugar for these operations, in the form of explicit membership testing and
+deletion.
+<div class="tabular"> <table id="TBL-8" class="tabular"
+cellspacing="0" cellpadding="0"
+><colgroup id="TBL-8-1g"><col
+id="TBL-8-1"><col
+id="TBL-8-2"></colgroup><tr
+ style="vertical-align:baseline;" id="TBL-8-1-"><td style="white-space:nowrap; text-align:right;" id="TBL-8-1-1"
+class="td11"><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">if</span><span
+class="cmtt-10"> ([4,"hello"]</span><span
+class="cmtt-10"> in</span><span
+class="cmtt-10"> foo)</span><span
+class="cmtt-10"> {</span><span
+class="cmtt-10"> }</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-8-1-2"
+class="td11">membership test </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-8-2-"><td style="white-space:nowrap; text-align:right;" id="TBL-8-2-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">delete</span><span
+class="cmtt-10"> times[tid()]</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-8-2-2"
+class="td11">deletion of a single element</td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-8-3-"><td style="white-space:nowrap; text-align:right;" id="TBL-8-3-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">delete</span><span
+class="cmtt-10"> times</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-8-3-2"
+class="td11">deletion of all elements </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-8-4-"><td style="white-space:nowrap; text-align:right;" id="TBL-8-4-1"
+class="td11"> </td>
+</tr></table></div>
+<!--l. 612--><p class="noindent" >One final and important operation is iteration over arrays. This uses the keyword <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">foreach</span></span></span>. Like <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">awk</span></span></span>, this creates a
+loop that <span
+class="bchri7t-">iterates over key tuples </span>of an array, not just <span
+class="bchri7t-">values</span>. In addition, the iteration may be <span
+class="bchri7t-">sorted </span>by any single key
+or the value by adding an extra <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">+</span></span></span> or <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">-</span></span></span> code.
+<!--l. 618--><p class="noindent" >The <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">break</span></span></span> and <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">continue</span></span></span> statements work inside <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">foreach</span></span></span> loops, too. Since arrays can be large but probe handlers
+must not run for long, it is a good idea to exit iteration early if possible. The <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">limit</span></span></span> option in the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">foreach</span></span></span> expression
+is one way. For simplicity, systemtap forbids any <span
+class="bchri7t-">modification </span>of an array while it is being iterated using a
+<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">foreach</span></span></span>.
+<div class="tabular"> <table id="TBL-9" class="tabular"
+cellspacing="0" cellpadding="0"
+><colgroup id="TBL-9-1g"><col
+id="TBL-9-1"><col
+id="TBL-9-2"></colgroup><tr
+ style="vertical-align:baseline;" id="TBL-9-1-"><td style="white-space:nowrap; text-align:right;" id="TBL-9-1-1"
+class="td11"><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">foreach</span><span
+class="cmtt-10"> (x</span><span
+class="cmtt-10"> =</span><span
+class="cmtt-10"> [a,b]</span><span
+class="cmtt-10"> in</span><span
+class="cmtt-10"> foo)</span><span
+class="cmtt-10"> {</span><span
+class="cmtt-10"> fuss_with(x)</span><span
+class="cmtt-10"> }</span></span></span></td><td style="white-space:normal; text-align:left;" id="TBL-9-1-2"
+class="td11"><!--l. 627--><p class="noindent" >simple loop in arbitrary sequence </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-9-2-"><td style="white-space:nowrap; text-align:right;" id="TBL-9-2-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">foreach</span><span
+class="cmtt-10"> ([a,b]</span><span
+class="cmtt-10"> in</span><span
+class="cmtt-10"> foo+</span><span
+class="cmtt-10"> limit</span><span
+class="cmtt-10"> 5)</span><span
+class="cmtt-10"> {</span><span
+class="cmtt-10"> }</span></span></span></td><td style="white-space:normal; text-align:left;" id="TBL-9-2-2"
+class="td11"><!--l. 628--><p class="noindent" >loop in increasing sequence of value, stop
+after 5 </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-9-3-"><td style="white-space:nowrap; text-align:right;" id="TBL-9-3-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">foreach</span><span
+class="cmtt-10"> ([a-,b]</span><span
+class="cmtt-10"> in</span><span
+class="cmtt-10"> foo)</span><span
+class="cmtt-10"> {</span><span
+class="cmtt-10"> }</span></span></span></td><td style="white-space:normal; text-align:left;" id="TBL-9-3-2"
+class="td11"><!--l. 629--><p class="noindent" >loop in decreasing sequence of first key </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-9-4-"><td style="white-space:nowrap; text-align:right;" id="TBL-9-4-1"
+class="td11"> </td>
+</tr></table></div>
+<!--l. 632--><p class="noindent" >
+<h4 class="subsectionHead"><span class="titlemark">3.5 </span> <a
+ id="x6-120003.5"></a>Aggregates</h4>
+
+<!--l. 634--><p class="noindent" >When we said above that values can only be strings or numbers, we lied a little. There is a third type: statistics
+aggregates, or aggregates for short. Instances of this type are used to collect statistics on numerical values, where it
+is important to accumulate new data quickly (<span
+class="bchri7t-">without </span>exclusive locks) and in large volume (storing only aggregated
+stream statistics). This type only makes sense for global variables, and may be stored individually or as elements of
+an array.
+<!--l. 645--><p class="noindent" >To add a value to a statistics aggregate, systemtap uses the special operator <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10"><<<</span></span></span>. Think of it like C++’s <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10"><<</span></span></span> output
+streamer: the left hand side object accumulates the data sample given on the right hand side. This operation is
+efficient (taking a shared lock) because the aggregate values are kept separately on each processor, and are only
+aggregated across processors on request.
+
+<pre class="verbatim" id="verbatim-8">
+a <<< delta_timestamp
+writes[execname()] <<< count
+</pre>
+<!--l. 655--><p class="nopar" >
+<!--l. 657--><p class="noindent" >To read the aggregate value, special functions are available to extract a selected statistical function. <span
+class="bchri7t-">The aggregate</span>
+<span
+class="bchri7t-">value cannot be read by simply naming it as if it were an ordinary variable. </span>These operations take an exclusive lock on
+the respective globals, and should therefore be relatively rare. The simple ones are: <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">@min</span></span></span>, <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">@max</span></span></span>, <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">@count</span></span></span>,
+<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">@avg</span></span></span>, and <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">@sum</span></span></span>, and evaluate to a single number. In addition, histograms of the data stream may be
+extracted using the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">@hist_log</span></span></span> and <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">@hist_linear</span></span></span>. These evaluate to a special sort of array that may at
+present<span class="footnote-mark"><a
+href="tutorial8.html#fn3x0"><sup class="textsuperscript">3</sup></a></span><a
+ id="x6-12001f3"></a>
+only be printed.
+<div class="tabular"> <table id="TBL-10" class="tabular"
+cellspacing="0" cellpadding="0"
+><colgroup id="TBL-10-1g"><col
+id="TBL-10-1"><col
+id="TBL-10-2"></colgroup><tr
+ style="vertical-align:baseline;" id="TBL-10-1-"><td style="white-space:nowrap; text-align:right;" id="TBL-10-1-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">@avg(a)</span></span></span></td><td style="white-space:normal; text-align:left;" id="TBL-10-1-2"
+class="td11"><!--l. 672--><p class="noindent" >the average of all the values accumulated into <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">a</span></span></span> </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-10-2-"><td style="white-space:nowrap; text-align:right;" id="TBL-10-2-1"
+class="td11"><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">print(@hist_linear(a,0,100,10))</span></span></span></td><td style="white-space:normal; text-align:left;" id="TBL-10-2-2"
+class="td11"><!--l. 674--><p class="noindent" >print an “ascii art” linear histogram of the same data
+stream, bounds <span
+class="cmr-10">0</span><span
+class="cmmi-10">…</span><span
+class="cmr-10">100</span>, bucket width is <span
+class="cmr-10">10</span> </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-10-3-"><td style="white-space:nowrap; text-align:right;" id="TBL-10-3-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">@count(writes["zsh"])</span></span></span></td><td style="white-space:normal; text-align:left;" id="TBL-10-3-2"
+class="td11"><!--l. 677--><p class="noindent" >the number of times “zsh” ran the probe handler </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-10-4-"><td style="white-space:nowrap; text-align:right;" id="TBL-10-4-1"
+class="td11"><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">print(@hist_log(writes["zsh"]))</span></span></span></td><td style="white-space:normal; text-align:left;" id="TBL-10-4-2"
+class="td11"><!--l. 679--><p class="noindent" >print an “ascii art” logarithmic histogram of the same
+data stream </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-10-5-"><td style="white-space:nowrap; text-align:right;" id="TBL-10-5-1"
+class="td11"> </td>
+</tr></table></div>
+<h4 class="subsectionHead"><span class="titlemark">3.6 </span> <a
+ id="x6-130003.6"></a>Safety</h4>
+<!--l. 686--><p class="noindent" >The full expressivity of the scripting language raises good questions of safety. Here is a set of Q&A:
+<!--l. 689--><p class="noindent" >
+ <dl class="description"><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">What about infinite loops? recursion? </span>A probe handler is bounded in time. The C code generated by
+ systemtap includes explicit checks that limit the total number of statements executed to a small number.
+ A similar limit is imposed on the nesting depth of function calls. When either limit is exceeded, that
+ probe handler cleanly aborts and signals an error. The systemtap session is normally configured to
+ abort as a whole at that time.
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">What about running out of memory? </span>No dynamic memory allocation whatsoever takes place during the
+ execution of probe handlers. Arrays, function contexts, and buffers are allocated during initialization.
+ These resources may run out during a session, and generally result in errors.
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">What about locking? </span>If multiple probes seek conflicting locks on the same global variables, one or more
+ of them will time out, and be aborted. Such events are tallied as “skipped” probes, and a count is
+ displayed at session end. A configurable number of skipped probes can trigger an abort of the session.
+
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">What about null pointers? division by zero? </span>The C code generated by systemtap translates potentially
+ dangerous operations to routines that check their arguments at run time. These signal errors if they
+ are invalid. Many arithmetic and string operations silently overflow if the results exceed representation
+ limits.
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">What about bugs in the translator? compiler? </span>While bugs in the translator, or the runtime layer certainly
+ exist<span class="footnote-mark"><a
+href="tutorial9.html#fn4x0"><sup class="textsuperscript">4</sup></a></span><a
+ id="x6-13001f4"></a>,
+ our test suite gives some assurance. Plus, the entire generated C code may be inspected (try the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">-p3</span></span></span>
+ option). Compiler bugs are unlikely to be of any greater concern for systemtap than for the kernel as a
+ whole. In other words, if it was reliable enough to build the kernel, it will build the systemtap modules
+ properly too.
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">Is that the whole truth? </span>In practice, there are several weak points in systemtap and the underlying kprobes
+ system at the time of writing. Putting probes indiscriminately into unusually sensitive parts of the
+ kernel (low level context switching, interrupt dispatching) has reportedly caused crashes in the past.
+ We are fixing these bugs as they are found, and constructing a probe point “blocklist”, but it is not
+ complete.
+ </dd></dl>
+<!--l. 741--><p class="noindent" >
+<h4 class="subsectionHead"><span class="titlemark">3.7 </span> <a
+ id="x6-140003.7"></a>Exercises</h4>
+<!--l. 742--><p class="noindent" >
+ <ol class="enumerate1" >
+ <li
+ class="enumerate" id="x6-14002x1">Alter the last probe in <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">timer-jiffies.stp</span></span></span> to reset the counters and continue reporting instead of
+ exiting.
+ </li>
+ <li
+ class="enumerate" id="x6-14004x2">Write a script that, every ten seconds, displays the top five most frequent users of <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">open</span></span></span> system call
+ during that interval.
+ </li>
+ <li
+ class="enumerate" id="x6-14006x3">Write a script that experimentally measures the speed of the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">get_cycles()</span></span></span> counter on each processor.
+ </li>
+ <li
+ class="enumerate" id="x6-14008x4">Use any suitable probe point to get an approximate profile of process CPU usage: which processes/users
+ use how much of each CPU.</li></ol>
+
+<!--l. 756--><div class="crosslinks"><p class="noindent">[<a
+href="tutorialse2.html" >prev</a>] [<a
+href="tutorialse2.html#tailtutorialse2.html" >prev-tail</a>] [<a
+href="tutorialse3.html" >front</a>] [<a
+href="tutorial.html#tutorialse4.html" >up</a>] </p></div>
+<!--l. 756--><p class="noindent" ><a
+ id="tailtutorialse3.html"></a>
+</body></html>
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd">
+<html >
+<head><title>Tapsets</title>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta name="generator" content="TeX4ht (https://tug.org/tex4ht/)">
+<meta name="originator" content="TeX4ht (https://tug.org/tex4ht/)">
+<!-- html,2 -->
+<meta name="src" content="tutorial.tex">
+<link rel="stylesheet" type="text/css" href="tutorial.css">
+</head><body
+>
+<!--l. 756--><div class="crosslinks"><p class="noindent">[<a
+href="tutorialse3.html" >prev</a>] [<a
+href="tutorialse3.html#tailtutorialse3.html" >prev-tail</a>] [<a
+href="tutorialse3.html#tailtutorialse4.html">tail</a>] [<a
+href="tutorial.html# " >up</a>] </p></div>
+<h3 class="sectionHead"><span class="titlemark">4 </span> <a
+ id="x10-150004"></a>Tapsets</h3>
+<!--l. 758--><p class="noindent" >After writing enough analysis scripts for yourself, you may become known as an expert to your colleagues, who will
+want to use your scripts. Systemtap makes it possible to share in a controlled manner; to build libraries of scripts
+that build on each other. In fact, all of the functions (<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">pid()</span></span></span>, etc.) used in the scripts above come from
+tapset scripts like that. A “tapset” is just a script that designed for reuse by installation into a special
+directory.
+<!--l. 766--><p class="noindent" >
+<h4 class="subsectionHead"><span class="titlemark">4.1 </span> <a
+ id="x10-160004.1"></a>Automatic selection</h4>
+<!--l. 768--><p class="noindent" >Systemtap attempts to resolve references to global symbols (probes, functions, variables) that are not defined within
+the script by a systematic search through the tapset library for scripts that define those symbols. Tapset scripts
+are installed under the default directory named <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">/usr/share/systemtap/tapset</span></span></span>. A user may give
+additional directories with the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">-I</span><span
+class="cmtt-10"> DIR</span></span></span> option. Systemtap searches these directories for script (<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">.stp</span></span></span>)
+files.
+<!--l. 776--><p class="noindent" >The search process includes subdirectories that are specialized for a particular kernel version and/or architecture,
+and ones that name only larger kernel families. Naturally, the search is ordered from specific to general, as shown in
+Figure <a
+href="#x10-160017">7<!--tex4ht:ref: fig:tapset-search --></a>.
+<!--l. 784--><p class="noindent" ><hr class="figure"><div class="figure"
+>
+
+<a
+ id="x10-160017"></a>
+
+<div class="center"
+>
+<!--l. 785--><p class="noindent" >
+<div class="fbox"><div class="minipage"><pre class="verbatim" id="verbatim-9">
+# stap -p1 -vv -e ’probe begin { }’ > /dev/null
+Created temporary directory "/tmp/staplnEBh7"
+Searched ’/usr/share/systemtap/tapset/2.6.15/i686/*.stp’, match count 0
+Searched ’/usr/share/systemtap/tapset/2.6.15/*.stp’, match count 0
+Searched ’/usr/share/systemtap/tapset/2.6/i686/*.stp’, match count 0
+Searched ’/usr/share/systemtap/tapset/2.6/*.stp’, match count 0
+Searched ’/usr/share/systemtap/tapset/i686/*.stp’, match count 1
+Searched ’/usr/share/systemtap/tapset/*.stp’, match count 12
+Pass 1: parsed user script and 13 library script(s) in 350usr/10sys/375real ms.
+Running rm -rf /tmp/staplnEBh7
+</pre>
+<!--l. 797--><p class="nopar" > </div></div>
+</div>
+<br /> <div class="caption"
+><span class="id">Figure 7: </span><span
+class="content">Listing the tapset search path.</span></div><!--tex4ht:label?: x10-160017 -->
+
+<!--l. 801--><p class="noindent" ></div><hr class="endfigure">
+<!--l. 803--><p class="noindent" >When a script file is found that <span
+class="bchri7t-">defines </span>one of the undefined symbols, that <span
+class="bchri7t-">entire file </span>is added to the probing session
+being analyzed. This search is repeated until no more references can become satisfied. Systemtap signals an error if
+any are still unresolved.
+<!--l. 808--><p class="noindent" >This mechanism enables several programming idioms. First, it allows some global symbols to be defined only
+for applicable kernel version/architecture pairs, and cause an error if their use is attempted on an
+inapplicable host. Similarly, the same symbol can be defined differently depending on kernels, in much
+the same way that different kernel <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">include/asm/ARCH/</span></span></span> files contain macros that provide a porting
+layer.
+<!--l. 816--><p class="noindent" >Another use is to separate the default parameters of a tapset routine from its implementation. For example, consider
+a tapset that defines code for relating elapsed time intervals to process scheduling activities. The data collection code
+can be generic with respect to which time unit (jiffies, wall-clock seconds, cycle counts) it can use. It should have a
+default, but should not require additional run-time checks to let a user choose another. Figure <a
+href="#x10-160028">8<!--tex4ht:ref: fig:tapset-default --></a> shows a
+way.
+<!--l. 825--><p class="noindent" ><hr class="figure"><div class="figure"
+>
+
+<a
+ id="x10-160028"></a>
+
+<div class="center"
+>
+<!--l. 826--><p class="noindent" >
+<div class="fbox"><div class="minipage"><pre class="verbatim" id="verbatim-10">
+# cat tapset/time-common.stp
+global __time_vars
+function timer_begin (name) { __time_vars[name] = __time_value () }
+function timer_end (name) { return __time_value() - __time_vars[name] }
+# cat tapset/time-default.stp
+function __time_value () { return gettimeofday_us () }
+# cat tapset-time-user.stp
+probe begin
+{
+  timer_begin ("bench")
+  for (i=0; i<100; i++) ;
+  printf ("%d cycles\n", timer_end ("bench"))
+  exit ()
+}
+function __time_value () { return get_ticks () } # override for greater precision
+</pre>
+<!--l. 846--><p class="nopar" > </div></div>
+</div>
+<br /> <div class="caption"
+><span class="id">Figure 8: </span><span
+class="content">Providing an overrideable default.</span></div><!--tex4ht:label?: x10-160028 -->
+
+<!--l. 850--><p class="noindent" ></div><hr class="endfigure">
+<!--l. 852--><p class="noindent" >A tapset that exports only <span
+class="bchri7t-">data </span>may be as useful as ones that exports functions or probe point aliases (see below).
+Such global data can be computed and kept up-to-date using probes internal to the tapset. Any outside reference to
+the global variable would incidentally activate all the required probes.
+<h4 class="subsectionHead"><span class="titlemark">4.2 </span> <a
+ id="x10-170004.2"></a>Probe point aliases</h4>
+<!--l. 861--><p class="noindent" >Probe point aliases allow creation of new probe points from existing ones. This is useful if the new probe points are
+named to provide a higher level of abstraction. For example, the system-calls tapset defines probe point aliases of the
+form <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">syscall.open</span></span></span> etc., in terms of lower level ones like <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">kernel.function("sys_open")</span></span></span>. Even if some future
+kernel renames <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">sys_open</span></span></span>, the aliased name can remain valid.
+<!--l. 869--><p class="noindent" >A probe point alias definition looks like a normal probe. Both start with the keyword <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">probe</span></span></span> and have a probe handler
+statement block at the end. But where a normal probe just lists its probe points, an alias creates a new name using
+the assignment (<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">=</span></span></span>) operator. Another probe that names the new probe point will create an actual probe, with the
+handler of the alias <span
+class="bchri7t-">prepended</span>.
+<!--l. 876--><p class="noindent" >This prepending behavior serves several purposes. It allows the alias definition to “preprocess” the
+context of the probe before passing control to the user-specified handler. This has several possible uses:
+<!--tex4ht:inline--><div class="tabular"> <table id="TBL-11" class="tabular"
+cellspacing="0" cellpadding="0"
+><colgroup id="TBL-11-1g"><col
+id="TBL-11-1"><col
+id="TBL-11-2"></colgroup><tr
+ style="vertical-align:baseline;" id="TBL-11-1-"><td style="white-space:nowrap; text-align:right;" id="TBL-11-1-1"
+class="td11"><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">if</span><span
+class="cmtt-10"> ($flag1</span><span
+class="cmtt-10"> !=</span><span
+class="cmtt-10"> $flag2)</span><span
+class="cmtt-10"> next</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-11-1-2"
+class="td11">skip probe unless given condition is met </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-11-2-"><td style="white-space:nowrap; text-align:right;" id="TBL-11-2-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">name</span><span
+class="cmtt-10"> =</span><span
+class="cmtt-10"> "foo"</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-11-2-2"
+class="td11">supply probe-describing values </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-11-3-"><td style="white-space:nowrap; text-align:right;" id="TBL-11-3-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">var</span><span
+class="cmtt-10"> =</span><span
+class="cmtt-10"> $var</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-11-3-2"
+class="td11">extract target variable to plain local variable</td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-11-4-"><td style="white-space:nowrap; text-align:right;" id="TBL-11-4-1"
+class="td11"> </td>
+</tr></table></div>
+<!--l. 885--><p class="noindent" >Figure <a
+href="#x10-170019">9<!--tex4ht:ref: fig:probe-alias --></a> demonstrates a probe point alias definition as well as its use. It demonstrates how a single probe point alias
+can expand to multiple probe points, even to other aliases. It also includes probe point wildcarding. These functions
+are designed to compose sensibly.
+<!--l. 891--><p class="noindent" ><hr class="figure"><div class="figure"
+>
+
+<a
+ id="x10-170019"></a>
+
+<div class="center"
+>
+<!--l. 892--><p class="noindent" >
+<div class="fbox"><div class="minipage"><pre class="verbatim" id="verbatim-11">
+# cat probe-alias.stp
+probe syscallgroup.io = syscall.open, syscall.close,
+                        syscall.read, syscall.write
+{ groupname = "io" }
+probe syscallgroup.process = syscall.fork, syscall.execve
+{ groupname = "process" }
+probe syscallgroup.*
+{ groups [execname() . "/" . groupname] ++ }
+probe end
+{
+  foreach (eg+ in groups)
+    printf ("%s: %d\n", eg, groups[eg])
+}
+global groups
+# stap probe-alias.stp
+05-wait_for_sys/io: 19
+10-udev.hotplug/io: 17
+20-hal.hotplug/io: 12
+X/io: 73
+apcsmart/io: 59
+[...]
+make/io: 515
+make/process: 16
+[...]
+xfce-mcs-manage/io: 3
+xfdesktop/io: 5
+[...]
+xmms/io: 7070
+zsh/io: 78
+zsh/process: 5
+</pre>
+<!--l. 929--><p class="nopar" > </div></div>
+</div>
+<br /> <div class="caption"
+><span class="id">Figure 9: </span><span
+class="content">Classified system call activity.</span></div><!--tex4ht:label?: x10-170019 -->
+
+<!--l. 933--><p class="noindent" ></div><hr class="endfigure">
+<h4 class="subsectionHead"><span class="titlemark">4.3 </span> <a
+ id="x10-180004.3"></a>Embedded C</h4>
+<!--l. 938--><p class="noindent" >Sometimes, a tapset needs provide data values from the kernel that cannot be extracted using ordinary target
+variables (<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">$var</span></span></span>). This may be because the values are in complicated data structures, may require lock
+awareness, or are defined by layers of macros. Systemtap provides an “escape hatch” to go beyond what the
+language can safely offer. In certain contexts, you may embed plain raw C in tapsets, exchanging power
+for the safety guarantees listed in section <a
+href="tutorialse3.html#x6-130003.6">3.6<!--tex4ht:ref: sec:safety --></a>. End-user scripts <span
+class="bchri7t-">may not </span>include embedded C code,
+unless systemtap is run with the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">-g</span></span></span> (“guru” mode) option. Tapset scripts get guru mode privileges
+automatically.
+<!--l. 951--><p class="noindent" >Embedded C can be the body of a script function. Instead enclosing the function body statements in <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">{</span></span></span> and <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">}</span></span></span>, use <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">%{</span></span></span>
+and <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">%}</span></span></span>. Any enclosed C code is literally transcribed into the kernel module: it is up to you to make it safe and correct.
+In order to take parameters and return a value, macros <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">STAP_ARG_*</span></span></span> and <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">STAP_RETVALUE</span></span></span> are made available. The
+familiar data-gathering functions <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">pid()</span></span></span>, <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">execname()</span></span></span>, and their neighbours are all embedded C functions. Figure <a
+href="#x10-1801010">10<!--tex4ht:ref: fig:embedded-C --></a>
+contains another example.
+<!--l. 961--><p class="noindent" >Since systemtap cannot examine the C code to infer these types, an
+optional<span class="footnote-mark"><a
+href="tutorial11.html#fn5x0"><sup class="textsuperscript">5</sup></a></span><a
+ id="x10-18001f5"></a>
+annotation syntax is available to assist the type inference process. Simply suffix parameter names and/or
+the function name with <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">:string</span></span></span> or <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">:long</span></span></span> to designate the string or numeric type. In addition, the
+script may include a <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">%{</span></span></span> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">%}</span></span></span> block at the outermost level of the script, in order to transcribe declarative
+code like <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">#include</span><span
+class="cmtt-10"> <linux/foo.h></span></span></span>. These enable the embedded C functions to refer to general kernel
+types.
+<!--l. 972--><p class="noindent" >There are a number of safety-related constraints that should be observed by developers of embedded C
+code.
+ <ol class="enumerate1" >
+ <li
+ class="enumerate" id="x10-18003x1">Do not dereference pointers that are not known or testable valid.
+ </li>
+ <li
+ class="enumerate" id="x10-18005x2">Do not call any kernel routine that may cause a sleep or fault.
+ </li>
+ <li
+ class="enumerate" id="x10-18007x3">Consider possible undesirable recursion, where your embedded C function calls a routine that may be
+ the subject of a probe. If that probe handler calls your embedded C function, you may suffer infinite
+ regress. Similar problems may arise with respect to non-reentrant locks.
+ </li>
+ <li
+ class="enumerate" id="x10-18009x4">If locking of a data structure is necessary, use a <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">trylock</span></span></span> type call to attempt to take the lock. If that
+ fails, give up, do not block.</li></ol>
+<!--l. 987--><p class="noindent" ><hr class="figure"><div class="figure"
+>
+
+<a
+ id="x10-1801010"></a>
+
+<div class="center"
+>
+<!--l. 988--><p class="noindent" >
+<div class="fbox"><div class="minipage"><pre class="verbatim" id="verbatim-12">
+# cat embedded-C.stp
+%{
+#include <linux/sched.h>
+#include <linux/list.h>
+%}
+function task_execname_by_pid:string (pid:long) %{
+  struct task_struct *p;
+  struct list_head *_p, *_n;
+  list_for_each_safe(_p, _n, &current->tasks) {
+    p = list_entry(_p, struct task_struct, tasks);
+    if (p->pid == (int)STAP_ARG_pid)
+      snprintf(STAP_RETVALUE, MAXSTRINGLEN, "%s", p->comm);
+  }
+%}
+probe begin
+{
+  printf("%s(%d)\n", task_execname_by_pid(target()), target())
+  exit()
+}
+# pgrep emacs
+16641
+# stap -g embedded-C.stp -x 16641
+emacs(16641)
+</pre>
+<!--l. 1016--><p class="nopar" > </div></div>
+</div>
+<br /> <div class="caption"
+><span class="id">Figure 10: </span><span
+class="content">Embedded C function.</span></div><!--tex4ht:label?: x10-1801010 -->
+
+<!--l. 1020--><p class="noindent" ></div><hr class="endfigure">
+<h4 class="subsectionHead"><span class="titlemark">4.4 </span> <a
+ id="x10-190004.4"></a>Naming conventions</h4>
+<!--l. 1024--><p class="noindent" >Using the tapset search mechanism just described, potentially many script files can become selected
+for inclusion in a single session. This raises the problem of name collisions, where different tapsets
+accidentally use the same names for functions/globals. This can result in errors at translate or run
+time.
+<!--l. 1030--><p class="noindent" >To control this problem, systemtap tapset developers are advised to follow naming conventions. Here is some of the
+guidance.
+ <ol class="enumerate1" >
+ <li
+ class="enumerate" id="x10-19002x1">Pick a unique name for your tapset, and substitute it for <span
+class="bchri7t-">TAPSET </span>below.
+ </li>
+ <li
+ class="enumerate" id="x10-19004x2">Separate identifiers meant to be used by tapset users from those that are internal implementation
+ artifacts.
+ </li>
+ <li
+ class="enumerate" id="x10-19006x3">Document the first set in the appropriate <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">man</span></span></span> pages.
+ </li>
+ <li
+ class="enumerate" id="x10-19008x4">Prefix the names of external identifiers with <span
+class="bchri7t-">TAPSET</span>_ if there is any likelihood of collision with other
+ tapsets or end-user scripts.
+ </li>
+ <li
+ class="enumerate" id="x10-19010x5">Prefix any probe point aliases with an appropriate prefix.
+ </li>
+ <li
+ class="enumerate" id="x10-19012x6">Prefix the names of internal identifiers with __<span
+class="bchri7t-">TAPSET</span>_.</li></ol>
+<!--l. 1047--><p class="noindent" >
+<h4 class="subsectionHead"><span class="titlemark">4.5 </span> <a
+ id="x10-200004.5"></a>Exercises</h4>
+<!--l. 1049--><p class="noindent" >
+ <ol class="enumerate1" >
+ <li
+ class="enumerate" id="x10-20002x1">Write a tapset that implements deferred and “cancelable” logging. Export a function that enqueues
+ a text string (into some private array), returning an id token. Include a timer-based probe that
+ periodically flushes the array to the standard log output. Export another function that, if the entry
+ was not already flushed, allows a text string to be cancelled from the queue. One might speculate that
+ similar functions and tapsets exist.
+ </li>
+ <li
+ class="enumerate" id="x10-20004x2">Create a “relative timestamp” tapset with functions return all the same values as the ones in the
+ timestamp tapset, except that they are made relative to the start time of the script.
+ </li>
+ <li
+ class="enumerate" id="x10-20006x3">Create a tapset that exports a global array that contains a mapping of recently seen process ID numbers
+ to process names. Intercept key system calls (<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">execve</span></span></span>?) to update the list incrementally.
+
+ </li>
+ <li
+ class="enumerate" id="x10-20008x4">Send your tapset ideas to the mailing list!</li></ol>
+
+<!--l. 1070--><div class="crosslinks"><p class="noindent">[<a
+href="tutorialse3.html" >prev</a>] [<a
+href="tutorialse3.html#tailtutorialse3.html" >prev-tail</a>] [<a
+href="tutorialse4.html" >front</a>] [<a
+href="tutorial.html# " >up</a>] </p></div>
+<!--l. 1070--><p class="noindent" ><a
+ id="tailtutorialse4.html"></a>
+</body></html>
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd">
+<html >
+<head><title>Further information</title>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta name="generator" content="TeX4ht (https://tug.org/tex4ht/)">
+<meta name="originator" content="TeX4ht (https://tug.org/tex4ht/)">
+<!-- html,2 -->
+<meta name="src" content="tutorial.tex">
+<link rel="stylesheet" type="text/css" href="tutorial.css">
+</head><body
+>
+<!--l. 1070--><div class="crosslinks"><p class="noindent">[<a
+href="tutorialse4.html" >prev</a>] [<a
+href="tutorialse4.html#tailtutorialse4.html" >prev-tail</a>] [<a
+href="#tailtutorialse5.html">tail</a>] [<a
+href="tutorial.html# " >up</a>] </p></div>
+<h3 class="sectionHead"><span class="titlemark">5 </span> <a
+ id="x12-210005"></a>Further information</h3>
+<!--l. 1072--><p class="noindent" >For further information about systemtap, several sources are available.
+<!--l. 1074--><p class="noindent" >There are <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">man</span></span></span> pages:
+<div class="tabular"> <table id="TBL-12" class="tabular"
+cellspacing="0" cellpadding="0"
+><colgroup id="TBL-12-1g"><col
+id="TBL-12-1"><col
+id="TBL-12-2"></colgroup><tr
+ style="vertical-align:baseline;" id="TBL-12-1-"><td style="white-space:nowrap; text-align:right;" id="TBL-12-1-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">stap</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-12-1-2"
+class="td11">systemtap program usage, language summary </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-12-2-"><td style="white-space:nowrap; text-align:right;" id="TBL-12-2-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">stappaths</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-12-2-2"
+class="td11">your systemtap installation paths </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-12-3-"><td style="white-space:nowrap; text-align:right;" id="TBL-12-3-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">stapprobes</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-12-3-2"
+class="td11">probes / probe aliases provided by built-in tapsets </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-12-4-"><td style="white-space:nowrap; text-align:right;" id="TBL-12-4-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">stapex</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-12-4-2"
+class="td11">a few basic example scripts </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-12-5-"><td style="white-space:nowrap; text-align:right;" id="TBL-12-5-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">tapset::*</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-12-5-2"
+class="td11">summaries of the probes and functions in each tapset</td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-12-6-"><td style="white-space:nowrap; text-align:right;" id="TBL-12-6-1"
+class="td11"> <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">probe::*</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-12-6-2"
+class="td11">detailed descriptions of each probe </td>
+</tr><tr
+ style="vertical-align:baseline;" id="TBL-12-7-"><td style="white-space:nowrap; text-align:right;" id="TBL-12-7-1"
+class="td11"><span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">function::*</span></span></span></td><td style="white-space:nowrap; text-align:left;" id="TBL-12-7-2"
+class="td11">detailed descriptions of each function </td>
+</tr></table></div>
+<!--l. 1086--><p class="noindent" >There is much more documentation and sample scripts included. You may find them under
+<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">/usr/share/doc/systemtap*/</span></span></span>.
+<!--l. 1089--><p class="noindent" >Then, there is the source code itself. Since systemtap is <span
+class="bchri7t-">free software</span>, you should have available the entire source
+code. The source files in the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">tapset/</span></span></span> directory are also packaged along with the systemtap binary. Since systemtap
+reads these files rather than their documentation, they are the most reliable way to see what’s inside
+all the tapsets. Use the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">-v</span></span></span> (verbose) command line option, several times if you like, to show inner
+workings.
+<!--l. 1100--><p class="noindent" >Finally, there is the project web site (<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">http://sourceware.org/systemtap/</span></span></span>) with several articles, an archived public
+mailing list for users and developers (<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">systemtap@sourceware.org</span></span></span>), IRC channels, and a live GIT source repository.
+Come join us!
+
+<!--l. 1116--><div class="crosslinks"><p class="noindent">[<a
+href="tutorialse4.html" >prev</a>] [<a
+href="tutorialse4.html#tailtutorialse4.html" >prev-tail</a>] [<a
+href="tutorialse5.html" >front</a>] [<a
+href="tutorial.html# " >up</a>] </p></div>
+<!--l. 1116--><p class="noindent" ><a
+ id="tailtutorialse5.html"></a>
+</body></html>
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd">
+<html >
+<head><title>Errors</title>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta name="generator" content="TeX4ht (https://tug.org/tex4ht/)">
+<meta name="originator" content="TeX4ht (https://tug.org/tex4ht/)">
+<!-- html,2 -->
+<meta name="src" content="tutorial.tex">
+<link rel="stylesheet" type="text/css" href="tutorial.css">
+</head><body
+>
+<!--l. 1116--><div class="crosslinks"><p class="noindent">[<a
+href="tutorialse5.html" >prev</a>] [<a
+href="tutorialse5.html#tailtutorialse5.html" >prev-tail</a>] [<a
+href="#tailtutorialse6.html">tail</a>] [<a
+href="tutorial.html# " >up</a>] </p></div>
+<h3 class="sectionHead"><span class="titlemark">A </span> <a
+ id="x13-22000A"></a>Errors</h3>
+<!--l. 1118--><p class="noindent" >We explain some common systemtap error messages in this section. Most error messages include line/character
+numbers with which one can locate the precise location of error in the script code. There is sometimes a subsequent
+or prior line that elaborates.
+<!--l. 1123--><p class="noindent" ><span
+class="bchri7t-x-x-120">error </span><span
+class="cmtt-12">at: </span><span
+class="bchri7t-x-x-120">filename</span><span
+class="bchr7t-x-x-120">:</span><span
+class="bchri7t-x-x-120">line</span><span
+class="bchr7t-x-x-120">:</span><span
+class="bchri7t-x-x-120">column</span><span
+class="bchr7t-x-x-120">: </span><span
+class="bchri7t-x-x-120">details</span>
+<!--l. 1125--><p class="noindent" >Many error messages contain a man-page key like this <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">[man</span><span
+class="cmtt-10"> foo]</span></span></span>. This indicates that more details are available as a
+<span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">man</span></span></span> page <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">foo</span></span></span>, so use the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">%</span><span
+class="cmtt-10"> man</span><span
+class="cmtt-10"> foo</span></span></span> command to view it.
+<!--l. 1130--><p class="noindent" >
+<h4 class="subsectionHead"><span class="titlemark">A.1 </span> <a
+ id="x13-23000A.1"></a>Parse errors</h4>
+<!--l. 1132--><p class="noindent" >
+ <dl class="description"><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">parse error: expected </span><span
+class="bchbi7t-">foo</span><span
+class="bchb7t-">, saw </span><span
+class="bchbi7t-">bar</span> <span
+class="cmmi-10">…</span> <br
+class="newline" />The script contained a grammar error. A different type of construct was expected in the given context.
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">parse error: embedded code in unprivileged script </span><br
+class="newline" />The script contained unsafe constructs such as embedded C (section <a
+href="tutorialse4.html#x10-180004.3">4.3<!--tex4ht:ref: embedded-c --></a>), but was run without the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">-g</span></span></span>
+ (guru mode) option. Confirm that the constructs are used safely, then try again with <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">-g</span></span></span>.</dd></dl>
+<!--l. 1144--><p class="noindent" >
+<h4 class="subsectionHead"><span class="titlemark">A.2 </span> <a
+ id="x13-24000A.2"></a>Type errors</h4>
+<!--l. 1146--><p class="noindent" >
+ <dl class="description"><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">semantic error: type mismatch for identifier ’</span><span
+class="bchbi7t-">foo</span><span
+class="bchb7t-">’</span> <span
+class="cmmi-10">…</span> <span
+class="bchb7t-">string vs. long </span><br
+class="newline" />In this case, the identifier <span
+class="bchri7t-">foo </span>was previously inferred as a numeric type (“long”), but at the given point
+ is being used as a string. Similar messages appear if an array index or function parameter slot is used
+ with conflicting types.
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">semantic error: unresolved type for identifier ’</span><span
+class="bchbi7t-">foo</span><span
+class="bchb7t-">’ </span><br
+class="newline" />The identifier <span
+class="bchri7t-">foo </span>was used, for example in a <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">print</span></span></span>, but without any operations that could assign it a
+ type. Similar messages may appear if a symbol is misspelled by a typo.
+
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">semantic error: Expecting symbol or array index expression </span><br
+class="newline" />Something other than an assignable lvalue was on the left hand sign of an assignment.</dd></dl>
+<!--l. 1163--><p class="noindent" >
+<h4 class="subsectionHead"><span class="titlemark">A.3 </span> <a
+ id="x13-25000A.3"></a>Symbol errors</h4>
+<!--l. 1165--><p class="noindent" >
+ <dl class="description"><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">while searching for arity </span><span
+class="bchbi7t-">N </span><span
+class="bchb7t-">function, semantic error: unresolved function call </span><br
+class="newline" />The script calls a function with <span
+class="bchri7t-">N </span>arguments that does not exist. The function may exist with different
+ arity.
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">semantic error: array locals not supported:</span> <span
+class="cmmi-10">…</span> <br
+class="newline" />An array operation is present for which no matching global declaration was found. Similar messages
+ appear if an array is used with inconsistent arities.
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">semantic error: variable ’</span><span
+class="bchbi7t-">foo</span><span
+class="bchb7t-">’ modified during ’foreach’ </span><br
+class="newline" />The array <span
+class="bchri7t-">foo </span>is being modified (being assigned to or deleted from) within an active <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">foreach</span></span></span> loop. This
+ invalid operation is also detected within a function called from within the loop.</dd></dl>
+<!--l. 1182--><p class="noindent" >
+<h4 class="subsectionHead"><span class="titlemark">A.4 </span> <a
+ id="x13-26000A.4"></a>Probing errors </h4>
+<!--l. 1184--><p class="noindent" >
+ <dl class="description"><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">semantic error: probe point mismatch at position </span><span
+class="bchbi7t-">N</span><span
+class="bchb7t-">, while resolving probe point </span><span
+class="bchbi7t-">foo </span><br
+class="newline" />A probe point was named that neither directly understood by systemtap, nor defined as an alias by a
+ tapset script. The divergence from the “tree” of probe point namespace is at position <span
+class="bchri7t-">N </span>(starting with
+ zero at left).
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">semantic error: no match for probe point, while resolving probe point </span><span
+class="bchbi7t-">foo </span><br
+class="newline" />A probe point cannot be resolved for any of a variety of reasons. It may be a debuginfo-based probe
+ point such as <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">kernel.function("foobar")</span></span></span> where no <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">foobar</span></span></span> function was found. This can occur if
+ the script specifies a wildcard on function names, or an invalid file name or source line number.
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">semantic error: unresolved target-symbol expression </span><br
+class="newline" />A target variable was referred to in a probe handler that was not resolvable. Or, a target variable is not
+ valid at all in a context such as a script function. This variable may have been elided by an optimizing
+ compiler, or may not have a suitable type, or there might just be an annoying bug somewhere. Try
+ again with a slightly different probe point (use <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">statement()</span></span></span> instead of <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">function()</span></span></span>) to search for a
+ more cooperative neighbour in the same area.
+
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">semantic error: libdwfl failure</span> <span
+class="cmmi-10">…</span> <br
+class="newline" />There was a problem processing the debugging information. It may simply be missing, or may
+ have some consistency / correctness problems. Later compilers tend to produce better debugging
+ information, so if you can upgrade and recompile your kernel/application, it may help.
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">semantic error: cannot find </span><span
+class="bchbi7t-">foo </span><span
+class="bchb7t-">debuginfo </span><br
+class="newline" />Similarly, suitable debugging information was not found. Check that your kernel build/installation
+ includes a matching version of debugging data.</dd></dl>
+<!--l. 1219--><p class="noindent" >
+<h4 class="subsectionHead"><span class="titlemark">A.5 </span> <a
+ id="x13-27000A.5"></a>Runtime errors</h4>
+<!--l. 1221--><p class="noindent" >Usually, run-time errors cause a script to terminate. Some of these may be caught with the <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">try</span><span
+class="cmtt-10"> {</span><span
+class="cmtt-10"> ...</span><span
+class="cmtt-10"> }</span><span
+class="cmtt-10"> catch</span><span
+class="cmtt-10"> {</span><span
+class="cmtt-10"> ...</span><span
+class="cmtt-10"> }</span></span></span>
+construct.
+<!--l. 1224--><p class="noindent" >
+ <dl class="description"><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">WARNING: Number of errors: </span><span
+class="bchbi7t-">N</span><span
+class="bchb7t-">, skipped probes: </span><span
+class="bchbi7t-">M </span><br
+class="newline" />Errors and/or skipped probes occurred during this run.
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">division by 0 </span><br
+class="newline" />The script code performed an invalid division.
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">aggregate element not found </span><br
+class="newline" />An statistics extractor function other than <span class="obeylines-h"><span class="verb"><span
+class="cmtt-10">@count</span></span></span> was invoked on an aggregate that has not had any
+ values accumulated yet. This is similar to a division by zero.
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">aggregation overflow </span><br
+class="newline" />An array containing aggregate values contains too many distinct key tuples at this time.
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">MAXNESTING exceeded </span><br
+class="newline" />Too many levels of function call nesting were attempted.
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">MAXACTION exceeded </span><br
+class="newline" />The probe handler attempted to execute too many statements.
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">kernel/user string copy fault at </span><span
+class="bchbi7t-">0xaddr </span><br
+class="newline" />The probe handler attempted to copy a string from kernel or user space at an invalid address.
+ </dd><dt class="description">
+ </dt><dd
+class="description"><span
+class="bchb7t-">pointer dereference fault </span><br
+class="newline" />There was a fault encountered during a pointer dereference operation such as a target variable
+ evaluation.
+ </dd></dl>
+
+
+<!--l. 1259--><div class="crosslinks"><p class="noindent">[<a
+href="tutorialse5.html" >prev</a>] [<a
+href="tutorialse5.html#tailtutorialse5.html" >prev-tail</a>] [<a
+href="tutorialse6.html" >front</a>] [<a
+href="tutorial.html# " >up</a>] </p></div>
+<!--l. 1259--><p class="noindent" ><a
+ id="tailtutorialse6.html"></a>
+</body></html>
--- /dev/null
+<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"
+ "http://www.w3.org/TR/html4/loose.dtd">
+<html >
+<head><title>Acknowledgments</title>
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+<meta name="generator" content="TeX4ht (https://tug.org/tex4ht/)">
+<meta name="originator" content="TeX4ht (https://tug.org/tex4ht/)">
+<!-- html,2 -->
+<meta name="src" content="tutorial.tex">
+<link rel="stylesheet" type="text/css" href="tutorial.css">
+</head><body
+>
+<!--l. 1259--><div class="crosslinks"><p class="noindent">[<a
+href="tutorialse6.html" >prev</a>] [<a
+href="tutorialse6.html#tailtutorialse6.html" >prev-tail</a>] [<a
+href="#tailtutorialse7.html">tail</a>] [<a
+href="tutorial.html# " >up</a>] </p></div>
+<h3 class="sectionHead"><span class="titlemark">B </span> <a
+ id="x14-28000B"></a>Acknowledgments</h3>
+<!--l. 1261--><p class="noindent" >The author thanks Martin Hunt, Will Cohen, and Jim Keniston for improvement advice for this paper.
+
+<!--l. 1264--><div class="crosslinks"><p class="noindent">[<a
+href="tutorialse6.html" >prev</a>] [<a
+href="tutorialse6.html#tailtutorialse6.html" >prev-tail</a>] [<a
+href="tutorialse7.html" >front</a>] [<a
+href="tutorial.html# " >up</a>] </p></div>
+<!--l. 1264--><p class="noindent" ><a
+ id="tailtutorialse7.html"></a>
+</body></html>
git://sourceware.org / systemtap-htdocs.git / commitdiff