Re: [PATCH] proof of concept: systemtap/git differential code-coverage

[BR Chrisman <brchrisman at gmail dot com>]

I felt this had a bit of a 'dog food' feel to it for the systemtap
project, so I extracted and posted.
If there's interest, I can clean stuff up or do whatever might be
helpful with it.

- Brian

On Wed, Sep 10, 2014 at 10:48 PM, Brian Chrisman <brchrisman@gmail.com> wrote:
> Using systemtap and a git-diff hook, I implemented a poor
> man's differential code coverage tool (roughly extracted from
> a tool I built for use internally).  The script
> testsuite/testAndRun generates and launches a systemtap script
> defining userspace probes and then executes 'make check'.
> At the end of the test, a report is output with identifiers
> showing whether a added/modified line is:
> a) coverage not available (no DWARF point)
> b) covered but not executed
> c) covered and executed by some test
>
> example output from a recent commit I moved up to HEAD:
> ...
> if test -n ""; then mail  < systemtap.sum; fi
> make[1]: Leaving directory `/root/systemtap/testsuite'
> cover_na       tapsets.cxx    1112 :                   || fi->descriptor) // ppc opd (and also undefined symbols)
> cover_na       tapsets.cxx    1113 :                 continue;
> cover_exec     tapsets.cxx    1111 :               if (!null_die(&fi->die) // already handled in query_module_dwarf()
> cover_exec     tapsets.cxx    1114 :               if (dw.function_name_matches_pattern(fi->name, function_str_val))
> ---
>  testsuite/testAndCover |   72 ++++++++++++++++++++++++++++++++++++++++++++++++
>  1 files changed, 72 insertions(+), 0 deletions(-)
>  create mode 100755 testsuite/testAndCover
>
> diff --git a/testsuite/testAndCover b/testsuite/testAndCover
> new file mode 100755
> index 0000000..00bdd96
> --- /dev/null
> +++ b/testsuite/testAndCover
> @@ -0,0 +1,72 @@
> +#!/bin/bash
> +
> +mkdir -p coverage
> +
> +# probediff
> +[[ $0 =~ probediff ]] &&
> +{
> +    read path oldFile oldHex oldMode newFile newHex newMode< <(echo $*)
> +    [[ $path =~ \.cxx$ || $path =~ \.cpp$ || $path =~ \.h$ ]] || exit 0
> +    (( $# == 7 )) && diff \
> +        --new-line-format="process(\"$stapBinary\").statement(\"*@$(pwd)/$path:%dn\")%c'\012'" \
> +        --old-line-format='' \
> +        --unchanged-line-format='' \
> +        $oldFile $newFile
> +    exit 0
> +}
> +
> +function coverageAvailable() { (( $(wc -l < coverage/diffCoverage.stp) > 1 )); }
> +
> +# generate stap script from git output, launch stap, run tests, killoff stap
> +(
> +    #export stapBinary=${PWD%/*}/stap
> +    export stapBinary=/usr/local/bin/stap
> +    export GIT_EXTERNAL_DIFF=${PWD}/probediff
> +    git diff HEAD^                          |
> +        sort -u                             |
> +        tee coverage/all_lines              |
> +        sed -e 's/"/\\"/g'                  |
> +        xargs -n 1 -P 16 /usr/bin/stap -l   |
> +        sort -u                             |
> +        tee coverage/valid_lines            |
> +        sed -e 's/"/\\"/g'                  |
> +        xargs -n 1 -P 16 /usr/bin/stap -l   |
> +        sort -u                             |
> +        awk 'BEGIN { print "global tracker" } { print "probe " $0 " { if (tracker[" NR "]++ == 0) println(pp()) }" }' \
> +            > coverage/diffCoverage.stp
> +
> +    if coverageAvailable; then
> +        stapPID=$(/usr/bin/stap -o coverage/covered_lines -F coverage/diffCoverage.stp)
> +        [[ $? -ne 0 ]] || trap "kill $stapPID" EXIT
> +       [[ -n $stapPID ]] || exit 1
> +    fi
> +    #runtest systemtap.base/statement.exp
> +    make check
> +) || exit 1
> +
> +coverageAvailable || exit 0
> +
> +# report on coverage
> +
> +# all_lines will not have fully resolved function names, yet valid_lines and covered_lines will, so blow away everything up to '@'
> +# binaries shouldn't be named with '@' symbols?
> +# SEDDY!!!!
> +function seddy() { sed -e 's/.*@//;' "$@" | sort -u ; }
> +
> +(
> +    cd coverage
> +    comm -23 <(seddy all_lines) <(seddy valid_lines) |
> +        tr '"):' '   ' |
> +        { while read file line; do printf "%-14.14s %-14.14s %-5.5s: " cover_na $(basename $file) $line; sed -n -e "${line}p" $file; done; }
> +
> +    comm -12 <(seddy valid_lines) <(seddy covered_lines) |
> +        tr '"):' '   ' |
> +        { while read file line; do printf "%-14.14s %-14.14s %-5.5s: " cover_exec $(basename $file) $line; sed -n -e "${line}p" $file ; done; }
> +
> +    comm -23 <(seddy valid_lines) <(seddy covered_lines) |
> +        tr '"):' '   ' |
> +        { while read file line; do printf "%-14.14s %-14.14s %-5.5s: " cover_no_exec $(basename $file) $line; sed -n -e "${line}p" $file ; done; }
> +
> +        #{ while read file line; do sed -n -e "${line}p" $file; done; } |
> +        #sed -e 's/^/cover_no_ex:/'
> +) | tee coverage/coverage.log
> --
> 1.7.1
>