[systemtap.git] / stap-client

#!/bin/bash

# Compile server client for systemtap
#
# Copyright (C) 2008, 2009 Red Hat Inc.
#
# This file is part of systemtap, and is free software.  You can
# redistribute it and/or modify it under the terms of the GNU General
# Public License (GPL); either version 2, or (at your option) any
# later version.

# This script examines the systemtap command line and packages the files and
# information needed to execute the command. This is then sent to a trusted
# systemtap server which will process the request and return the resulting
# kernel module (if requested) and any other information generated by the
# request. If a kernel module is generated, this script will load the module
# and execute it using 'staprun', if requested.

# Catch ctrl-c and other termination signals
trap 'terminate' SIGTERM
trap 'interrupt' SIGINT
trap 'ignore_signal' SIGHUP SIGPIPE

#-----------------------------------------------------------------------------
# Helper functions.
#-----------------------------------------------------------------------------
# function: configuration
function configuration {
    # INSTALL-HOOK These settings work for running the client from the source tree
    # INSTALL-HOOK using the dejagnu test harness and will be overridden at install
    # INSTALL-HOOK time.
    exec_prefix=
    sysconfdir=`pwd`/net

    # General configuration
    tmpdir_prefix_client=stap.client
    tmpdir_prefix_server=stap.server
    avahi_service_tag=_stap._tcp
}

# function: initialization
function initialization {
    our_host_name=`expr "$HOSTNAME" : "\\\([a-zA-Z0-9-]*\\\).*"`
    our_domain_name=`expr "$HOSTNAME" : "$our_host_name\\\(.*\\\)"`

    rc=0
    wd=`pwd`
    umask 0
    staprun_running=0

    # Default location for server certificates if we're not root
    # Must be owned by us.
    local uid uname
    if test $EUID != 0; then
	if test -e $HOME/.systemtap/ssl/client; then
	    if check_db $HOME/.systemtap/ssl/client $EUID $USER; then
		local_ssl_dbs=$HOME/.systemtap/ssl/client
	    fi
	fi
    fi
    # Additional location for all users. Must be owned by root.
    if test "X$sysconfdir" != "X"; then
	if test -e $sysconfdir/systemtap/ssl/client; then
	    if check_db $sysconfdir/systemtap/ssl/client 0 root; then
		public_ssl_dbs=$sysconfdir/systemtap/ssl/client
	    fi
	fi
    fi

    # Default options settings
    p_phase=5
    v_level=0
    keep_temps=0
    b_specified=0

    # Default variable settings
    find_all=

    # Create a temporary directory to package things in
    # Do this before parsing the command line so that there is a place
    # to put -I and -R directories.
    tmpdir_client=`mktemp -dt $tmpdir_prefix_client.XXXXXX` || \
	fatal "Cannot create temporary directory " $tmpdir_client
    tmpdir_env=`dirname $tmpdir_client`
}

# function: parse_options [ STAP-OPTIONS ]
#
# Examine the command line. We need not do much checking, but we do need to
# parse all options in order to discover the ones we're interested in.
# The server will take care of most situations and return the appropriate
# output.
#
function parse_options {
    cmdline=
    cmdline1=
    cmdline2=
    while test $# != 0
    do
	advance_p=0
	dash_seen=0

        # Start of a new token.
	first_token=$1
	until test $advance_p != 0
	do
            # Identify the next option
	    first_char=`expr "$first_token" : '\(.\).*'`
	    second_char=
	    if test $dash_seen = 0; then
		if test "$first_char" = "-"; then
		    if test "$first_token" != "-"; then
	                # It's not a lone dash, so it's an option.
			# Is it a long option (i.e. --option)?
			second_char=`expr "$first_token" : '.\(.\).*'`
			if test "X$second_char" = "X-"; then
			    long_option=`expr "$first_token" : '--\(.*\)=.*'`
			    test "X$long_option" != "X" || long_option=`expr "$first_token" : '--\(.*\)'`
			    case $long_option in
				ssl)
				    process_ssl $first_token
				    ;;
				server)
				    process_server $first_token
				    ;;
				*)
		                    # An unknown or unimportant option.
				    # Ignore it, but pass it on to the server.
				    cmdline2="$cmdline2 $first_token"
				    ;;
			    esac
			    advance_p=$(($advance_p + 1))
			    break
			fi
	                # It's not a lone dash, or a long option, so it's a short option string.
			# Remove the dash.
			first_token=`expr "$first_token" : '-\(.*\)'`
			dash_seen=1
			first_char=`expr "$first_token" : '\(.\).*'`
			cmdline2="$cmdline2 -"
		    fi
		fi
		if test $dash_seen = 0; then
	            # The dash has not been seen. This is either the script file
	            # name, a long argument or an argument to be passed to the probe module.
	            # If this is the first time, and -e has not been specified,
	            # then it could be the name of the script file.
		    if test "X$second_char" = "X-"; then
			cmdline2="$cmdline2 $first_token"
		    elif test "X$e_script" = "X" -a "X$script_file" = "X"; then
			script_file=$first_token
			cmdline1="$cmdline2"
			cmdline2=
		    elif test "$first_char" != "'"; then
			cmdline2="$cmdline2 '$first_token'"
		    else
			cmdline2="$cmdline2 $first_token"
		    fi
		    advance_p=$(($advance_p + 1))
		    break
		fi
	    fi

            # We are at the start of an option. Look at the first character.
	    case $first_char in
		b)
		    b_specified=1
		    ;;
		c)
		    get_arg $first_token "$2"
		    process_c "$stap_arg"
		    ;;
		D)
		    get_arg $first_token $2
		    cmdline2="${cmdline2}D '$stap_arg'"
		    ;;
		e)
		    get_arg $first_token "$2"
		    process_e "$stap_arg"
		    ;;
		I)
		    get_arg $first_token $2
		    process_I $stap_arg
		    ;;	
		k)
		    keep_temps=1
		    ;;
		l)
		    get_arg $first_token $2
		    cmdline2="${cmdline2}l '$stap_arg'"
		    ;;
		m)
		    get_arg $first_token $2
		    process_m $stap_arg
		    ;;
		o)
		    get_arg $first_token $2
		    process_o $stap_arg
		    ;;
		p)
		    get_arg $first_token $2
		    process_p $stap_arg
		    ;;
		r)
		    get_arg $first_token $2
		    cmdline2="${cmdline2}r '$stap_arg'"
		    ;;	
		R)
		    get_arg $first_token $2
		    process_R $stap_arg
		    ;;	
		s)
		    get_arg $first_token $2
		    cmdline2="${cmdline2}s '$stap_arg'"
		    ;;	
		v)
		    v_level=$(($v_level + 1))
		    ;;
		x)
		    get_arg $first_token $2
		    cmdline2="${cmdline2}x '$stap_arg'"
		    ;;
		*)
		    # An unknown or unimportant flag. Ignore it, but pass it on to the server.
		    ;;
	    esac

	    if test $advance_p = 0; then
	        # Just another flag character. Consume it.
		cmdline2="$cmdline2$first_char"
		first_token=`expr "$first_token" : '.\(.*\)'`
		if test "X$first_token" = "X"; then
		    advance_p=$(($advance_p + 1))
		fi
	    fi
	done

        # Consume the arguments we just processed.
	while test $advance_p != 0
	do
	    shift
	    advance_p=$(($advance_p - 1))
	done
    done

    # If the script file was given and it's not '-', then replace it with its
    # client-temp-name in the command string.
    if test "X$script_file" != "X"; then
	local local_name
	if test "$script_file" != "-"; then
	    local_name=`generate_client_temp_name $script_file`
	else
	    local_name=$script_file
	fi
	cmdline="$cmdline1 script/$local_name $cmdline2"
    else
	cmdline="$cmdline1 $cmdline2"
    fi

    # Processing based on final options settings
    # Complete the list of local certificate databases
    local_ssl_dbs="$additional_local_ssl_dbs $local_ssl_dbs"

    # We must have at least one usable certificate database.
    test "X$local_ssl_dbs" != "X " -o "X$public_ssl_dbs" != "X" || \
	fatal "No usable certificate databases found"

    # We can use any server if the phase is less than 5
    if test $p_phase -lt 5; then
	find_all="--all"
    fi
}

# function: get_arg FIRSTWORD SECONDWORD
#
# Collect an argument to the given option
function get_arg {
    # Remove first character.
    local opt=`expr "$1" : '\(.\).*'`
    local first=`expr "$1" : '.\(.*\)'`

    # Advance to the next token, if the first one is exhausted.
    if test "X$first" = "X"; then
	shift
	advance_p=$(($advance_p + 1))
	first=$1
    fi

    test "X$first" != "X" || \
	fatal "Missing argument to -$opt"

    stap_arg="$first"
    advance_p=$(($advance_p + 1))
}

# function: process_ssl ARGUMENT
#
# Process the --ssl option.
function process_ssl {
    local db=`expr "$1" : '--ssl=\(.*\)'`

    test "X$db" != "X" || \
	fatal "Missing argument to --ssl"

    check_db $db || return

    additional_local_ssl_dbs="$additional_local_ssl_dbs $db"
}

# function: process_server ARGUMENT
#
# Process the --server option.
function process_server {
    local spec=`expr "$1" : '--server=\(.*\)'`

    test "X$spec" != "X" || \
	fatal "Missing argument to --server"

    specified_servers="$specified_servers $spec"
}

# function: process_c ARGUMENT
#
# Process the -c flag.
function process_c {
    c_cmd="$1"
    cmdline2="${cmdline2}c '$1'"
}

# function: process_e ARGUMENT
#
# Process the -e flag.
function process_e {
    # Only the first -e option is recognized and it overrides any script file name
    # which may have already been identified.
    if test "X$e_script" = "X"; then
	e_script="$1"
	if test "X$script_file" != "X"; then
	    cmdline1="$cmdline1 $script_file $cmdline2"
	    cmdline2=
	    script_file=
	fi
    fi
    cmdline2="${cmdline2}e '$1'"
}

# function: process_I ARGUMENT
#
# Process the -I flag.
function process_I {
    local local_name=`include_file_or_directory tapsets $1`
    test "X$local_name" != "X" || return
    cmdline2="${cmdline2}I 'tapsets/$local_name'"
}

# function: process_m ARGUMENT
#
# Process the -m flag.
function process_m {
    m_name="$1"
    cmdline2="${cmdline2}m '$1'"
}

# function: process_o ARGUMENT
#
# Process the -o flag.
function process_o {
    stdout_redirection="$1"
    cmdline2="${cmdline2}o '$1'"
}

# function: process_p ARGUMENT
#
# Process the -p flag.
function process_p {
    p_phase=$1
    cmdline2="${cmdline2}p '$1'"
}

# function: process_R ARGUMENT
#
# Process the -R flag.
function process_R {
    local local_name=`include_file_or_directory runtime $1`
    test "X$local_name" != "X" || return
    cmdline2="${cmdline2}R 'runtime/$local_name'"
}

# function: include_file_or_directory PREFIX NAME
#
# Include the given file or directory in the client's temporary
# tree to be sent to the server.
function include_file_or_directory {
    # Add a symbolic link of the named file or directory to our temporary directory
    local local_name=`generate_client_temp_name $2`
    mkdir -p $tmpdir_client/$1/`dirname $local_name` || \
	fatal "Could not create $tmpdir_client/$1/`dirname $local_name`"
    ln -s /$local_name $tmpdir_client/$1/$local_name || \
	fatal "Could not link $tmpdir_client/$1/$local_name to /$local_name"
    echo "$local_name"
}

# function: generate_client_temp_name NAME
#
# Generate the name to be used for the given file/directory relative to the
# client's temporary directory.
function generate_client_temp_name {
    # Transform the name into a fully qualified path name
    local full_name=`echo "$1" | sed "s,^\\\([^/]\\\),$wd/\\\\1,"`

    # The same name without the initial / or trailing /
    local local_name=`echo "$full_name" | sed 's,^/\(.*\),\1,'`
    local_name=`echo "$local_name" | sed 's,\(.*\)/$,\1,'`
    echo "$local_name"
}

# function: create_request
#
# Add information to the client's temp directory representing the request
# to the server.
function create_request {
    # Work in our temporary directory
    cd $tmpdir_client

    if test "X$script_file" != "X"; then
	if test "$script_file" = "-"; then
	    mkdir -p $tmpdir_client/script || \
		fatal "Cannot create temporary directory " $tmpdir_client/script
	    cat > $tmpdir_client/script/$script_file
	else
	    include_file_or_directory script $script_file > /dev/null
	fi
    fi

    # Add the necessary info to special files in our temporary directory.
    echo "cmdline: $cmdline" > cmdline
    echo "sysinfo: `client_sysinfo`" > sysinfo
}

# function client_sysinfo
#
# Generate the client's sysinfo and echo it to stdout
function client_sysinfo {
    if test "X$sysinfo_client" = "X"; then
	# Add some info from uname
	sysinfo_client="`uname -rvm`"
    fi
    echo "$sysinfo_client"
}

# function: package_request
#
# Package the client's temp directory into a form suitable for sending to the
# server.
function package_request {
    # Package up the temporary directory into a zip file
    cd $tmpdir_env

    local tmpdir_client_base=`basename $tmpdir_client`
    zip_client=$tmpdir_env/`mktemp $tmpdir_client_base.zip.XXXXXX` || \
	fatal "Cannot create temporary file " $zip_client

    (rm $zip_client && zip -r $zip_client $tmpdir_client_base > /dev/null) || \
	fatal "zip of request tree, $tmpdir_client, failed"
}

# function: unpack_response
#
# Unpack the zip file received from the server and make the contents available
# for printing the results and/or running 'staprun'.
function unpack_response {
    tmpdir_server=`mktemp -dt $tmpdir_prefix_client.server.XXXXXX` || \
	fatal "Cannot create temporary file " $tmpdir_server

    # Unpack the server output directory
    unzip -d $tmpdir_server $zip_server > /dev/null || \
	fatal "Cannot unpack server response, $zip_server"

    # Check the contents of the expanded directory. It should contain a
    # single directory whose name matches stap.server.??????
    local num_files=`ls $tmpdir_server | wc -l`
    test $num_files = 1 || \
	fatal "Wrong number of files in server's temp directory"
    test -d $tmpdir_server/stap.server.?????? || \
	fatal "`ls $tmpdir_server` does not match the expected name or is not a directory"
    # Move the contents of the directory down one level.
    mv $tmpdir_server/stap.server.??????/* $tmpdir_server
    rm -fr $tmpdir_server/stap.server.??????

    # Check the contents of the directory. It should contain:
    # 1) a file called stdout
    # 2) a file called stderr
    # 3) a file called rc
    # 4) optionally a directory named to match stap??????
    num_files=`ls $tmpdir_server | wc -l`
    test $num_files = 4 -o $num_files = 3 || \
	fatal "Wrong number of files in server's temp directory"
    test -f $tmpdir_server/stdout || \
	fatal "`pwd`/$tmpdir_server/stdout does not exist or is not a regular file"
    test -f $tmpdir_server/stderr || \
	fatal "`pwd`/$tmpdir_server/stderr does not exist or is not a regular file"
    test -f $tmpdir_server/rc || \
	fatal "`pwd`/$tmpdir_server/rc does not exist or is not a regular file"

    # See if there is a systemtap temp directory
    tmpdir_stap=`cd $tmpdir_server && ls | grep stap......\$ 2>/dev/null`
    if test "X$tmpdir_stap" != "X"; then
	test -d $tmpdir_server/$tmpdir_stap || \
	    fatal "$tmpdir_server/$tmpdir_stap is not a directory"

        # Move the systemtap temp directory to a local temp location, if -k
        # was specified.
	if test $keep_temps = 1; then
	    local local_tmpdir_stap=`mktemp -dt stapXXXXXX` || \
		fatal "Cannot create temporary directory " $local_tmpdir_stap
	    mv $tmpdir_server/$tmpdir_stap/* $local_tmpdir_stap 2>/dev/null
	    rm -fr $tmpdir_server/$tmpdir_stap

	    # Correct the name of the temp directory in the server's stderr output
	    sed -i "s,^Keeping temporary directory.*,Keeping temporary directory \"$local_tmpdir_stap\"," $tmpdir_server/stderr
	    tmpdir_stap=$local_tmpdir_stap
	else
            # Make sure we own the systemtap temp directory if we are root.
	    test $EUID = 0 && chown $EUID:$EUID $tmpdir_server/$tmpdir_stap
	fi
    fi
}

# function: find_and_connect_to_server
#
# Find and establish connection with a compatible stap server.
function find_and_connect_to_server {
    local num_servers=0

    # Make a place to receive the response file.
    zip_server=`mktemp -t $tmpdir_prefix_client.server.zip.XXXXXX` || \
	fatal "Cannot create temporary file " $zip_server

    # Make a place to record connection errors
    touch $tmpdir_client/connect

    # If servers were specified on the command line, then try them
    # in sequence. Don't try any other servers.
    if test "X$specified_servers" != "X"; then
	for server in $specified_servers; do
	    num_servers=$(($num_servers + 1))

	    # If the server is completely specified, (i.e. server:port),
            # then try it directly.
	    port=`expr "$server" : '.\+:\([0-9]\+\)'`
            if test "X$port" != "X"; then
		name=`expr "$server" : '\(.\+\):[0-9]\+'`

                # If we have been given an ip address, then try to resolve it to a name.
                # If we have been given a name, try to resolve the full name.
                # The full name is needed in order to validate the server's certificate.
                address=`expr "$name" : '\([0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+\)'`
		if test "X$address" = "X"; then
		    # We've been given a host name
		    full_name=`nslookup $name | awk '/^Name\:/ {print $2}'`
		    if test "X$full_name" != "X"; then
			name=$full_name
		    fi
		else
		    # We've been given an ip address.
		    name=`nslookup $address | awk '/in-addr\.arpa/ {print $4}'`
		    name=`expr "$name" : '\(.*\)\.'`
                    if test "X$name" = "X"; then
			echo "Cannot resolve ip address $address" >> $tmpdir_client/connect
			continue
		    fi
		fi

		# Now try to contact the given server.
		ssl_db=`send_receive $name $port`
		test "X$ssl_db" != "X" && return
		continue
	    fi

	    # Otherwise select the matching server from the available servers
	    # and use the port it is advertizing.
	    #
	    # Have we been given an ip address? If so, just use it.
	    address=`expr "$server" : '\([0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+\)'`
	    if test "X$address" = "X"; then
		# We have not been given an ip address. Try to resolve it as a host name.
		if test "X$server" = "Xlocalhost"; then
		    # We don't want the address of the loopback interface here. Avahi will present
		    # the actual ip address.
		    server=$our_host_name$our_domain_name
		fi
		address=`nslookup $server | awk '/^Address\:[ \t][0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/ {print $2}'`
                if test "X$address" = "X"; then
		    echo "Cannot resolve server $server" >> $tmpdir_client/connect
		    continue
		fi
	    fi

	    if test `${exec_prefix}stap-find-servers $find_all | grep $address | wc -l` = "0"; then
		warning "No server is available on $server" 2>> $tmpdir_client/connect
		continue
	    fi

            ssl_db=`${exec_prefix}stap-find-servers $find_all | grep $address | choose_server`
	    test "X$ssl_db" != "X" && return
	done
    else
        # No servers specified. Find available servers and choose one of them.
	# Remember which ssl certificate database was used to authenticate the chosen
        # server.
	ssl_db=`${exec_prefix}stap-find-servers $find_all | choose_server`
	test "X$ssl_db" != "X" && return

	num_servers=`${exec_prefix}stap-find-servers $find_all | wc -l`
    fi

    if test $num_servers = 0; then
	fatal "Unable to find a server"
    fi

    cat $tmpdir_client/connect >&2
    fatal "Unable to connect to a server"
}

# function: choose_server
#
# Examine each line from stdin and attempt to connect to each server
# specified until successful.
# echo the name of the ssl certificate database used to successfully authenticate
# the server.
function choose_server {
    local name ip port remain

    while read name ip port remain
    do
	if test "X$name" = "X"; then
	    fatal "Server name not provided by avahi"
	fi

#	if test "X$ip" = "X"; then
#	    fatal "Server ip address not provided by avahi"
#	fi

	if test "X$port" = "X"; then
	    fatal "Server port not provided by avahi"
	fi

	ssl_db=`send_receive $name $port`
	test "X$ssl_db" != "X" && echo $ssl_db && return
    done
}

# function: send_receive SERVER PORT
#
# Connect to the server, send the request and receive the response
# echo the name of the ssl certificate database used to successfully authenticate
# the server.
function send_receive {
    local server=$1
    local port=$2

    # The server must match the dns name on the certificate
    # and must be 'localhost' if the server is on the local host.
    local server_host_name=`expr "$server" : "\\\([a-zA-Z0-9-]*\\\).*"`
    local server_domain_name=`expr "$server" : "$server_host_name\\\(.*\\\)"`

    if test "X$server_domain_name" = "X.local"; then
	server_domain_name=$our_domain_name
    fi
    if test "X$server_host_name$server_domain_name" = "Xlocalhost$our_domain_name"; then
	server=localhost
    elif test "X$server_host_name$server_domain_name" = "X$our_host_name$our_domain_name"; then
	server=localhost
    else
	server=$server_host_name$server_domain_name
    fi

    # Try to connect using each of the given local certificate databases in turn
    # for verification.
    for db in $local_ssl_dbs
    do
        # Send the request and receive the response using stap-client-connect
	echo "Attempting connection with $server:$port using certificate database in '$db'" >> $tmpdir_client/connect
	${exec_prefix}stap-client-connect -i $zip_client -o $zip_server -d $db -p $port -h $server >> $tmpdir_client/connect 2>&1 &
	wait '%${exec_prefix}stap-client-connect'
	test $? = 0 && echo $db && return
	sleep 1
    done

    # Next, try the public certificate databases.
    for db in $public_ssl_dbs
    do
        # Send the request and receive the response using stap-client-connect
	echo "Attempting connection with $server:$port using certificate database in '$db'" >> $tmpdir_client/connect
	${exec_prefix}stap-client-connect -i $zip_client -o $zip_server -d $db -p $port -h $server >> $tmpdir_client/connect 2>&1 &
	wait '%${exec_prefix}stap-client-connect'
	test $? = 0 && echo $db && return
	sleep 1
    done

    # Could not connect using any of the certificate databases
}

# function: process_response
#
# Write the stdout and stderr from the server to stdout and stderr respectively.
function process_response {
    # Pick up the results of running stap on the server.
    cd $tmpdir_server
    rc=`cat rc`

    # Copy the module to the current directory, if -m was specified
    if test "X$m_name" != "X"; then
	if test -f $tmpdir_stap/$m_name.ko; then
	    cp $tmpdir_stap/$m_name.ko $wd
	else
	    stream_output
	    fatal "module $tmpdir_stap/$m_name.ko does not exist"
	fi
    fi

    # Output stdout and stderr as directed
    stream_output
}

# function: stream_output
#
# Output stdout and stderr as directed
function stream_output {
    cd $tmpdir_server
    cat stderr >&2
    cat stdout
}

# function: maybe_call_staprun
#
# Call staprun using the module returned from the server, if requested.
function maybe_call_staprun {
    if test $rc != 0; then
	# stap run on the server failed, so don't bother
	return
    fi

    if test $p_phase -ge 4; then
        # There should be a systemtap temporary directory.
	if test "X$tmpdir_stap" = "X"; then
	    # OK if no script specified
	    if test "X$e_script" != "X" -o "X$script_file" != "X"; then
		fatal "systemtap temporary directory is missing in server response"
	    fi
	    return
	fi

        # There should be a module.
	local mod_name=`ls $tmpdir_stap | grep '.ko$'`
	if test "X$mod_name" = "X"; then
	    fatal "No module was found in $tmpdir_stap"
	fi

	if test $p_phase = 5; then
	    test $v_level -gt 0 && echo "Pass 5: starting run." >&2

	    # We have a module. Try to run it
	    # If a -c command was specified, pass it along.
	    if test "X$c_cmd" != "X"; then
		staprun_opts="-c '$c_cmd'"
	    fi

	    # The -v level will be one less than what was specified
	    # for us.
	    for ((vl=$((v_level - 1)); $vl > 0; --vl))
	    do
		staprun_opts="$staprun_opts -v"
	    done

	    # if -o was specified, pass it along
	    if test "X$stdout_redirection" != "X"; then
		staprun_opts="$staprun_opts -o $stdout_redirection"
	    fi

	    # Run it in the background and wait for it. This
	    # way any signals send to us can be caught.
	    if test $v_level -ge 2; then
		echo "running `which staprun` $staprun_opts $tmpdir_stap/`ls $tmpdir_stap | grep '.ko$'`" >&2
	    fi
	    eval `staprun_PATH` "$staprun_opts" \
		$tmpdir_stap/`ls $tmpdir_stap | grep '.ko$'`
	    staprun_running=1
	    wait '%?staprun' > /dev/null 2>&1
	    rc=$?
	    staprun_running=0
	    # 127 from wait means that the job was already finished.
	    test $rc=127 && rc=0

	    # Wait until the job actually disappears so that its output is complete.
	    while jobs '%?staprun' >/dev/null 2>&1
	    do
		sleep 1
	    done

	    test $v_level -gt 0 && echo "Pass 5: run completed in 0usr/0sys/0real ms." >&2
	fi
    fi
}

# function: staprun_PATH
#
# Compute a PATH suitable for running staprun.
function staprun_PATH {
    # If $SYSTEMTAP_STAPRUN is set, then use that
    if test "X$SYSTEMTAP_STAPRUN" != "X"; then
	echo $SYSTEMTAP_STAPRUN
	return
    fi

    # Otherwise, if there is an exec_prefix, then use it.
    if test "X$exec_prefix" != "X"; then
	echo ${exec_prefix}staprun
	return
    fi

    # Otherwise, we have been called by the dejagnu test harness as 'stap'
    # and we are the first 'stap' on the path. Since staprun may call
    # 'stap', remove the PATH component where we live from the PATH in order to
    # avoid recursion.
    local first_stap=`which stap`
    local PATH_component=`dirname $first_stap`
    echo "PATH=$PATH staprun" | sed "s,$PATH_component,,g"
}

# function: check_db DBNAME [ EUID USER ]
#
# Check the security of the given database directory.
function check_db {
    local dir=$1
    local euid=$2
    local user=$3
    local rc=0

    # Check that we have been given a directory
    if ! test -e $dir; then
	warning "Certificate database '$dir' does not exist"
	return 1
    fi
    if ! test -d $dir; then
	warning "Certificate database '$dir' is not a directory"
	return 1
    fi

    # If euid has been specified, then this directory must be owned by that
    #  user.
    if test "X$euid" != "X"; then
	local ownerid=`stat -c "%u" $dir`
	if test "X$ownerid" != "X$euid"; then
	    warning "Certificate database '$dir' must be owned by $user"
	    rc=1
	fi
    fi

    # Check that we can read the directory
    if ! test -r $dir; then
	warning "Certificate database '$dir' is not readble"
	rc=1
    fi

    # Check the access permissions of the directory
    local perm=0`stat -c "%a" $dir`
    if test $((($perm & 0400) == 0400)) = 0; then
	warning "Certificate database '$dir' should be readable by the owner"
    fi
    if test $((($perm & 0200) == 0200)) = 0; then
	warning "Certificate database '$dir' should be writeable by the owner"
    fi
    if test $((($perm & 0100) == 0100)) = 0; then
	warning "Certificate database '$dir' should be searchable by the owner"
    fi
    if test $((($perm & 0040) == 0040)) = 0; then
	warning "Certificate database '$dir' should be readable by the group"
    fi
    if test $((($perm & 0020) == 0020)) = 1; then
	warning "Certificate database '$dir' must not be writable by the group"
	rc=1
    fi
    if test $((($perm & 0010) == 0010)) = 0; then
	warning "Certificate database '$dir' should be searchable by the group"
    fi
    if test $((($perm & 0004) == 0004)) = 0; then
	warning "Certificate database '$dir' should be readable by others"
    fi
    if test $((($perm & 0002) == 0002)) = 1; then
	warning "Certificate database '$dir' must not be writable by others"
	rc=1
    fi
    if test $((($perm & 0001) == 0001)) = 0; then
	warning "Certificate database '$dir' should be searchable by others"
    fi

    # Now check the permissions of the critical files.
    check_db_file $dir/cert8.db  $euid $user || rc=1
    check_db_file $dir/key3.db   $euid $user || rc=1
    check_db_file $dir/secmod.db $euid $user || rc=1

    test $rc = 1 && warning "Unable to use certificate database '$dir' due to errors"

    return $rc
}

# function: check_db_file FILENAME [ EUID USER ]
#
# Check the security of the given database file.
function check_db_file {
    local file=$1
    local rc=0

    # Check that we have been given a file
    if ! test -e $file; then
	warning "Certificate database file '$file' does not exist"
	return 1
    fi
    if ! test -f $file; then
	warning "Certificate database file '$file' is not a regular file"
	return 1
    fi

    # If euid has been specified, then this directory must be owned by that
    #  user.
    if test "X$euid" != "X"; then
	local ownerid=`stat -c "%u" $file`
	if test "X$ownerid" != "X$euid"; then
	    warning "Certificate database file '$file' must be owned by $user"
	    rc=1
	fi
    fi

    # Check that we can read the file
    if ! test -r $file; then
	warning "Certificate database file '$file' is not readble"
	rc=1
    fi

    # Check the access permissions of the file
    local perm=0`stat -c "%a" $file`
    if test $((($perm & 0400) == 0400)) = 0; then
	warning "Certificate database file '$file' should be readable by the owner"
    fi
    if test $((($perm & 0200) == 0200)) = 0; then
	warning "Certificate database file '$file' should be writeable by the owner"
    fi
    if test $((($perm & 0100) == 0100)) = 1; then
	warning "Certificate database file '$file' must not be executable by the owner"
	rc=1
    fi
    if test $((($perm & 0040) == 0040)) = 0; then
	warning "Certificate database file '$file' should be readable by the group"
    fi
    if test $((($perm & 0020) == 0020)) = 1; then
	warning "Certificate database file '$file' must not be writable by the group"
	rc=1
    fi
    if test $((($perm & 0010) == 0010)) = 1; then
	warning "Certificate database file '$file' must not be executable by the group"
	rc=1
    fi
    if test $((($perm & 0004) == 0004)) = 0; then
	warning "Certificate database file '$file' should be readable by others"
    fi
    if test $((($perm & 0002) == 0002)) = 1; then
	warning "Certificate database file '$file' must not be writable by others"
	rc=1
    fi
    if test $((($perm & 0001) == 0001)) = 1; then
	warning "Certificate database file '$file' must not be executable by others"
	rc=1
    fi

    return $rc
}

# function: warning [ MESSAGE ]
#
# Warning error
# Prints its arguments to stderr
function warning {
    echo "$0: WARNING:" "$@" >&2
}

# function: fatal [ MESSAGE ]
#
# Fatal error
# Prints its arguments to stderr and exits
function fatal {
    echo "$0: ERROR:" "$@" >&2
    cleanup
    exit 1
}

# function cleanup
#
# Cleanup work files unless asked to keep them.
function cleanup {
    # Clean up.
    cd $tmpdir_env
    if test $keep_temps != 1; then
	rm -fr $tmpdir_client
	rm -f  $zip_client
	rm -f  $zip_server
	rm -fr $tmpdir_server
    fi
}

# function: terminate
#
# Terminate gracefully.
function terminate {
    # Clean up
    echo "$0: terminated by signal" >&2
    cleanup

    # Kill any running staprun job
    kill -s SIGTERM '%?staprun' 2>/dev/null

    # Kill any stap-client-connect job
    kill -s SIGTERM '%${exec_prefix}stap-client-connect' 2>/dev/null

    exit 1
}

# function: interrupt
#
# Pass an interrupt (ctrl-C) to staprun
function interrupt {
    # Pass the signal on to any running staprun job
    if test $staprun_running = 1; then
	kill -s SIGINT '%?staprun' 2>/dev/null
	return
    fi

    # Kill any stap-client-connect job
    # SIGINT won't do it.
    kill -s SIGTERM '%${exec_prefix}stap-client-connect' 2>/dev/null

    # If staprun was not running, then exit.
    cleanup
    exit 1
}

# function: ignore_signal
#
# Called in order to ignore a signal
function ignore_signal {
    :
}

#-----------------------------------------------------------------------------
# Beginning of main line execution.
#-----------------------------------------------------------------------------
configuration
initialization
parse_options "$@"
create_request
package_request
find_and_connect_to_server
unpack_response
process_response
maybe_call_staprun
cleanup

exit $rc