Add the testsuite parallel testing feature.

[systemtap.git] / testsuite / Makefile.am

diff --git a/testsuite/Makefile.am b/testsuite/Makefile.am
index 0870415faf07541efc40b991e258f35ddea9ae8f..f75c1c1d15efedea84cf7ad66851383a60b53891 100644 (file)
--- a/testsuite/Makefile.am
+++ b/testsuite/Makefile.am
@@ -14,6 +14,7 @@ clean-local:
        -rm -f flightrec*
        -rm -f *.so
        -rm -f uprobe_*
+       -rm -rf artifacts
 
 DEJAZILLA=@dejazilla@
 TESTAPPS=@enable_testapps@
@@ -31,6 +32,72 @@ installcheck: site.exp
        -$(MAKE) $(AM_MAKEFLAGS) check-DEJAGNU RUNTESTFLAGS="$(RUNTESTFLAGS) --tool_opts \'install $(TOOL_OPTS)\'"
        if test -n "$(DEJAZILLA)"; then mail $(DEJAZILLA) < systemtap.sum; fi
 
+# In the parallel mode, the dg-extract-results.sh generates common .sum and .log
+# files for all the tests, plus separate logs per testcase are kept in the artifacts
+# directory. Serial tests (see below) log to artifacts/__serialtests.
+installcheck-parallel: site.exp
+       -rmmod uprobes 2>/dev/null
+       rm -rf artifacts
+       mkdir -p artifacts/__serialtests
+       MAKEFLAGS= runtest systemtap.base/environment_sanity.exp
+       -$(MAKE) -k do-installcheck-parallel; \
+       result=$$?; \
+       $(SHELL) $(srcdir)/dg-extract-results.sh \
+       `find artifacts -name systemtap.sum -print` > systemtap.sum; \
+       $(SHELL) $(srcdir)/dg-extract-results.sh -L \
+       `find artifacts -name systemtap.log -print` > systemtap.log; \
+       exit $$result
+
+# TESTS exists for the user to pass on the command line to easily
+# say "Only run these tests."  With check or installcheck it's not necessary, but
+# with installcheck-parallel there's no other way to (easily) specify a subset
+# of tests. To specify all tests in a subdirectory, use TESTS=systemtap.subdir/*.exp.
+# E.g., make check TESTS="gdb.server/*.exp gdb.threads/*.exp".
+expanded_tests := $(patsubst $(srcdir)/%,%,$(wildcard $(addprefix $(srcdir)/,$(TESTS))))
+
+# Following are all tests we want to run within the installcheck-parallel target.
+# Those are either specified via the TESTS env var, or all the available tests.
+all_tests := $(or $(expanded_tests),$(shell cd $(srcdir) && find systemtap.* -name '*.exp' -print))
+
+# Tests that need to run serially. This namely applies to systemtap.server tests.
+# They start a new systemtap server and only expect one local server to run.
+all_serial_tests := $(shell cd $(srcdir) && find systemtap.server -name '*.exp' -print)
+serial_tests := $(filter $(all_tests),$(all_serial_tests))
+
+# There are a few tests known to be slow. We run them right from the beginning
+# to try to lessen the overall time taken by the test suite -- if one of these
+# tests happens to be run late, it will cause the overall time to increase.
+all_slow_tests = systemtap.examples/check.exp systemtap.pass1-4/buildok.exp \
+       systemtap.base/listing_mode.exp systemtap.syscall/nd_syscall.exp \
+       systemtap.unprivileged/unprivileged_myproc.exp systemtap.syscall/syscall.exp \
+       systemtap.base/sdt_misc.exp systemtap.pass1-4/parseok.exp \
+       systemtap.base/listing_mode_sanity.exp systemtap.pass1-4/parse-semok.exp \
+       systemtap.pass1-4/semok.exp systemtap.unprivileged/unprivileged_probes.exp \
+       systemtap.pass1-4/buildok-interactive.exp \
+       systemtap.onthefly/uprobes_onthefly.exp \
+       systemtap.onthefly/kprobes_onthefly.exp systemtap.base/sdt.exp \
+       systemtap.onthefly/hrtimer_onthefly.exp \
+       systemtap.onthefly/tracepoint_onthefly.exp systemtap.exelib/exelib.exp \
+       systemtap.pass1-4/semko.exp systemtap.pass1-4/parseko.exp \
+       systemtap.base/rlimit.exp systemtap.base/tracepoints.exp \
+       systemtap.base/systemtap-service.exp systemtap.maps/map_hash.exp \
+       systemtap.base/stmt_rel.exp systemtap.base/library.exp
+slow_tests = $(filter $(all_tests),$(all_slow_tests))
+
+regular_tests := $(filter-out $(serial_tests) $(slow_tests) ,$(all_tests))
+
+REGULAR_TEST_TARGETS := $(addprefix installcheck/,$(slow_tests) $(regular_tests))
+
+do-installcheck-parallel: installcheck-serial-tests $(REGULAR_TEST_TARGETS)
+       @:
+
+installcheck-serial-tests: site.exp
+       test -z $(serial_tests) || MAKEFLAGS= $(MAKE) $(AM_MAKEFLAGS) check-DEJAGNU RUNTESTFLAGS="--outdir=artifacts/__serialtests $(serial_tests) $(RUNTESTFLAGS) --tool_opts \'install $(TOOL_OPTS)\'" STAP_PARALLEL=yes
+
+installcheck/%.exp: site.exp
+       mkdir -p artifacts/$*
+       -MAKEFLAGS= $(MAKE) $(AM_MAKEFLAGS) check-DEJAGNU RUNTESTFLAGS="--outdir=artifacts/$* $*.exp $(RUNTESTFLAGS) --tool_opts \'install $(TOOL_OPTS)\'" STAP_PARALLEL=yes
+
 # $(srcdir)/These values point the test suite to the install tree, and
 # are overridden by "make check" from full source/build tree
 SYSTEMTAP_RUNTIME=$(DESTDIR)$(pkgdatadir)/runtime
@@ -40,6 +107,6 @@ CRASH_LIBDIR=$(DESTDIR)$(libdir)/systemtap
 SYSTEMTAP_PATH=$(DESTDIR)$(bindir)
 SYSTEMTAP_INCLUDES=$(DESTDIR)$(includedir)
 
-RUNTESTDEFAULTFLAGS = --tool $$tool --tool_opts \'$(TOOL_OPTS)\' --srcdir $$srcdir
+RUNTESTDEFAULTFLAGS = --tool $$tool --tool_opts \'$(TOOL_OPTS)\' --srcdir $(shell readlink -f $(srcdir))
 EXPECT = expect
 RUNTEST= env XDG_DATA_DIRS= SYSTEMTAP_SYNC=1 LANG=C SYSTEMTAP_TESTREMOTES=$(TESTREMOTES) SYSTEMTAP_TESTAPPS=$(TESTAPPS) SYSTEMTAP_RUNTIME=$(SYSTEMTAP_RUNTIME) SYSTEMTAP_TAPSET=$(SYSTEMTAP_TAPSET) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH) CRASH_LIBDIR=$(CRASH_LIBDIR) PATH=$(SYSTEMTAP_PATH):$$PATH SYSTEMTAP_PATH=$(SYSTEMTAP_PATH) SYSTEMTAP_INCLUDES=$(SYSTEMTAP_INCLUDES)  PKGLIBDIR=$(pkglibexecdir) $(srcdir)/execrc runtest