This is the mail archive of the libc-alpha@sourceware.org mailing list for the glibc project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

Generate overall summary of test results


This patch, an updated version of
<https://sourceware.org/ml/libc-alpha/2014-01/msg00197.html> now
proposed for inclusion in glibc, makes testsuite runs generate an
overall summary of test results.

A new script merge-test-results.sh deals both with collecting results
within a directory to a file with all the results from that directory,
and collecting the results from subdirectories into a single overall
file (there's not much in common between the two modes of operation of
the script, but it seemed silly to have two separate scripts for
this).  Within a directory, missing results produce UNRESOLVED lines;
at top level, missing results for a whole directory produce an ERROR
line (since toplevel can't identify what the specific missing tests
are in this case).

Note that this does not change the rules for when "make" considers
there has been an error, or terminates, so unexpected failures will
still cause make to terminate, or, with -k, mean the commands for
"tests" don't get run because of failure of a dependency.

Tested x86_64, including that the summary does in fact reflect all the
tests with .test-result files.

(Changes from previous version: an error message is adjusted as
suggested by Brooks; the merge script is updated to reflect that
.test-result files now contain a second line with the original exit
status, which shouldn't end up in the .sum file, rather than just the
main test result; the makefile computations of the list of tests whose
results need merging are adjusted to reflect that tests producing
results in subdirectories now get the .test-result file and test name
properly reflecting those subdirectories.)

2014-03-06  Joseph Myers  <joseph@codesourcery.com>

	* scripts/merge-test-results.sh: New file.
	* Makefile (tests-special-notdir): New variable.
	(tests): Run merge-test-results.sh.
	(xtests): Likewise.
	* Rules (tests-special-notdir): New variable.
	(xtests-special-notdir): Likewise.
	(tests): Run merge-test-results.sh
	(xtests): Likewise.

diff --git a/Makefile b/Makefile
index aee88e1..8214dda 100644
--- a/Makefile
+++ b/Makefile
@@ -316,7 +316,18 @@ $(objpfx)begin-end-check.out: scripts/begin-end-check.pl
 	$(evaluate-test)
 endif
 
+tests-special-notdir = $(patsubst $(objpfx)%, %, $(tests-special))
 tests: $(tests-special)
+	$(..)scripts/merge-test-results.sh -s $(objpfx) "" \
+	  $(sort $(tests-special-notdir:.out=)) \
+	  > $(objpfx)subdir-tests.sum
+	$(..)scripts/merge-test-results.sh -t $(objpfx) subdir-tests.sum \
+	  $(sort $(subdirs) .) \
+	  > $(objpfx)tests.sum
+xtests:
+	$(..)scripts/merge-test-results.sh -t $(objpfx) subdir-xtests.sum \
+	  $(sort $(subdirs)) \
+	  > $(objpfx)xtests.sum
 
 # The realclean target is just like distclean for the parent, but we want
 # the subdirs to know the difference in case they care.
diff --git a/Rules b/Rules
index d37ae52..feb304d 100644
--- a/Rules
+++ b/Rules
@@ -102,6 +102,17 @@ tests: $(tests:%=$(objpfx)%.out) $(tests-special)
 xtests: tests $(xtests:%=$(objpfx)%.out) $(xtests-special)
 endif
 
+tests-special-notdir = $(patsubst $(objpfx)%, %, $(tests-special))
+xtests-special-notdir = $(patsubst $(objpfx)%, %, $(xtests-special))
+tests:
+	$(..)scripts/merge-test-results.sh -s $(objpfx) $(subdir) \
+	  $(sort $(tests) $(tests-special-notdir:.out=)) \
+	  > $(objpfx)subdir-tests.sum
+xtests:
+	$(..)scripts/merge-test-results.sh -s $(objpfx) $(subdir) \
+	  $(sort $(xtests) $(xtests-special-notdir:.out=)) \
+	  > $(objpfx)subdir-xtests.sum
+
 ifeq ($(build-programs),yes)
 binaries-all-notests = $(others) $(sysdep-others)
 binaries-all-tests = $(tests) $(xtests) $(test-srcs)
diff --git a/scripts/merge-test-results.sh b/scripts/merge-test-results.sh
new file mode 100755
index 0000000..6255e24
--- /dev/null
+++ b/scripts/merge-test-results.sh
@@ -0,0 +1,61 @@
+#! /bin/sh
+# Merge test results of individual tests or subdirectories.
+# Copyright (C) 2014 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public
+# License as published by the Free Software Foundation; either
+# version 2.1 of the License, or (at your option) any later version.
+
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <http://www.gnu.org/licenses/>.
+
+# usage: merge-test-results.sh -s objpfx subdir test-name...
+# (subdirectory tests; empty subdir at top level), or
+#        merge-test-results.sh -t objpfx subdir-file-name subdir...
+# (top-level merge)
+
+set -e
+
+type=$1
+objpfx=$2
+shift 2
+
+case $type in
+  -s)
+    subdir=$1
+    shift
+    subdir=${subdir:+$subdir/}
+    for t in "$@"; do
+      if [ -s "$objpfx$t.test-result" ]; then
+	head -n1 "$objpfx$t.test-result"
+      else
+	echo "UNRESOLVED: $subdir$t"
+      fi
+    done
+    ;;
+
+  -t)
+    subdir_file_name=$1
+    shift
+    for d in "$@"; do
+      if [ -f "$objpfx$d/$subdir_file_name" ]; then
+	cat "$objpfx$d/$subdir_file_name"
+      else
+	echo "ERROR: test results for $d directory missing"
+      fi
+    done
+    ;;
+
+  *)
+    echo "unknown type $type" >&2
+    exit 1
+    ;;
+esac

-- 
Joseph S. Myers
joseph@codesourcery.com


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]