Re: Do not terminate default test runs on test failure

["Joseph S. Myers" <joseph at codesourcery dot com>]

Here is a version revised to exit with error status if any FAILs occur 
(and with correspondingly reworked documentation).  Tested x86_64.

2014-03-13  Joseph Myers  <joseph@codesourcery.com>

	* scripts/evaluate-test.sh: Handle fourth argument to determine
	whether test run should stop on failure.
	* Makeconfig (stop-on-test-failure): New variable.
	(evaluate-test): Pass fourth argument to evaluate-test.sh based on
	$(stop-on-test-failure).
	* Makefile (tests): Give a summary of results from testing and
	exit with failure status if they include an ERROR or FAIL.
	(xtests): Likewise.
	* manual/install.texi (Configuring and compiling): Mention
	stop-on-test-failure=y.
	* INSTALL: Regenerated.

diff --git a/INSTALL b/INSTALL
index f845940..13fb678 100644
--- a/INSTALL
+++ b/INSTALL
@@ -203,6 +203,12 @@ The tests (and later installation) use some pre-existing files of the
 system such as `/etc/passwd', `/etc/nsswitch.conf' and others.  These
 files must all contain correct and sensible content.
 
+   Normally, `make check' will run all the tests before reporting all
+problems found and exiting with error status if any problems occurred.
+You can specify `stop-on-test-failure=y' when running `make check' to
+make the test run stop and exit with an error status immediately when a
+failure occurs.
+
    To format the `GNU C Library Reference Manual' for printing, type
 `make dvi'.  You need a working TeX installation to do this.  The
 distribution builds the on-line formatted version of the manual, as
diff --git a/Makeconfig b/Makeconfig
index 9078b29..0a2e12b 100644
--- a/Makeconfig
+++ b/Makeconfig
@@ -601,6 +601,12 @@ run-built-tests = yes
 endif
 endif
 
+# Whether to stop immediately when a test fails.  Nonempty means to
+# stop, empty means not to stop.
+ifndef stop-on-test-failure
+stop-on-test-failure =
+endif
+
 # How to run a program we just linked with our library.
 # The program binary is assumed to be $(word 2,$^).
 built-program-file = $(dir $(word 2,$^))$(notdir $(word 2,$^))
@@ -1091,6 +1097,7 @@ test-xfail-name = $(strip $(patsubst %.out, %, $(patsubst $(objpfx)%, %, $@)))
 # XPASS or XFAIL rather than PASS or FAIL.
 evaluate-test = $(..)scripts/evaluate-test.sh $(test-name) $$? \
 		  $(if $(test-xfail-$(test-xfail-name)),true,false) \
+		  $(if $(stop-on-test-failure),true,false) \
 		  > $(common-objpfx)$(test-name).test-result
 
 endif # Makeconfig not yet included
diff --git a/Makefile b/Makefile
index 8214dda..e026fb1 100644
--- a/Makefile
+++ b/Makefile
@@ -324,10 +324,22 @@ tests: $(tests-special)
 	$(..)scripts/merge-test-results.sh -t $(objpfx) subdir-tests.sum \
 	  $(sort $(subdirs) .) \
 	  > $(objpfx)tests.sum
+	@grep '^ERROR:' $(objpfx)tests.sum || true
+	@grep '^FAIL:' $(objpfx)tests.sum || true
+	@echo "Summary of test results:"
+	@sed 's/:.*//' < $(objpfx)tests.sum | sort | uniq -c
+	@if grep -q '^ERROR:' $(objpfx)tests.sum; then exit 1; fi
+	@if grep -q '^FAIL:' $(objpfx)tests.sum; then exit 1; fi
 xtests:
 	$(..)scripts/merge-test-results.sh -t $(objpfx) subdir-xtests.sum \
 	  $(sort $(subdirs)) \
 	  > $(objpfx)xtests.sum
+	@grep '^ERROR:' $(objpfx)xtests.sum || true
+	@grep '^FAIL:' $(objpfx)xtests.sum || true
+	@echo "Summary of test results for extra tests:"
+	@sed 's/:.*//' < $(objpfx)xtests.sum | sort | uniq -c
+	@if grep -q '^ERROR:' $(objpfx)xtests.sum; then exit 1; fi
+	@if grep -q '^FAIL:' $(objpfx)xtests.sum; then exit 1; fi
 
 # The realclean target is just like distclean for the parent, but we want
 # the subdirs to know the difference in case they care.
diff --git a/NEWS b/NEWS
index e8e7f80..f00f147 100644
--- a/NEWS
+++ b/NEWS
@@ -13,6 +13,13 @@ Version 2.20
   16611, 16613, 16623, 16632, 16639, 16642, 16670, 16674, 16677, 16683,
   16689, 16695.
 
+* Running the testsuite no longer terminates as soon as a test fails.
+  Instead, a file tests.sum (xtests.sum from "make xcheck") is generated,
+  with PASS or FAIL lines for individual tests.  A summary of the results is
+  printed, including a list of failing lists, and "make check" exits with
+  error status if there were any unexpected failures.  "make check
+  stop-on-test-failure=y" may be used to keep the old behavior.
+
 * The am33 port, which had not worked for several years, has been removed
   from ports.
 
diff --git a/manual/install.texi b/manual/install.texi
index 8562bdc..d2b8c1e 100644
--- a/manual/install.texi
+++ b/manual/install.texi
@@ -235,6 +235,12 @@ The tests (and later installation) use some pre-existing files of the
 system such as @file{/etc/passwd}, @file{/etc/nsswitch.conf} and others.
 These files must all contain correct and sensible content.
 
+Normally, @code{make check} will run all the tests before reporting
+all problems found and exiting with error status if any problems
+occurred.  You can specify @samp{stop-on-test-failure=y} when running
+@code{make check} to make the test run stop and exit with an error
+status immediately when a failure occurs.
+
 To format the @cite{GNU C Library Reference Manual} for printing, type
 @w{@code{make dvi}}.  You need a working @TeX{} installation to do
 this.  The distribution builds the on-line formatted version of the
diff --git a/scripts/evaluate-test.sh b/scripts/evaluate-test.sh
index c8f5012..2a5c156 100755
--- a/scripts/evaluate-test.sh
+++ b/scripts/evaluate-test.sh
@@ -17,12 +17,13 @@
 # License along with the GNU C Library; if not, see
 # <http://www.gnu.org/licenses/>.
 
-# usage: evaluate-test.sh test_name rc xfail
+# usage: evaluate-test.sh test_name rc xfail stop_on_failure
 
 test_name=$1
 rc=$2
 orig_rc=$rc
 xfail=$3
+stop_on_failure=$4
 
 if [ $rc -eq 0 ]; then
   result="PASS"
@@ -37,4 +38,8 @@ fi
 
 echo "$result: $test_name"
 echo "original exit status $orig_rc"
-exit $rc
+if $stop_on_failure; then
+  exit $rc
+else
+  exit 0
+fi

-- 
Joseph S. Myers
joseph@codesourcery.com