[PATCH roland/test-unsupported] Let tests result in UNSUPPORTED; use that for unbuildable C++ cases

[Roland McGrath <roland at hack dot frob dot com>]

This gives our test infrastructure the beginnings of handling UNSUPPORTED
as an outcome for a test (as opposed to PASS, FAIL, or ERROR).
evaluate-test.sh now follows the Autotest convention of exit code 77
meaning a test that cannot be built or cannot be usefully run,
i.e. UNSUPPORTED.

For tests that cannot be built, the infrastructure for subdir makefiles now
supports a tests-unsupported variable.  This patch uses that for all the
missing-C++ cases that I touched earlier this week.

We have many existing cases of tests that do compile but are turned into
no-ops that always pass either at compile time or at run time.  Once this
lands, we should start converting those such that they report UNSUPPORTED
rather that PASS.  But that is a later step.

First I'd like to reach consensus on the script/makefile level changes here.

Notably, I've changed the summarizing stuff so that it prints out all
results that are neither PASS nor XFAIL, so that catches all ERROR and
XPASS as well as the new UNSUPPORTED.  Similarly, I've changed the logic
that decides whether 'make check' as a whole succeeded or not so that it
rejects anything that is not PASS, XFAIL, or UNSUPPORTED, rather than
rejecting only things that are exactly FAIL or ERROR.  This means that any
existing XPASS cases now cause 'make check' to fail.  I think that's the
right thing, since XPASS means something funny is going on and either the
test needs to be changed or the XFAIL expectation needs to be dropped.  I
also think it's definitely the right thing to both display and barf on any
statuses that are not in the expected set.


Thanks,
Roland


2015-03-06  Roland McGrath  <roland@hack.frob.com>

	* scripts/evaluate-test.sh: Grok exit code 77 as UNSUPPORTED and exit
	with 0 in that case.
	* Makefile (summarize-tests): New canned sequence, factored out of
	commands for targets tests and xtests.  Display summary lines that
	don't start with PASS: or XFAIL: rather than ones that do start with
	ERROR: or FAIL:.  Make the commands fail if any summary lines fail
	to start with PASS: or XFAIL: or UNSUPPORTED: rather than if any
	do start with ERROR: or FAIL:.
	* dlfcn/Makefile (tests): Add bug-atexit3 back here unconditionally
	(except for [$(build-shared) = yes]).
	(tests-unsupported) [$(CXX) empty]: Add bug-atexit3.
	(LDLIBS-bug-atexit3-lib.so): Conditionalize on [$(CXX) nonempty].
	($(objpfx)bug-atexit3, $(objpfx)bug-atexit3.out): Likewise.
	* nptl/Makefile: Revert 2015-03-04 changes.
	[$(CXX) empty] (tests-unsupported): New variable.
	* debug/Makefile: Likewise.

--- a/Makefile
+++ b/Makefile
@@ -316,6 +316,13 @@ $(objpfx)begin-end-check.out: scripts/begin-end-check.pl
 	$(evaluate-test)
 endif
 
+define summarize-tests
+@egrep -v '^(PASS|XFAIL):' $(objpfx)$1 || true
+@echo "Summary of test results$2:"
+@sed 's/:.*//' < $(objpfx)$1 | sort | uniq -c
+@egrep -q -v '^(PASS|XFAIL|UNSUPPORTED):' $(objpfx)$1 && false
+endef
+
 tests-special-notdir = $(patsubst $(objpfx)%, %, $(tests-special))
 tests: $(tests-special)
 	$(..)scripts/merge-test-results.sh -s $(objpfx) "" \
@@ -324,22 +331,12 @@ tests: $(tests-special)
 	$(..)scripts/merge-test-results.sh -t $(objpfx) subdir-tests.sum \
 	  $(sort $(subdirs) .) \
 	  > $(objpfx)tests.sum
-	@grep '^ERROR:' $(objpfx)tests.sum || true
-	@grep '^FAIL:' $(objpfx)tests.sum || true
-	@echo "Summary of test results:"
-	@sed 's/:.*//' < $(objpfx)tests.sum | sort | uniq -c
-	@if grep -q '^ERROR:' $(objpfx)tests.sum; then exit 1; fi
-	@if grep -q '^FAIL:' $(objpfx)tests.sum; then exit 1; fi
+	$(call summarize-tests,tests.sum)
 xtests:
 	$(..)scripts/merge-test-results.sh -t $(objpfx) subdir-xtests.sum \
 	  $(sort $(subdirs)) \
 	  > $(objpfx)xtests.sum
-	@grep '^ERROR:' $(objpfx)xtests.sum || true
-	@grep '^FAIL:' $(objpfx)xtests.sum || true
-	@echo "Summary of test results for extra tests:"
-	@sed 's/:.*//' < $(objpfx)xtests.sum | sort | uniq -c
-	@if grep -q '^ERROR:' $(objpfx)xtests.sum; then exit 1; fi
-	@if grep -q '^FAIL:' $(objpfx)xtests.sum; then exit 1; fi
+	$(call summarize-tests,xtests.sum, for extra tests)
 
 # The realclean target is just like distclean for the parent, but we want
 # the subdirs to know the difference in case they care.
--- a/Rules
+++ b/Rules
@@ -198,6 +198,18 @@ $(objpfx)%.out: /dev/null $(objpfx)%	# Make it 2nd arg for canned sequence.
 	$(make-test-out) > $@; \
 	$(evaluate-test)
 
+# tests-unsupported lists tests that we will not try to build at all in
+# this configuration.  Note this runs every time because it does not
+# actually create its target.  The dependency on Makefile is meant to
+# ensure that it runs after a Makefile change to add a test to the list
+# when it previously ran and produced a .out file (probably for a failure).
+ifneq "$(strip $(tests-unsupported))" ""
+$(tests-unsupported:%=$(objpfx)%.out): $(objpfx)%.out: Makefile
+	@rm -f $@
+	$(..)scripts/evaluate-test.sh $(patsubst $(common-objpfx)%.out,%,$@) \
+				      77 false false > $(@:.out=.test-result)
+endif
+
 endif	# tests
 
 
--- a/debug/Makefile
+++ b/debug/Makefile
@@ -133,11 +133,13 @@ LDFLAGS-tst-backtrace6 = -rdynamic
 
 tests = backtrace-tst tst-longjmp_chk tst-chk1 tst-chk2 tst-chk3 \
 	tst-lfschk1 tst-lfschk2 tst-lfschk3 test-strcpy_chk test-stpcpy_chk \
+	tst-chk4 tst-chk5 tst-chk6 tst-lfschk4 tst-lfschk5 tst-lfschk6 \
 	tst-longjmp_chk2 tst-backtrace2 tst-backtrace3 tst-backtrace4 \
 	tst-backtrace5 tst-backtrace6
 
-ifneq (,$(CXX))
-tests += tst-chk4 tst-chk5 tst-chk6 tst-lfschk4 tst-lfschk5 tst-lfschk6
+ifeq (,$(CXX))
+tests-unsupported = tst-chk4 tst-chk5 tst-chk6 \
+		    tst-lfschk4 tst-lfschk5 tst-lfschk6
 endif
 
 extra-libs = libSegFault libpcprofile
--- a/dlfcn/Makefile
+++ b/dlfcn/Makefile
@@ -36,7 +36,7 @@ endif
 ifeq (yes,$(build-shared))
 tests = glrefmain failtest tst-dladdr default errmsg1 tstcxaatexit \
 	bug-dlopen1 bug-dlsym1 tst-dlinfo bug-atexit1 bug-atexit2 \
-	tstatexit bug-dl-leaf tst-rec-dlopen
+	bug-atexit3 tstatexit bug-dl-leaf tst-rec-dlopen
 endif
 modules-names = glreflib1 glreflib2 glreflib3 failtestmod defaultmod1 \
 		defaultmod2 errmsg1mod modatexit modcxaatexit \
@@ -59,8 +59,9 @@ tststatic4-ENV = $(tststatic-ENV)
 tststatic5-ENV = $(tststatic-ENV)
 
 ifneq (,$(CXX))
-tests += bug-atexit3
 modules-names += bug-atexit3-lib
+else
+tests-unsupported += bug-atexit3
 endif
 endif
 
@@ -136,9 +137,11 @@ $(objpfx)bug-atexit1.out: $(objpfx)bug-atexit1-lib.so
 $(objpfx)bug-atexit2: $(libdl)
 $(objpfx)bug-atexit2.out: $(objpfx)bug-atexit2-lib.so
 
+ifneq (,$(CXX))
 LDLIBS-bug-atexit3-lib.so = -lstdc++ -lgcc_eh
 $(objpfx)bug-atexit3: $(libdl)
 $(objpfx)bug-atexit3.out: $(objpfx)bug-atexit3-lib.so
+endif
 
 $(objpfx)bug-dl-leaf: $(objpfx)bug-dl-leaf-lib.so
 $(objpfx)bug-dl-leaf.out: $(objpfx)bug-dl-leaf-lib-cb.so
--- a/nptl/Makefile
+++ b/nptl/Makefile
@@ -245,8 +245,8 @@ tests = tst-typesizes \
 	tst-cancel6 tst-cancel7 tst-cancel8 tst-cancel9 tst-cancel10 \
 	tst-cancel11 tst-cancel12 tst-cancel13 tst-cancel14 tst-cancel15 \
 	tst-cancel16 tst-cancel17 tst-cancel18 tst-cancel19 tst-cancel20 \
-	tst-cancel21 tst-cancel22 tst-cancel23 $(if $(CXX),tst-cancel24) \
-	tst-cancel25 tst-cancel-self tst-cancel-self-cancelstate \
+	tst-cancel21 tst-cancel22 tst-cancel23 tst-cancel24 tst-cancel25 \
+	tst-cancel-self tst-cancel-self-cancelstate \
 	tst-cancel-self-canceltype tst-cancel-self-testcancel \
 	tst-cleanup0 tst-cleanup1 tst-cleanup2 tst-cleanup3 tst-cleanup4 \
 	tst-flock1 tst-flock2 \
@@ -364,19 +364,14 @@ link-libc-static := $(common-objpfx)libc.a $(static-gnulib) \
 		    $(common-objpfx)libc.a
 
 tests-static += tst-locale1 tst-locale2 tst-stackguard1-static \
-		tst-cancel21-static tst-cond8-static \
+		tst-cancel21-static tst-cancel24-static tst-cond8-static \
 		tst-mutex8-static tst-mutexpi8-static tst-sem11-static \
 		tst-sem12-static
-tests += tst-stackguard1-static tst-cancel21-static \
+tests += tst-stackguard1-static tst-cancel21-static tst-cancel24-static \
 	 tst-cond8-static tst-mutex8-static tst-mutexpi8-static \
 	 tst-sem11-static tst-sem12-static
 xtests-static += tst-setuid1-static
 
-ifneq (,$(CXX))
-tests += tst-cancel24-static
-tests-static += tst-cancel24-static
-endif
-
 # These tests are linked with libc before libpthread
 tests-reverse += tst-cancel5 tst-cancel23 tst-vfork1x tst-vfork2x
 
@@ -388,6 +383,11 @@ tests-special += $(objpfx)tst-tls6.out $(objpfx)tst-cleanup0-cmp.out \
 endif
 endif
 
+ifeq (,$(CXX))
+# These tests require a C++ compiler and runtime.
+tests-unsupported += tst-cancel24 tst-cancel24-static
+endif
+
 include ../Rules
 
 ifeq (yes,$(build-shared))
--- a/scripts/evaluate-test.sh
+++ b/scripts/evaluate-test.sh
@@ -25,15 +25,20 @@ orig_rc=$rc
 xfail=$3
 stop_on_failure=$4
 
-if [ $rc -eq 0 ]; then
-  result="PASS"
-else
-  result="FAIL"
-fi
-
-if $xfail; then
-  result="X$result"
+if [ $rc -eq 77 ]; then
+  result="UNSUPPORTED"
   rc=0
+else
+  if [ $rc -eq 0 ]; then
+    result="PASS"
+  else
+    result="FAIL"
+  fi
+
+  if $xfail; then
+    result="X$result"
+    rc=0
+  fi
 fi
 
 echo "$result: $test_name"