[PATCH v2] Makefile: add allow-list for failures

[Florian Weimer]

* Martin Coufal:

> Enable adding known failures to allowed-failures.txt and ignore failures
> in case they are in the list.  In case the allowed-failures.txt does not
> exist, all failures lead to a failed status as before.
>
> When the file is present, failures of listed tests are ignored and reported
> on stdout. If tests not in the allowed list fail, summarize-tests exits with
> status 1 and reports the failing tests.
>
> The expected format of allowed-failures.txt file is:
> <test_name> # <comment>
> ---
>  Makefile | 30 +++++++++++++++++++++++++++++-
>  1 file changed, 29 insertions(+), 1 deletion(-)
>
> diff --git a/Makefile b/Makefile
> index f81572c20a..6b0e055518 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -594,7 +594,35 @@ define summarize-tests
>  @grep -E '^[A-Z]+:' $(objpfx)$1 | grep -E -v '^(PASS|XFAIL):' || true
>  @echo "		=== Summary of results$2 ==="
>  @sed -e '/:.*/!d' -e 's/:.*//' < $(objpfx)$1 | sort | uniq -c
> -@! grep -E '^[A-Z]+:' $(objpfx)$1 | grep -E -q -v '^(X?PASS|XFAIL|UNSUPPORTED):'
> +@{ \
> +	grep -E '^[A-Z]+:' $(objpfx)$1 | \
> +	grep -E -v '^(X?PASS|XFAIL|UNSUPPORTED):' | \
> +	( \
> +	  if ! test -f $(..)allowed-failures.txt; then \
> +	    read -r _; exit $$(( $$? == 0 )); \
> +	  fi; \
> +	  status=0; \
> +	  while IFS= read -r line; do \
> +	    case "$$line" in \
> +	      FAIL:*) \
> +	        name=$${line#FAIL: }; \
> +	        escaped_name=`printf '%s' "$$name" | sed 's/[.+]/\\&/g'`; \
> +	        if grep -Eq -- "^$${escaped_name}[[:space:]]*#" \
> +	          $(..)allowed-failures.txt; then \
> +	          echo "Ignoring allowed FAIL: $${name}"; \
> +	          continue; \
> +	        fi; \
> +	        echo "Unallowed FAIL: $${name}"; \
> +	        status=1; \
> +	        ;; \
> +	      *) \
> +	        status=1; \
> +	        ;; \
> +	    esac; \
> +	  done; \
> +	  exit $$status; \
> +	); \
> +      }
>  endef
>  
>  # The intention here is to do ONE install of our build into the

This version looks okay to me.

Reviewed-by: Florian Weimer <fweimer@redhat.com>

Thanks,
Florian