[PATCH v4 5/5] manual: Replace summary.awk with summary.pl.

[Rical Jasan <ricaljasan at pacific dot net>]

This commit returns the build to a working state.  The Summary is now
generated from @standards, and syntax-checking is performed.  If
invalid @standards syntax is detected, summary.pl will fail, reporting
all errors.  Failure and error reporting is disabled for now, however,
since much of the manual is still incomplete wrt. header and standards
annotations.

Note that the sorting order of the Summary has changed; summary.pl
respects the locale, like summary.awk did, but the use of LC_ALL=C is
introduced in the Makefile.  Other notable deviations are improved
detection of the annotated elements' names, which are used for
sorting, and improved detection of the @node used to reference into
the manual.  The most noticeable difference in the rendered Summary is
that entries may now contain multiple lines, one for each header and
standard combination.

The summary.pl script also accepts a `--help' option, which details
the expected syntax of @standards.  If errors are reported, the user
is directed to this feature for further information.

	* manual/Makefile: Generate summary.texi with summary.pl.
	Force use of the C locale.
	* manual/header.texi: Update reference to summary.awk.
	* manual/summary.awk: Remove file.
	* manual/summary.pl: New file.  Generate summary.texi, or fail
	in the face of syntax errors, reporting them.
---
 manual/Makefile    |   7 +-
 manual/header.texi |   2 +-
 manual/summary.awk | 133 -----------------
 manual/summary.pl  | 423 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 427 insertions(+), 138 deletions(-)
 delete mode 100644 manual/summary.awk
 create mode 100755 manual/summary.pl

diff --git a/manual/Makefile b/manual/Makefile
index 510f160d3b..f05af4aefa 100644
--- a/manual/Makefile
+++ b/manual/Makefile
@@ -83,11 +83,10 @@ $(objpfx)libc/index.html: $(addprefix $(objpfx),$(libc-texi-generated))
 
 # Generate the summary from the Texinfo source files for each chapter.
 $(objpfx)summary.texi: $(objpfx)stamp-summary ;
-$(objpfx)stamp-summary: summary.awk $(filter-out $(objpfx)summary.texi, \
+$(objpfx)stamp-summary: summary.pl $(filter-out $(objpfx)summary.texi, \
 					$(texis-path))
 	$(SHELL) ./check-safety.sh $(filter-out $(objpfx)%, $(texis-path))
-	$(AWK) -f $^ | sort -t'' -df -k 1,1 | tr '\014' '\012' \
-		> $(objpfx)summary-tmp
+	LC_ALL=C $(PERL) $^ > $(objpfx)summary-tmp
 	$(move-if-change) $(objpfx)summary-tmp $(objpfx)summary.texi
 	touch $@
 
@@ -154,7 +153,7 @@ $(objpfx)%.pdf: %.texinfo
 
 
 # Distribution.
-minimal-dist = summary.awk texis.awk tsort.awk libc-texinfo.sh libc.texinfo \
+minimal-dist = summary.pl texis.awk tsort.awk libc-texinfo.sh libc.texinfo \
 	       libm-err.texi stamp-libm-err check-safety.sh		    \
 	       $(filter-out summary.texi, $(nonexamples))		    \
 	       $(patsubst %.c.texi,examples/%.c, $(examples))
diff --git a/manual/header.texi b/manual/header.texi
index 2a551cd6e1..ce661df43b 100644
--- a/manual/header.texi
+++ b/manual/header.texi
@@ -14,7 +14,7 @@ it.
 @end iftex
 @table @code
 @comment summary.texi is generated from the other Texinfo files.
-@comment See the Makefile and summary.awk for the details.
+@comment See the Makefile and summary.pl for the details.
 @include summary.texi
 @end table
 @iftex
diff --git a/manual/summary.awk b/manual/summary.awk
deleted file mode 100644
index 1defe616f7..0000000000
--- a/manual/summary.awk
+++ /dev/null
@@ -1,133 +0,0 @@
-# awk script to create summary.texinfo from the library texinfo files.
-# Copyright (C) 1992-2017 Free Software Foundation, Inc.
-# This file is part of the GNU C Library.
-
-# The GNU C Library is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2.1 of the License, or (at your option) any later version.
-
-# The GNU C Library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-# Lesser General Public License for more details.
-
-# You should have received a copy of the GNU Lesser General Public
-# License along with the GNU C Library; if not, see
-# <http://www.gnu.org/licenses/>.
-
-# This script recognizes sequences that look like:
-#	@comment HEADER.h
-#	@comment STANDARD
-#	@def... ITEM | @item ITEM | @vindex ITEM
-
-BEGIN { header = 0;
-nameword["@defun"]=1
-nameword["@defunx"]=1
-nameword["@defmac"]=1
-nameword["@defmacx"]=1
-nameword["@defspec"]=1
-nameword["@defspecx"]=1
-nameword["@defvar"]=1
-nameword["@defvarx"]=1
-nameword["@defopt"]=1
-nameword["@defoptx"]=1
-nameword["@deffn"]=2
-nameword["@deffnx"]=2
-nameword["@defvr"]=2
-nameword["@defvrx"]=2
-nameword["@deftp"]=2
-nameword["@deftpx"]=2
-nameword["@deftypefun"]=2
-nameword["@deftypefunx"]=2
-nameword["@deftypevar"]=2
-nameword["@deftypevarx"]=2
-nameword["@deftypefn"]=3
-nameword["@deftypefnx"]=3
-nameword["@deftypevr"]=3
-nameword["@deftypevrx"]=3
-firstword["@defun"]=1
-firstword["@defunx"]=1
-firstword["@defmac"]=1
-firstword["@defmacx"]=1
-firstword["@defspec"]=1
-firstword["@defspecx"]=1
-firstword["@defvar"]=1
-firstword["@defvarx"]=1
-firstword["@defopt"]=1
-firstword["@defoptx"]=1
-firstword["@deffn"]=2
-firstword["@deffnx"]=2
-firstword["@defvr"]=2
-firstword["@defvrx"]=2
-firstword["@deftp"]=2
-firstword["@deftpx"]=2
-firstword["@deftypefun"]=1
-firstword["@deftypefunx"]=1
-firstword["@deftypevar"]=1
-firstword["@deftypevarx"]=1
-firstword["@deftypefn"]=2
-firstword["@deftypefnx"]=2
-firstword["@deftypevr"]=2
-firstword["@deftypevrx"]=2
-nameword["@item"]=1
-firstword["@item"]=1
-nameword["@itemx"]=1
-firstword["@itemx"]=1
-nameword["@vindex"]=1
-firstword["@vindex"]=1
-
-print "@c DO NOT EDIT THIS FILE!"
-print "@c This file is generated by summary.awk from the Texinfo sources."
-}
-
-$1 == "@node" { node=$2;
-		for (i = 3; i <= NF; ++i)
-		 { node=node " " $i; if ( $i ~ /,/ ) break; }
-		sub (/,[, ]*$/, "", node);
-	      }
-
-$1 == "@comment" && $2 ~ /\.h$/ { header="@file{" $2 "}";
-				  for (i = 3; i <= NF; ++i)
-				    header=header ", @file{" $i "}"
-				}
-
-$1 == "@comment" && $2 == "(none)" { header = -1; }
-
-$1 == "@comment" && header != 0 { std=$2;
-				  for (i=3;i<=NF;++i) std=std " " $i }
-
-header != 0 && $1 ~ /@def|@item|@vindex/ \
-	{ defn=""; name=""; curly=0; n=1;
-	  for (i = 2; i <= NF; ++i) {
-	    if ($i ~ /^{/ && $i !~ /}/) {
-	      curly=1
-	      word=substr ($i, 2, length ($i))
-	    }
-	    else {
-	      if (curly) {
-	        if ($i ~ /}$/) {
-		  curly=0
-		  word=word " " substr ($i, 1, length ($i) - 1)
-	        } else
-		  word=word " " $i
-	      }
-	      # Handle a single word in braces.
-	      else if ($i ~ /^{.*}$/)
-		word=substr ($i, 2, length ($i) - 2)
-	      else
-	        word=$i
-	      if (!curly) {
-		if (n >= firstword[$1])
-		  defn=defn " " word
-		if (n == nameword[$1])
-		  name=word
-		++n
-	      }
-	    }
-	  }
-	  printf "@comment %s%c", name, 12 # FF
-	  printf "@item%s%c%c", defn, 12, 12
-	  if (header != -1) printf "%s ", header;
-	  printf "(%s):  @ref{%s}.%c\n", std, node, 12;
-	  header = 0 }
diff --git a/manual/summary.pl b/manual/summary.pl
new file mode 100755
index 0000000000..ceadbddf72
--- /dev/null
+++ b/manual/summary.pl
@@ -0,0 +1,423 @@
+#!/usr/bin/perl
+# Copyright (C) 2017 Free Software Foundation, Inc.
+# This file is part of the GNU C Library.
+# Contributed by Rical Jasan <ricaljasan@pacific.net>, 2017.
+
+# The GNU C Library is free software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public License
+# as published by the Free Software Foundation; either version 2.1 of
+# the License, or (at your option) any later version.
+
+# The GNU C Library is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# Lesser General Public License for more details.
+
+# You should have received a copy of the GNU Lesser General Public
+# License along with the GNU C Library; if not, see
+# <http://www.gnu.org/licenses/>.
+
+# Generate the Summary of Library Facilities (summary.texi).
+
+# Anything declared in a header or defined in a standard should have
+# its origins annotated using the @standards macro (see macro.texi).
+# This script checks all such elements in the manual (generally,
+# @def|item*-commands), ensuring annotations are present and correct.
+# If any errors are detected, they are all reported at the end and
+# failure is indicated.
+
+use strict;
+use warnings;
+use locale;
+use File::Basename;
+
+$| = 1;
+my $script = basename $0;
+
+&help if $ARGV[0] eq "--help"; # Will exit(0).
+
+my @texis = @ARGV;
+
+# Various regexes.
+my $nde = qr/^\@node /;
+my $def = qr/^\@def/;
+my $dfx = qr/^\@def\w+x /;
+my $itm = qr/^\@item /;
+my $itx = qr/^\@itemx /;
+my $item = qr/^\@itemx? /; # Don't match @itemize.
+my $ann = qr/^\@(def\w+|item)x? /; # Annotatable.
+my $std = qr/^\@standards\{/;
+my $stx = qr/^\@standardsx\{/;
+my $stds = qr/^\@standardsx?\{/;
+my $strict_std = qr/^\@standards\{([^,]+, )[^,\}]+\}$/;
+my $strict_stx = qr/^\@standardsx\{([^,]+, ){2}[^,\}]+\}$/;
+my $lcon = qr/([vf]?table|itemize|enumerate)/;
+my $list = qr/^\@${lcon}/;
+my $endl = qr/^\@end ${lcon}/;
+my $ign = qr/^\@ignore/;
+my $eig = qr/^\@end ignore/;
+
+# Global scope.
+my $node;
+our $texi;
+my $input;
+my %entries;
+my %errors;
+my $ignore;
+
+for $texi (@texis) {
+    open $input, '<', $texi or die "open $texi: $!";
+    while (my $line = <$input>) {
+	if ($line =~ $nde) {
+	    $node = &get_node($line);
+	} elsif ($line =~ $def) {
+	    &process_annotation($line);
+	} elsif ($line =~ $list) {
+	    &process_list($1); # @items occur in list or table context.
+	} elsif ($line =~ $stds) {
+	    &record_error("Misplaced \@standards", [$line]);
+	} elsif ($line =~ $ign) {
+	    while (<$input> !~ $eig) {}
+	}
+    }
+    close $input or die "close $texi: $!";
+}
+
+# Disabled until annotations are complete.
+&print_errors() if %errors && 0; # Will exit(1).
+
+print("\@c DO NOT EDIT THIS FILE!\n".
+      "\@c This file is generated by $script from the Texinfo sources.\n".
+      "\@c The \@items are \@include'd from a \@table in header.texi.\n\n");
+
+&print_entry($_) for sort keys %entries;
+
+# Processes an annotatable element, including any subsequent elements
+# in an @*x chain, ensuring @standards are present, with valid syntax,
+# either recording any errors detected or creating Summary entries.
+# This function is the heart of the script.
+#
+# Elements, prototypes, and standards are gathered into separate lists
+# and used to evaluate the completeness and correctness of annotations
+# before generating the Summary entry.  "Prototype" is used to refer
+# to an element's entire definition while avoiding conflation with
+# @def*-commands.  "Element" is strictly used here to refer to the
+# name extracted from the prototype, used for sorting the Summary, and
+# in @standardsx.
+sub process_annotation
+{
+    my $line = shift;
+    my ($i, $j);
+    my (@elements, @prototypes, @standards);
+
+    # Gather elements and prototypes.
+    push @prototypes, $line;
+    push @elements, &get_element($line);
+    while ($line = <$input>) {
+	last if $line !~ $ann;
+	push @prototypes, $line;
+	push @elements, &get_element($line);
+    }
+
+    # The fundamental error.
+    if ($line !~ $stds) {
+	return &record_error("Missing \@standards", \@prototypes);
+    }
+
+    # Gather standards.
+    push @standards, $line;
+    while (($line = <$input>) =~ $stds) {
+	push @standards, $line;
+    }
+
+    # Catch @standards embedded in @*x chains.  Don't match @item b/c
+    # they may occur consecutively, and should be considered
+    # independent.  @def*-commands will not be, however.
+    if ($line =~ $def || $line =~ $itx) {
+	push @prototypes, $line;
+	while (($line = <$input>) =~ $ann) {
+	    push @prototypes, $line;
+	}
+	return &record_error("Misplaced \@standards", \@prototypes);
+    }
+    # If it was an @item, seek back so we catch it on the next
+    # iteration.  This avoids imposing artificial Texinfo syntax
+    # requirements, like blank lines between consecutive annotated
+    # @items.
+    elsif ($line =~ $itm) {
+	seek $input, -length($line), 1 or die "seek: $!";
+    }
+
+    &check_standards(\@elements, \@prototypes, \@standards) or return;
+
+    # The @standards are aligned; make the Summary entry.  Stripping
+    # down the prototype was deferred until now because the syntax
+    # checks expect to have the full Texinfo input line.
+
+    for ($i=0, $j=0; $i<@elements; ++$i) {
+	my $element = $elements[$i];
+	my $prototype = &get_prototype($prototypes[$i]);
+	while ($standards[$j]
+	       && ($standards[$j] =~ $std
+		   || $standards[$j] =~ /$stx${element},/))
+	{
+	    my (undef, $standard, $header)
+		= $standards[$j++] =~ m/${stds}(([^,]+), ){1,2}([^,\}]+)/;
+	    # Key on prototypes too, as some elements have multiple
+	    # prototypes.  See isnan in arith.texi for one example.
+	    push(@{$entries{$element}{$prototype}},
+		 [$header, $standard, $node]);
+	}
+    }
+}
+
+# Performs various syntax checks on annotations.  Only called by
+# process_annotation, but separated out to keep the subroutines from
+# getting overly long, and maintain some logical separation.
+sub check_standards
+{
+    my ($i, $j);
+    my ($elements, $prototypes, $standards) = @_;
+
+    # Strict check for syntax errors.  Other matches are loose, which
+    # aids error detection and reporting by ensuring things that look
+    # like standards aren't simply passed over.
+    my @tmp;
+    for ($i=0; $i<@{$standards}; ++$i) {
+	my $standard = $standards->[$i];
+	if ($standard !~ $strict_std && $standard !~ $strict_stx) {
+	    push @tmp, $standard;
+	}
+    }
+    return &record_error("Invalid \@standards", \@tmp) if @tmp;
+
+    # All @standards must be either @*x or not @*x.  The equivalent
+    # test for @prototypes is omitted on the grounds that would be
+    # invalid Texinfo anyway.  This will also detect the syntax error
+    # of making the first @standards in an @*x chain non-x.
+    my $isx = $standards->[0] =~ $stx ? 1 : 0;
+    for ($i=1; $i<@{$standards}; ++$i) {
+	if (($standards->[$i] =~ $stx && ! $isx)
+	    || ($standards->[$i] =~ $std && $isx))
+	{
+	    return &record_error("Heterogeneous \@standards", $prototypes);
+	}
+    }
+
+    # Detect if an @*x-chain was completely annotated with @standards.
+    if (@{$prototypes} > 1 && ! $isx) {
+	my $x = 0;
+	for ($i=0; $i<@{$standards}; ++$i) {
+	    if ($standards->[$i] =~ $stx) {
+		$x = 1; last;
+	    }
+	}
+	if (!$x) {
+	    return &record_error("Requires \@standardsx", $prototypes);
+	}
+    }
+
+    # @*x chains may have multiple @standardsx, per-prototype.  Ensure
+    # at least one is present for each.  This check also assumes (er,
+    # enforces) elements and their @standardsx are in the same order.
+    if ($isx) {
+	for ($i=0, $j=0; $i<@{$elements}; ++$i) {
+	    my $lj = $j;
+	    my $e = $elements->[$i];
+	    ++$j while $standards->[$j] && $standards->[$j] =~ /${stx}${e},/;
+	    if ($j == $lj) {
+		return &record_error("Misordered \@standardsx", $prototypes)
+		    if $standards->[$j];
+		return &record_error("Partial \@standardsx", $prototypes);
+	    }
+	}
+	# This will catch @standardsx at the end of an otherwise
+	# complete and well-ordered list that didn't correspond to any
+	# element.  An extraneous @standardsx in the middle or at the
+	# beginning of the list will be reported as "Misordered
+	# @standards".  Figuring that out is left as an exercise for
+	# the writer.
+	if ($j < @{$standards}) {
+	    return &record_error("Extraneous \@standardsx", $prototypes);
+	}
+    }
+
+    return 1;
+}
+
+# Processes list or table contexts, with nesting.
+sub process_list
+{
+    my $type = shift;
+    my $in_vtbl = $type eq "vtable" ? 1 : 0;
+
+    while (my $line = <$input>) {
+	if ($line =~ $item) {
+	    next if ! $in_vtbl; # Not an annotatable context.
+	    &process_annotation($line);
+	} elsif ($line =~ $def) {
+	    &process_annotation($line);
+	} elsif ($line =~ $stds) {
+	    &record_error("Misplaced \@standards", [$line]);
+	} elsif ($line =~ $endl) {
+	    return; # All done.
+	} elsif ($line =~ $list) {
+	    &process_list($1); # Nested list.
+	}
+    }
+}
+
+# Returns the current node from an @node line.  Used for referencing
+# from the Summary.
+sub get_node
+{
+    my $line = shift;
+    chomp $line;
+    $line =~ s/$nde//;
+    my ($n) = split ',', $line;
+    return $n
+}
+
+# Returns the cleaned up prototype from @def|item* lines.
+sub get_prototype
+{
+    my $dfn = shift;
+    chomp $dfn;
+    $dfn =~ s/\s+/ /g; # Collapse whitespace.
+    $dfn =~ s/ \{([^\}]*)\} / $1 /g; # Remove grouping braces.
+    $dfn =~ s/^\@\S+ //; # Remove @-command.
+    $dfn =~ s/^Macro //i; # Scrape off cruft...
+    $dfn =~ s/^Data Type //i;
+    $dfn =~ s/^Variable //i;
+    $dfn =~ s/^Deprecated Function //i;
+    $dfn =~ s/^SVID Macro //i;
+    $dfn =~ s/^Obsolete function //i;
+    $dfn =~ s/^Constant //i;
+    $dfn =~ s/^Type //i;
+    $dfn =~ s/^Function //i;
+    $dfn =~ s/^\{(.*)\}$/$1/; # Debrace yourself.
+    $dfn =~ s/^\{([^\}]*)\} /$1 /; # These ones too.
+    return $dfn;
+}
+
+# Returns an annotated element's name.
+#
+# Takes a line defining an annotatable element (e.g., @def|item*),
+# splitting it on whitespace.  The element is generally detected as
+# the member immediately preceding the first parenthesized expression
+# (e.g., a function), or the last token in the list.  Some additional
+# cleanup is applied to the element before returning it.
+sub get_element
+{
+    my $i = 0;
+    my @toks = split /\s+/, shift;
+    # tzname array uses '['; don't match function pointers.
+    ++$i while $toks[$i] && $toks[$i] !~ /^[\(\[](?!\*)/;
+    $toks[$i-1] =~ s/^\*//; # Strip pointer type syntax.
+    $toks[$i-1] =~ s/^\{?([^\}]+)\}?$/$1/; # Strip braces.
+    $toks[$i-1] =~ s/^\(\*([^\)]+)\)$/$1/; # Function pointers.
+    return $toks[$i-1];
+}
+
+# Records syntax errors detected in the manual related to @standards.
+# The @def|item*s are grouped by file, then errors, to make it easier
+# to track down exactly where and what the problems are.
+sub record_error
+{
+    my ($err, $list) = @_;
+    push @{$errors{$texi}{$err}}, $_ for (@{$list});
+    return 0;
+}
+
+# Reports all detected errors and exits with failure.  Indentation is
+# used for readability, and "ERROR" is used for visibility.
+sub print_errors
+{
+    for $texi (sort keys %errors) {
+	print STDERR "ERRORS in $texi:\n";
+	for my $err (sort keys %{$errors{$texi}}) {
+	    print STDERR "  $err:\n";
+	    print STDERR "    $_" for (@{$errors{$texi}{$err}});
+	}
+    }
+    print(STDERR "\nFor a description of expected syntax, see ".
+	  "\`$script --help'\n\n");
+    exit 1;
+}
+
+# Prints an entry in the Summary.
+#
+# All the blank lines in summary.texi may seem strange at first, but
+# they have significant impact on how Texinfo renders the output.
+# Essentially, each line is its own paragraph.  There is a @comment
+# with the element name, arguably unnecessary, but useful for seeing
+# the sorting order and extracted element names, and maintains the
+# format established by summary.awk.  Each @item in the @table is the
+# prototype, which may be anything from just a variable name to a
+# function declaration.  The body of each @item contains lines
+# annotating the headers and standards each element is declared
+# in/comes from, with a reference to the @node documenting the element
+# wrt. each header and standard combination.
+sub print_entry
+{
+    my $element = shift;
+    for my $prototype (sort keys %{$entries{$element}}) {
+	print "\@comment $element\n\@item $prototype\n\n";
+	for (@{$entries{$element}{$prototype}}) {
+	    my ($header, $standard, $node)
+		= ($_->[0], $_->[1], $_->[2]);
+	    if ($header =~ /^\(none\)$/i) {
+		$header = "\@emph{no header}";
+	    } elsif ($header ne '???') {
+		$header = "\@file{$header}";
+	    }
+	    print "$header ($standard):  \@ref{$node}.\n\n";
+	}
+    }
+}
+
+# Document the syntax of @standards.
+sub help
+{
+    print <<EOH;
+$script generates the Summary of Library Facilities
+(summary.texi) from \@standards and \@standardsx macros in the
+Texinfo sources (see macros.texi).  While generating the Summary,
+it also checks that \@standards are used, and used correctly.
+
+In general, any \@def*-command or \@item in a \@vtable is considered
+annotatable.  "Misplaced \@standards" refers to \@standards macros
+detected outside an annotatable context.  \@standards are expected to
+immediately follow the elements being annotated.
+
+The syntax of \@standards annotations is designed to accomodate
+multiple headers and\/or standards for any element (function, macro,
+variable, etc.).
+
+Examples:
+
+  \@deftp FOO
+  \@standards{STD, HDR}
+
+  \@defvar BAR
+  \@standards{STD, HDR1}
+  \@standards{STD, HDR2}
+
+  \@deftypefun foo
+  \@deftypefunx fool
+  \@standardsx{foo, STD, HDR}
+  \@standardsx{fool, STD, HDR}
+
+  \@item bar
+  \@itemx baz
+  \@standardsx{bar, STD1, HDR1}
+  \@standardsx{baz, STD1, HDR1}
+  \@standardsx{baz, STD2, HDR2}
+
+Note that \@standardsx deviates from the Texinfo convention of the
+first \@-command being non-x, in order to provide a means to
+distinguish the \@standards of each annotated element.  \@standardsx
+must occur in the same order as the annotated elements.
+EOH
+    ; exit 0;
+}