argp fixes for some size-calculation overflows

[Paul Eggert]

I looked through gnulib code for size-calculation overflows when
allocating or reallocating memory, and found two potential problems in
argp.  Not many people have gigabytes' worth of arguments, but on the
other hand GNU software isn't supposed to have arbitrary limits.
There are several other problems in argp (e.g., many objects are of
type 'unsigned' but store size_t-related values, which doesn't work on
most 64-bit hosts) but I didn't fix them.

Here are proposed patches.  They are relative to gnulib but also apply
cleanly to glibc.

2003-09-24  Paul Eggert  <eggert@twinsun.com>

	* argp/argp-fmtstream.c (__argp_fmtstream_ensure): Check for
	size_t overflow when reallocating storage.
	* argp/argp-help.c (make_hol, hol_append): Likewise.
	(SIZE_MAX): New macro.

Index: lib/argp-fmtstream.c
===================================================================
RCS file: /cvsroot/gnulib/gnulib/lib/argp-fmtstream.c,v
retrieving revision 1.2
diff -p -u -r1.2 argp-fmtstream.c
--- lib/argp-fmtstream.c	8 Jul 2003 23:25:14 -0000	1.2
+++ lib/argp-fmtstream.c	24 Sep 2003 21:13:28 -0000
@@ -384,10 +384,11 @@ __argp_fmtstream_ensure (struct argp_fmt
       if ((size_t) (fs->end - fs->buf) < amount)
 	/* Gotta grow the buffer.  */
 	{
-	  size_t new_size = fs->end - fs->buf + amount;
-	  char *new_buf = realloc (fs->buf, new_size);
+	  size_t old_size = fs->end - fs->buf;
+	  size_t new_size = old_size + amount;
+	  char *new_buf;
 
-	  if (! new_buf)
+	  if (new_size < old_size || ! (new_buf = realloc (fs->buf, new_size)))
 	    {
 	      __set_errno (ENOMEM);
 	      return 0;
Index: lib/argp-help.c
===================================================================
RCS file: /cvsroot/gnulib/gnulib/lib/argp-help.c,v
retrieving revision 1.4
diff -p -u -r1.4 argp-help.c
--- lib/argp-help.c	13 Aug 2003 20:23:14 -0000	1.4
+++ lib/argp-help.c	24 Sep 2003 21:13:29 -0000
@@ -72,6 +72,10 @@ char *alloca ();
 #include "argp.h"
 #include "argp-fmtstream.h"
 #include "argp-namefrob.h"
+
+#ifndef SIZE_MAX
+# define SIZE_MAX ((size_t) -1)
+#endif 
 
 /* User-selectable (using an environment variable) formatting parameters.
 
@@ -440,7 +444,8 @@ make_hol (const struct argp *argp, struc
       hol->entries = malloc (sizeof (struct hol_entry) * hol->num_entries);
       hol->short_options = malloc (num_short_options + 1);
 
-      assert (hol->entries && hol->short_options);
+      assert (hol->entries && hol->short_options
+	      && hol->num_entries <= SIZE_MAX / sizeof (struct hol_entry));
 
       /* Fill in the entries.  */
       so = hol->short_options;
@@ -832,6 +837,9 @@ hol_append (struct hol *hol, struct hol 
 	  unsigned hol_so_len = strlen (hol->short_options);
 	  char *short_options =
 	    malloc (hol_so_len + strlen (more->short_options) + 1);
+
+	  assert (entries && short_options
+		  && num_entries <= SIZE_MAX / sizeof (struct hol_entry));
 
 	  __mempcpy (__mempcpy (entries, hol->entries,
 				hol->num_entries * sizeof (struct hol_entry)),