[patch] Fix BZ 19165 -- overflow in fread / fwrite

[Paul Pluzhnikov]

Ok, let's try one more time.

Previous attempt: https://sourceware.org/ml/libc-alpha/2015-10/msg00894.html
Tested on Linux/x86_64 with 6.0.0 (r222386).

Testing with (gcc-4.8 Ubuntu 4.8.4-2ubuntu1~14.04) 4:4.8.2-1ubuntu6
fails for apparently unrelated reasons:

build/math/test-double-vlen2-wrappers.o: In function `cos_vlen2':
/glibc-git/math/../sysdeps/x86_64/fpu/test-double-vlen2-wrappers.c:24:
undefined reference to `_ZGVbN2v_cos'
/glibc-git/build-system-gcc/math/test-double-vlen2-wrappers.o: In
function `sin_vlen2':
/glibc-git/math/../sysdeps/x86_64/fpu/test-double-vlen2-wrappers.c:25:
undefined reference to `_ZGVbN2v_sin'
... etc.

Thanks,


2015-12-07  Paul Pluzhnikov  <ppluzhnikov@google.com>

        [BZ #19165]
        * misc/sys/cdefs.h (__umul_size_t_overflow): New.
        (__umul_size_t_saturated): New.
        * libio/iofread.c (_IO_fread): Protect against overflow.
        * libio/iofread_u.c (__fread_unlocked): Likewise.
        * b/libio/iofwrite.c (_IO_fwrite): Likewise.
        * libio/iofwrite_u.c (fwrite_unlocked): Likewise.

-- 
Paul Pluzhnikov
-------------- next part --------------
diff --git a/libio/iofread.c b/libio/iofread.c
index eb69b05..515f862 100644
--- a/libio/iofread.c
+++ b/libio/iofread.c
@@ -29,7 +29,7 @@
 _IO_size_t
 _IO_fread (void *buf, _IO_size_t size, _IO_size_t count, _IO_FILE *fp)
 {
-  _IO_size_t bytes_requested = size * count;
+  _IO_size_t bytes_requested = __umul_size_t_saturated (size, count);
   _IO_size_t bytes_read;
   CHECK_FILE (fp, 0);
   if (bytes_requested == 0)
diff --git a/libio/iofread_u.c b/libio/iofread_u.c
index 997b714..9e2d213 100644
--- a/libio/iofread_u.c
+++ b/libio/iofread_u.c
@@ -32,7 +32,7 @@
 _IO_size_t
 __fread_unlocked (void *buf, _IO_size_t size, _IO_size_t count, _IO_FILE *fp)
 {
-  _IO_size_t bytes_requested = size * count;
+  _IO_size_t bytes_requested = __umul_size_t_saturated (size, count);
   _IO_size_t bytes_read;
   CHECK_FILE (fp, 0);
   if (bytes_requested == 0)
diff --git a/libio/iofwrite.c b/libio/iofwrite.c
index 48ad4bc..bb34257 100644
--- a/libio/iofwrite.c
+++ b/libio/iofwrite.c
@@ -29,7 +29,7 @@
 _IO_size_t
 _IO_fwrite (const void *buf, _IO_size_t size, _IO_size_t count, _IO_FILE *fp)
 {
-  _IO_size_t request = size * count;
+  _IO_size_t request = __umul_size_t_saturated (size, count);
   _IO_size_t written = 0;
   CHECK_FILE (fp, 0);
   if (request == 0)
diff --git a/libio/iofwrite_u.c b/libio/iofwrite_u.c
index 2b1c47a..6e002bf 100644
--- a/libio/iofwrite_u.c
+++ b/libio/iofwrite_u.c
@@ -33,7 +33,7 @@ _IO_size_t
 fwrite_unlocked (const void *buf, _IO_size_t size, _IO_size_t count,
 		 _IO_FILE *fp)
 {
-  _IO_size_t request = size * count;
+  _IO_size_t request = __umul_size_t_saturated (size, count);
   _IO_size_t written = 0;
   CHECK_FILE (fp, 0);
   if (request == 0)
diff --git a/misc/sys/cdefs.h b/misc/sys/cdefs.h
index 99e94cc..4474f09 100644
--- a/misc/sys/cdefs.h
+++ b/misc/sys/cdefs.h
@@ -441,4 +441,37 @@
 # endif
 #endif
 
+#include <stddef.h>  /* for size_t.  */
+
+__always_inline
+static int
+__umul_size_t_overflow (size_t a, size_t b)
+{
+#if __GNUC_PREREQ (5, 0)
+  size_t result;
+
+  /* _Static_assert, but without triggering conformance test failures.  */
+  __attribute__ ((__unused__))
+  char sizeof_size_t_eq_sizeof_ulong[
+    sizeof (size_t) == sizeof (unsigned long) ? 1 : -1];
+
+  return __builtin_umull_overflow (a, b, &result);
+#else
+  const size_t half_size_t = (size_t) 1 << 4 * sizeof (size_t);
+  const size_t size_max = (size_t) -1;
+
+  return __glibc_unlikely ((a | b) >= half_size_t) && b > 1 && a > size_max / b;
+#endif
+}
+
+__always_inline
+static size_t
+__umul_size_t_saturated (size_t a, size_t b)
+{
+  const size_t size_max = (size_t) -1;
+
+  if (__umul_size_t_overflow (a, b)) return size_max;
+  return a * b;
+}
+
 #endif	 /* sys/cdefs.h */