[PATCH v2] malloc: Improve checked_request2size

[Wilco Dijkstra]

v2: Use SIZE_MAX, add missing case in malloc-check.c

Change checked_request2size to return SIZE_MAX for huge inputs.  This
ensures large allocation request stay large and can't be confused with a
small allocation.

Passes regress, OK for commit?

---

diff --git a/malloc/malloc-check.c b/malloc/malloc-check.c
index fbb030116c3cc7e0b3a577822f11467d97c82b8d..44415e48a9de865cb3dc4b10c5e5c40be0f8f1ef 100644
--- a/malloc/malloc-check.c
+++ b/malloc/malloc-check.c
@@ -275,12 +275,12 @@ realloc_check (void *oldmem, size_t bytes)
     malloc_printerr ("realloc(): invalid pointer");
   const INTERNAL_SIZE_T oldsize = chunksize (oldp);
 
-  chnb = checked_request2size (rb);
-  if (chnb == 0)
+  if (rb > PTRDIFF_MAX)
     {
       __set_errno (ENOMEM);
       goto invert;
     }
+  chnb = checked_request2size (rb);
 
   __libc_lock_lock (main_arena.mutex);
 
diff --git a/malloc/malloc.c b/malloc/malloc.c
index c40b793f4a5d6b3b80e086f47a9ec72ad62b6526..efce6380d2fe575d7f2b96963167a869384238f1 100644
--- a/malloc/malloc.c
+++ b/malloc/malloc.c
@@ -1322,8 +1322,8 @@ nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
 /* Check if REQ overflows when padded and aligned and if the resulting
    value is less than PTRDIFF_T.  Returns the requested size or
-   MINSIZE in case the value is less than MINSIZE, or 0 if any of the
-   previous checks fail.  */
+   MINSIZE in case the value is less than MINSIZE, or SIZE_MAX if any
+   of the previous checks fail.  */
 static __always_inline size_t
 checked_request2size (size_t req) __nonnull (1)
 {
@@ -1331,7 +1331,7 @@ checked_request2size (size_t req) __nonnull (1)
                   "PTRDIFF_MAX is not more than half of SIZE_MAX");
 
   if (__glibc_unlikely (req > PTRDIFF_MAX))
-    return 0;
+    return SIZE_MAX;
 
   /* When using tagged memory, we cannot share the end of the user
      block with the header for the next chunk, so ensure that we
@@ -3513,12 +3513,12 @@ __libc_realloc (void *oldmem, size_t bytes)
        || __builtin_expect (misaligned_chunk (oldp), 0)))
       malloc_printerr ("realloc(): invalid pointer");
 
-  nb = checked_request2size (bytes);
-  if (nb == 0)
+  if (bytes > PTRDIFF_MAX)
     {
       __set_errno (ENOMEM);
       return NULL;
     }
+  nb = checked_request2size (bytes);
 
   if (chunk_is_mmapped (oldp))
     {
@@ -3887,12 +3887,12 @@ _int_malloc (mstate av, size_t bytes)
      aligned.
    */
 
-  nb = checked_request2size (bytes);
-  if (nb == 0)
+  if (bytes > PTRDIFF_MAX)
     {
       __set_errno (ENOMEM);
       return NULL;
     }
+  nb = checked_request2size (bytes);
 
   /* There are no usable arenas.  Fall back to sysmalloc to get a chunk from
      mmap.  */
@@ -5047,12 +5047,12 @@ _int_memalign (mstate av, size_t alignment, size_t bytes)
   unsigned long remainder_size;   /* its size */
   INTERNAL_SIZE_T size;
 
-  nb = checked_request2size (bytes);
-  if (nb == 0)
+  if (bytes > PTRDIFF_MAX)
     {
       __set_errno (ENOMEM);
       return NULL;
     }
+  nb = checked_request2size (bytes);
 
   /* We can't check tcache here because we hold the arena lock, which
      tcache doesn't expect.  We expect it has been checked