Fix BZ #18757 - fmemopen fails to set errno on failure

[Paul Pluzhnikov]

On Fri, Aug 14, 2015 at 12:32 AM, Andreas Schwab <schwab@linux-m68k.org> wrote:
> Paul Pluzhnikov <ppluzhnikov@gmail.com> writes:
>
>> @@ -196,7 +197,10 @@ _IO_fopencookie (cookie, mode, io_functions)
>>
>>    new_f = (struct locked_FILE *) malloc (sizeof (struct locked_FILE));
>>    if (new_f == NULL)
>> -    return NULL;
>> +    {
>> +      __set_errno (ENOMEM);
>
> This will overwrite the errno set by malloc.

Thanks. Corrected patch attached.

2015-08-13  Paul Pluzhnikov  <ppluzhnikov@google.com>

        [BZ #18757]
        * libio/iofopncook.c (_IO_fopencookie): Set errno on failure
        * libio/test-fmemopen.c (do_bz18820): Extend the test to cover BZ #18757



-- 
Paul Pluzhnikov
-------------- next part --------------
diff --git a/libio/iofopncook.c b/libio/iofopncook.c
index b845d29..978a7fa 100644
--- a/libio/iofopncook.c
+++ b/libio/iofopncook.c
@@ -189,6 +189,7 @@ _IO_fopencookie (cookie, mode, io_functions)
       read_write = _IO_NO_READS|_IO_IS_APPENDING;
       break;
     default:
+      __set_errno (EINVAL);
       return NULL;
   }
   if (mode[0] == '+' || (mode[0] == 'b' && mode[1] == '+'))
diff --git a/libio/test-fmemopen.c b/libio/test-fmemopen.c
index e8e757f..a62f664 100644
--- a/libio/test-fmemopen.c
+++ b/libio/test-fmemopen.c
@@ -19,6 +19,7 @@
 
 static char buffer[] = "foobar";
 
+#include <errno.h>
 #include <stdio.h>
 #include <string.h>
 #include <errno.h>
@@ -30,6 +31,7 @@ do_bz18820 (void)
   char ch;
   FILE *stream;
 
+  errno = 0;
   stream = fmemopen (&ch, 1, "?");
   if (stream)
     {
@@ -37,6 +39,11 @@ do_bz18820 (void)
       fclose (stream);
       return 1;
     }
+  if (errno != EINVAL)
+    {
+      printf ("fmemopen: got %i, expected EINVAL (%i)\n", errno, EINVAL);
+      return 10;
+    }
 
   stream = fmemopen (NULL, 42, "?");
   if (stream)
@@ -46,6 +53,20 @@ do_bz18820 (void)
       return 2;
     }
 
+  errno = 0;
+  stream = fmemopen (NULL, ~0, "w");
+  if (stream)
+    {
+      printf ("fmemopen: expected NULL, got %p\n", stream);
+      fclose (stream);
+      return 3;
+    }
+  if (errno != ENOMEM)
+    {
+      printf ("fmemopen: got %i, expected ENOMEM (%i)\n", errno, ENOMEM);
+      return 20;
+    }
+
   return 0;
 }