[PATCH] Fix gettext ENOMEM handling

[Jakub Jelinek <jakub at redhat dot com>]

Hi!

This patch fixes the rest of the issues on the
http://sources.redhat.com/ml/libc-hacker/2007-07/msg00010.html
testcase.  __gconv_open returns __GCONV_NOCONV both for various errors
and also for GCONV_AVOID_NOCONV requested copy only conversion.
For errors we want gettext etc. to print untranslated message
preferrably over e.g. printing translated message in wrong encoding.
For no conversion we of course don't need to convert.
Removing GCONV_AVOID_NOCONV argument to __gconv_open in dcigettext.c
would fix this, but would be inefficient.
Fortunately it seems GCONV_AVOID_NOCONV is libc.so internal flag
and dcigettext.c is the only user of that flag, so IMHO best is just
to let __gconv_open return a different value for "no conversion
needed and GCONV_AVOID_NOCONV requested" from "no conversion available
or whatever other error happened".
I'm defining the new error code in gconv_int.h, so that it is gconv
internal thing, not part of the exported API e.g. for gconv modules.

2007-07-19  Jakub Jelinek  <jakub@redhat.com>

	* iconv/gconv_int.h (__GCONV_NULCONV): New internal only error code.
	* iconv/gconv_cache.c (__gconv_lookup_cache): Return __GCONV_NULCONV
	if from and to charsets are the same.
	* iconv/gconv_db.c (__gconv_find_transform): Likewise.
	* intl/dcigettext.c (_nl_find_msg): Return NULL even if __gconv_open
	returns __GCONV_NOCONV, but not for __GCONV_NULCONV.

--- libc/iconv/gconv_int.h.jj	2006-05-15 22:14:44.000000000 +0200
+++ libc/iconv/gconv_int.h	2007-07-19 21:07:16.000000000 +0200
@@ -1,4 +1,4 @@
-/* Copyright (C) 1997-2005, 2006 Free Software Foundation, Inc.
+/* Copyright (C) 1997-2005, 2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
 
@@ -114,6 +114,12 @@ enum
   GCONV_AVOID_NOCONV = 1 << 0
 };
 
+/* When GCONV_AVOID_NOCONV is set and no conversion is needed,
+   __GCONV_NULCONV should be returned.  */
+enum
+{
+  __GCONV_NULCONV = -1
+};
 
 /* Global variables.  */
 
--- libc/iconv/gconv_cache.c.jj	2006-01-11 08:08:29.000000000 +0100
+++ libc/iconv/gconv_cache.c	2007-07-19 21:09:19.000000000 +0200
@@ -1,5 +1,5 @@
 /* Cache handling for iconv modules.
-   Copyright (C) 2001, 2002, 2003, 2005 Free Software Foundation, Inc.
+   Copyright (C) 2001, 2002, 2003, 2005, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 2001.
 
@@ -285,7 +285,7 @@ __gconv_lookup_cache (const char *toset,
 
   /* Avoid copy-only transformations if the user requests.   */
   if (__builtin_expect (flags & GCONV_AVOID_NOCONV, 0) && fromidx == toidx)
-    return __GCONV_NOCONV;
+    return __GCONV_NULCONV;
 
   /* If there are special conversions available examine them first.  */
   if (fromidx != 0 && toidx != 0
--- libc/iconv/gconv_db.c.jj	2006-04-25 19:45:02.000000000 +0200
+++ libc/iconv/gconv_db.c	2007-07-19 21:08:58.000000000 +0200
@@ -1,5 +1,6 @@
 /* Provide access to the collection of available transformation modules.
-   Copyright (C) 1997-2003, 2004, 2005, 2006 Free Software Foundation, Inc.
+   Copyright (C) 1997-2003, 2004, 2005, 2006, 2007
+   Free Software Foundation, Inc.
    This file is part of the GNU C Library.
    Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
 
@@ -756,7 +757,7 @@ __gconv_find_transform (const char *tose
     {
       /* Both character sets are the same.  */
       __libc_lock_unlock (__gconv_lock);
-      return __GCONV_NOCONV;
+      return __GCONV_NULCONV;
     }
 
   result = find_derivation (toset, toset_expand, fromset, fromset_expand,
--- libc/intl/dcigettext.c.jj	2006-06-23 01:58:37.000000000 +0200
+++ libc/intl/dcigettext.c	2007-07-19 21:19:45.000000000 +0200
@@ -1,5 +1,5 @@
 /* Implementation of the internal dcigettext function.
-   Copyright (C) 1995-2005, 2006 Free Software Foundation, Inc.
+   Copyright (C) 1995-2005, 2006, 2007 Free Software Foundation, Inc.
    This file is part of the GNU C Library.
 
    The GNU C Library is free software; you can redistribute it and/or
@@ -948,7 +948,7 @@ _nl_find_msg (domain_file, domainbinding
 			/* If the output encoding is the same there is
 			   nothing to do.  Otherwise do not use the
 			   translation at all.  */
-			if (__builtin_expect (r != __GCONV_NOCONV, 1))
+			if (__builtin_expect (r != __GCONV_NULCONV, 1))
 			  return NULL;
 
 			convd->conv = (__gconv_t) -1;

	Jakub