Unicode 3.1 support (6)
Bruno Haible
haible@ilog.fr
Tue Jun 5 06:31:00 GMT 2001
Unicode 3.1 introduces special characters called "tag characters" which
serve to declare (in a stateful manner!) the language of the following
plain text. It is ugly, but it is a compromise. Without these characters,
Unicode is apparently not acceptable to some Japanese people. Fortunately,
the support for us is easy: Unicode recommends
"If language codes are not relevant to the particular processing
operation, then they should be ignored."
This means that all converters from Unicode to a non-Unicode encoding must
ignore these tag characters if they occur in the input stream. Only the
ISO-2022-JP-2 encoder can make use of them.
2001-06-04 Bruno Haible <haible@clisp.cons.org>
* iconv/loop.c (UNICODE_TAG_HANDLER): New macro.
* iconv/gconv_simple.c (__gconv_transform_internal_ascii): Invoke
UNICODE_TAG_HANDLER.
(__gconv_transform_internal_ucs2): Likewise.
(__gconv_transform_internal_ucs2reverse): Likewise.
* iconvdata/8bit-gap.c (BODY for TO_LOOP): Invoke UNICODE_TAG_HANDLER.
* iconvdata/8bit-generic.c (BODY for TO_LOOP): Likewise.
* iconvdata/ansi_x3.110.c (BODY for TO_LOOP): Likewise.
* iconvdata/big5.c (BODY for TO_LOOP): Likewise.
* iconvdata/big5hkscs.c (BODY for TO_LOOP): Likewise.
* iconvdata/cp1255.c (BODY for TO_LOOP): Likewise.
* iconvdata/cp1258.c (BODY for TO_LOOP): Likewise.
* iconvdata/euc-cn.c (BODY for TO_LOOP): Likewise.
* iconvdata/euc-jp.c (BODY for TO_LOOP): Likewise.
* iconvdata/euc-kr.c (BODY for TO_LOOP): Likewise.
* iconvdata/euc-tw.c (BODY for TO_LOOP): Likewise.
* iconvdata/gbk.c (BODY for TO_LOOP): Likewise.
* iconvdata/ibm930.c (BODY for TO_LOOP): Likewise.
* iconvdata/ibm932.c (BODY for TO_LOOP): Likewise.
* iconvdata/ibm933.c (BODY for TO_LOOP): Likewise.
* iconvdata/ibm935.c (BODY for TO_LOOP): Likewise.
* iconvdata/ibm937.c (BODY for TO_LOOP): Likewise.
* iconvdata/ibm939.c (BODY for TO_LOOP): Likewise.
* iconvdata/ibm943.c (BODY for TO_LOOP): Likewise.
* iconvdata/iso646.c (BODY for TO_LOOP): Likewise.
* iconvdata/iso8859-1.c (BODY for TO_LOOP): Likewise.
* iconvdata/iso_6937.c (BODY for TO_LOOP): Likewise.
* iconvdata/iso_6937-2.c (BODY for TO_LOOP): Likewise.
* iconvdata/iso-2022-cn.c (BODY for TO_LOOP): Likewise.
* iconvdata/iso-2022-cn-ext.c (BODY for TO_LOOP): Likewise.
* iconvdata/iso-2022-kr.c (BODY for TO_LOOP): Likewise.
* iconvdata/johab.c (BODY for TO_LOOP): Likewise.
* iconvdata/sjis.c (BODY for TO_LOOP): Likewise.
* iconvdata/t.61.c (BODY for TO_LOOP): Likewise.
* iconvdata/uhc.c (BODY for TO_LOOP): Likewise.
* iconvdata/unicode.c (BODY for TO_LOOP): Likewise.
* iconvdata/iso-2022-jp.c (TAG_none, TAG_language, TAG_language_j,
TAG_language_ja, TAG_language_k, TAG_language_ko, TAG_language_z,
TAG_language_zh, CURRENT_TAG_MASK): New enum values.
(EMIT_SHIFT_TO_INIT): Don't emit an escape sequence if ASCII_set
is already selected but set2 or tag are set.
(conversion): New enum type.
(cvlist_t): New type.
(CVLIST, CVLIST_FIRST, CVLIST_REST): New macros.
(conversion_lists): New array.
(BODY for TO_LOOP): Keep track of Unicode 3.1 language tag. If "ja",
prefer conversion to Japanese character sets. If "zh", prefer
conversion to GB2312. If "ko", prefer conversion to KSC5601. Small
optimizations.
(INIT_PARAMS): Add tag.
(UPDATE_PARAMS): Add tag.
--- glibc-20010430/iconv/loop.c.bak Mon Nov 27 23:28:38 2000
+++ glibc-20010430/iconv/loop.c Mon May 28 03:53:17 2001
@@ -1,5 +1,5 @@
/* Conversion loop frame work.
- Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
@@ -212,6 +233,21 @@
}
+/* Handling of Unicode 3.1 TAG characters. Unicode recommends
+ "If language codes are not relevant to the particular processing
+ operation, then they should be ignored."
+ This macro is usually called right before STANDARD_ERR_HANDLER (Incr). */
+#define UNICODE_TAG_HANDLER(Character, Incr) \
+ { \
+ /* TAG characters are those in the range U+E0000..U+E007F. */ \
+ if (((Character) >> 7) == (0xe0000 >> 7)) \
+ { \
+ inptr += Incr; \
+ continue; \
+ } \
+ }
+
+
/* The function returns the status, as defined in gconv.h. */
static inline int
FCTNAME (LOOPFCT) (struct __gconv_step *step,
--- glibc-20010430/iconv/gconv_simple.c.bak Tue Apr 10 23:52:15 2001
+++ glibc-20010430/iconv/gconv_simple.c Mon May 28 03:30:18 2001
@@ -807,6 +807,7 @@
{ \
if (__builtin_expect (*((const uint32_t *) inptr), 0) > 0x7f) \
{ \
+ UNICODE_TAG_HANDLER (*((const uint32_t *) inptr), 4); \
STANDARD_ERR_HANDLER (4); \
} \
else \
@@ -1199,6 +1200,7 @@
\
if (__builtin_expect (val, 0) >= 0x10000) \
{ \
+ UNICODE_TAG_HANDLER (val, 4); \
STANDARD_ERR_HANDLER (4); \
} \
else if (__builtin_expect (val >= 0xd800 && val < 0xe000, 0)) \
@@ -1288,6 +1290,7 @@
uint32_t val = *((const uint32_t *) inptr); \
if (__builtin_expect (val, 0) >= 0x10000) \
{ \
+ UNICODE_TAG_HANDLER (val, 4); \
STANDARD_ERR_HANDLER (4); \
} \
else if (__builtin_expect (val >= 0xd800 && val < 0xe000, 0)) \
--- glibc-20010430/iconvdata/8bit-gap.c.bak Mon Jul 3 16:39:26 2000
+++ glibc-20010430/iconvdata/8bit-gap.c Mon May 28 03:33:33 2001
@@ -1,6 +1,6 @@
/* Generic conversion to and from 8bit charsets,
converting from UCS using gaps.
- Copyright (C) 1997, 1998, 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1997, 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
@@ -83,7 +83,10 @@
unsigned char res; \
\
if (__builtin_expect (ch, 0) >= 0xffff) \
- rp = NULL; \
+ { \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ rp = NULL; \
+ } \
else \
while (ch > rp->end) \
++rp; \
--- glibc-20010430/iconvdata/8bit-generic.c.bak Mon Jul 3 16:39:26 2000
+++ glibc-20010430/iconvdata/8bit-generic.c Mon May 28 03:34:15 2001
@@ -1,5 +1,5 @@
/* Generic conversion to and from 8bit charsets.
- Copyright (C) 1997, 1998, 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1997, 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
@@ -67,6 +67,8 @@
if (__builtin_expect (ch >= sizeof (from_ucs4) / sizeof (from_ucs4[0]), 0)\
|| (__builtin_expect (from_ucs4[ch], '\1') == '\0' && ch != 0)) \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ \
/* This is an illegal character. */ \
STANDARD_ERR_HANDLER (4); \
} \
--- glibc-20010430/iconvdata/ansi_x3.110.c.bak Mon Dec 4 19:53:41 2000
+++ glibc-20010430/iconvdata/ansi_x3.110.c Mon May 28 03:34:56 2001
@@ -1,5 +1,5 @@
/* Generic conversion to and from ANSI_X3.110-1983.
- Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
@@ -549,6 +549,8 @@
cp = "\xd5"; \
else \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ \
/* Illegal characters. */ \
STANDARD_ERR_HANDLER (4); \
} \
--- glibc-20010430/iconvdata/big5.c.bak Mon Dec 4 19:53:42 2000
+++ glibc-20010430/iconvdata/big5.c Mon May 28 03:35:19 2001
@@ -1,5 +1,5 @@
/* Mapping tables for Big5 handling.
- Copyright (C) 1997, 1998, 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1997, 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
@@ -8564,6 +8564,7 @@
cp = from_ucs4_tab15[ch - 0xff01]; \
break; \
default: \
+ UNICODE_TAG_HANDLER (ch, 4); \
/* Illegal character. */ \
cp = ""; \
break; \
--- glibc-20010430/iconvdata/big5hkscs.c.bak Mon Dec 4 19:53:42 2000
+++ glibc-20010430/iconvdata/big5hkscs.c Mon May 28 03:35:42 2001
@@ -1,5 +1,5 @@
/* Mapping tables for Big5-HKSCS handling.
- Copyright (C) 1997, 1998, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1997, 1998, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
Modified for Big5-HKSCS by Roger So <spacehunt@e-fever.org>, 2000.
@@ -11772,6 +11772,8 @@
\
if (cp[0] == '\0' && ch != 0) \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ \
/* Illegal character. */ \
STANDARD_ERR_HANDLER (4); \
} \
--- glibc-20010430/iconvdata/cp1255.c.bak Sun May 20 17:44:32 2001
+++ glibc-20010430/iconvdata/cp1255.c Mon May 28 03:36:41 2001
@@ -451,7 +451,10 @@
else if (ch == 0x2122) \
res = 0x99; \
else \
- res = 0; \
+ { \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ res = 0; \
+ } \
\
if (__builtin_expect (res != 0, 1)) \
{ \
--- glibc-20010430/iconvdata/cp1258.c.bak Sun May 20 03:53:53 2001
+++ glibc-20010430/iconvdata/cp1258.c Mon May 28 03:37:17 2001
@@ -776,7 +776,10 @@
else if (ch == 0x2122) \
res = 0x99; \
else \
- res = 0; \
+ { \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ res = 0; \
+ } \
\
if (__builtin_expect (res != 0, 1)) \
{ \
--- glibc-20010430/iconvdata/euc-cn.c.bak Mon Jul 3 16:39:27 2000
+++ glibc-20010430/iconvdata/euc-cn.c Mon May 28 03:37:49 2001
@@ -1,5 +1,5 @@
/* Mapping tables for EUC-CN handling.
- Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
@@ -140,6 +140,8 @@
{ \
if (__builtin_expect (found, 0) == __UNKNOWN_10646_CHAR) \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ \
/* Illegal character. */ \
STANDARD_ERR_HANDLER (4); \
} \
--- glibc-20010430/iconvdata/euc-jp.c.bak Wed Jul 12 18:11:43 2000
+++ glibc-20010430/iconvdata/euc-jp.c Mon May 28 03:38:19 2001
@@ -1,5 +1,5 @@
/* Mapping tables for EUC-JP handling.
- Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
@@ -227,6 +227,8 @@
} \
else \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ \
/* Illegal character. */ \
STANDARD_ERR_HANDLER (4); \
} \
--- glibc-20010430/iconvdata/euc-kr.c.bak Wed Nov 8 22:01:58 2000
+++ glibc-20010430/iconvdata/euc-kr.c Mon May 28 03:38:39 2001
@@ -1,5 +1,5 @@
/* Mapping tables for EUC-KR handling.
- Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Jungshik Shin <jshin@pantheon.yale.edu>
and Ulrich Drepper <drepper@cygnus.com>, 1998.
@@ -142,10 +142,12 @@
\
/* Decomposing Hangul syllables not available in KS C 5601 into \
Jamos should be considered either here or in euckr_from_ucs4() */ \
- euckr_from_ucs4 (ch, cp) ; \
+ euckr_from_ucs4 (ch, cp); \
\
if (__builtin_expect (cp[0], '\1') == '\0' && ch != 0) \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ \
/* Illegal character. */ \
STANDARD_ERR_HANDLER (4); \
} \
--- glibc-20010430/iconvdata/euc-tw.c.bak Tue Sep 5 15:24:48 2000
+++ glibc-20010430/iconvdata/euc-tw.c Mon May 28 03:38:54 2001
@@ -1,5 +1,5 @@
/* Mapping tables for EUC-TW handling.
- Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
@@ -61,14 +61,14 @@
} \
else \
{ \
- /* Two or more byte character. First test whether the next \
- character is also available. */ \
+ /* Two or more byte character. First test whether the next byte \
+ is also available. */ \
uint32_t ch2; \
\
if (inptr + 1 >= inend) \
{ \
- /* The second character is not available. Store the \
- intermediate result. */ \
+ /* The second byte is not available. Store the intermediate \
+ result. */ \
result = __GCONV_INCOMPLETE_INPUT; \
break; \
} \
@@ -99,7 +99,7 @@
\
if (ch == 0) \
{ \
- /* The third or fourth character is not available. Store \
+ /* The third or fourth byte is not available. Store \
the intermediate result. */ \
result = __GCONV_INCOMPLETE_INPUT; \
break; \
@@ -170,7 +170,7 @@
*outptr++ = ch; \
else \
{ \
- /* Try the JIS character sets. */ \
+ /* Try the CNS 11643 planes. */ \
size_t found; \
\
found = ucs4_to_cns11643l1 (ch, outptr, outend - outptr); \
@@ -199,6 +199,8 @@
} \
if (__builtin_expect (found, 0) == __UNKNOWN_10646_CHAR) \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ \
/* Illegal character. */ \
STANDARD_ERR_HANDLER (4); \
} \
--- glibc-20010430/iconvdata/gbk.c.bak Tue Sep 26 14:39:33 2000
+++ glibc-20010430/iconvdata/gbk.c Mon May 28 03:39:18 2001
@@ -1,5 +1,5 @@
/* Mapping tables for GBK handling.
- Copyright (C) 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Sean Chen <sean.chen@turbolinux.com>, 1999.
@@ -13471,6 +13471,7 @@
cp = "\xa1\xe9\0\0\xa1\xea\0\0\xa9\x56\0\0\xa3\xfe\0\0\xa9\x57\0\0\xa3\xa4" + ((ch - 0xffe0) * 4); \
break; \
default: \
+ UNICODE_TAG_HANDLER (ch, 4); \
cp = ""; \
break; \
} \
--- glibc-20010430/iconvdata/ibm930.c.bak Mon May 14 13:05:08 2001
+++ glibc-20010430/iconvdata/ibm930.c Mon May 28 03:39:34 2001
@@ -203,6 +203,8 @@
\
if (__builtin_expect (ch, 0) >= 0xffff) \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ \
if (! ignore_errors_p ()) \
{ \
result = __GCONV_ILLEGAL_INPUT; \
--- glibc-20010430/iconvdata/ibm932.c.bak Sat Jan 20 22:16:44 2001
+++ glibc-20010430/iconvdata/ibm932.c Mon May 28 03:39:51 2001
@@ -1,5 +1,5 @@
/* Conversion from and to IBM932.
- Copyright (C) 2000 Free Software Foundation, Inc.
+ Copyright (C) 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Masahide Washizawa <washi@jp.ibm.com>, 2000.
@@ -148,7 +148,7 @@
{ \
const struct gap *rp = __ucs4_to_ibm932sb_idx; \
unsigned char sc; \
- uint32_t ch = get32(inptr); \
+ uint32_t ch = get32 (inptr); \
uint16_t found = TRUE; \
uint32_t i; \
uint32_t low; \
@@ -156,7 +156,10 @@
uint16_t pccode; \
\
if (__builtin_expect (ch, 0) >= 0xffff) \
+ { \
+ UNICODE_TAG_HANDLER (ch, 4); \
rp = NULL; \
+ } \
else \
while (ch > rp->end) \
++rp; \
@@ -188,26 +191,26 @@
break; \
} \
} \
- if (found) \
- { \
- if (__builtin_expect (outptr + 2 > outend, 0)) \
- { \
- result = __GCONV_FULL_OUTPUT; \
- break; \
- } \
- *outptr++ = pccode >> 8 & 0xff; \
- *outptr++ = pccode & 0xff; \
- } \
- else \
- { \
- /* This is an illegal character. */ \
- if (! ignore_errors_p ()) \
- { \
- result = __GCONV_ILLEGAL_INPUT; \
- break; \
- } \
- ++*irreversible; \
- } \
+ if (found) \
+ { \
+ if (__builtin_expect (outptr + 2 > outend, 0)) \
+ { \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
+ *outptr++ = pccode >> 8 & 0xff; \
+ *outptr++ = pccode & 0xff; \
+ } \
+ else \
+ { \
+ /* This is an illegal character. */ \
+ if (! ignore_errors_p ()) \
+ { \
+ result = __GCONV_ILLEGAL_INPUT; \
+ break; \
+ } \
+ ++*irreversible; \
+ } \
} \
else \
{ \
@@ -224,8 +227,8 @@
*outptr++ = sc; \
} \
\
- /* Now that we wrote the output increment the input pointer. */ \
- inptr += 4; \
+ /* Now that we wrote the output increment the input pointer. */ \
+ inptr += 4; \
}
#define LOOP_NEED_FLAGS
#include <iconv/loop.c>
--- glibc-20010430/iconvdata/ibm933.c.bak Mon May 14 13:05:35 2001
+++ glibc-20010430/iconvdata/ibm933.c Mon May 28 03:40:20 2001
@@ -205,6 +205,8 @@
\
if (__builtin_expect (ch, 0) >= 0xffff) \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ \
if (! ignore_errors_p ()) \
{ \
result = __GCONV_ILLEGAL_INPUT; \
--- glibc-20010430/iconvdata/ibm935.c.bak Mon May 14 13:05:59 2001
+++ glibc-20010430/iconvdata/ibm935.c Mon May 28 03:41:02 2001
@@ -203,6 +203,8 @@
\
if (__builtin_expect (ch, 0) >= 0xffff) \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ \
if (! ignore_errors_p ()) \
{ \
result = __GCONV_ILLEGAL_INPUT; \
--- glibc-20010430/iconvdata/ibm937.c.bak Mon May 14 13:06:23 2001
+++ glibc-20010430/iconvdata/ibm937.c Mon May 28 03:41:21 2001
@@ -195,6 +195,8 @@
/ sizeof (__ucs4_to_ibm937db[0])), 0) \
|| __builtin_expect (cp[0], '\1') == '\0') \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ \
/* This is an illegal character. */ \
if (! ignore_errors_p ()) \
{ \
--- glibc-20010430/iconvdata/ibm939.c.bak Mon May 14 13:06:43 2001
+++ glibc-20010430/iconvdata/ibm939.c Mon May 28 03:41:40 2001
@@ -203,6 +203,7 @@
\
if (__builtin_expect (ch, 0) >= 0xffff) \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
rp1 = NULL; \
rp2 = NULL; \
} \
--- glibc-20010430/iconvdata/ibm943.c.bak Sat Jan 20 22:16:54 2001
+++ glibc-20010430/iconvdata/ibm943.c Mon May 28 03:41:56 2001
@@ -1,5 +1,5 @@
/* Conversion from and to IBM943.
- Copyright (C) 2000 Free Software Foundation, Inc.
+ Copyright (C) 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Masahide Washizawa <washi@jp.ibm.com>, 2000.
@@ -156,7 +156,10 @@
uint16_t pccode; \
\
if (__builtin_expect (ch, 0) >= 0xffff) \
+ { \
+ UNICODE_TAG_HANDLER (ch, 4); \
rp = NULL; \
+ } \
else \
while (ch > rp->end) \
++rp; \
@@ -188,26 +191,26 @@
break; \
} \
} \
- if (found) \
- { \
- if (__builtin_expect (outptr + 2 > outend, 0)) \
- { \
- result = __GCONV_FULL_OUTPUT; \
- break; \
- } \
- *outptr++ = pccode >> 8 & 0xff; \
- *outptr++ = pccode & 0xff; \
- } \
- else \
- { \
- /* This is an illegal character. */ \
- if (! ignore_errors_p ()) \
- { \
- result = __GCONV_ILLEGAL_INPUT; \
- break; \
- } \
- ++*irreversible; \
- } \
+ if (found) \
+ { \
+ if (__builtin_expect (outptr + 2 > outend, 0)) \
+ { \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
+ *outptr++ = pccode >> 8 & 0xff; \
+ *outptr++ = pccode & 0xff; \
+ } \
+ else \
+ { \
+ /* This is an illegal character. */ \
+ if (! ignore_errors_p ()) \
+ { \
+ result = __GCONV_ILLEGAL_INPUT; \
+ break; \
+ } \
+ ++*irreversible; \
+ } \
} \
else \
{ \
@@ -224,8 +227,8 @@
*outptr++ = sc; \
} \
\
- /* Now that we wrote the output increment the input pointer. */ \
- inptr += 4; \
+ /* Now that we wrote the output increment the input pointer. */ \
+ inptr += 4; \
}
#define LOOP_NEED_FLAGS
#include <iconv/loop.c>
--- glibc-20010430/iconvdata/iso646.c.bak Mon Dec 4 19:53:43 2000
+++ glibc-20010430/iconvdata/iso646.c Mon May 28 03:42:37 2001
@@ -1,5 +1,5 @@
/* Conversion to and from the various ISO 646 CCS.
- Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
@@ -879,7 +879,10 @@
break; \
default: \
if (__builtin_expect (ch, 0) > 0x7f) \
- failure = __GCONV_ILLEGAL_INPUT; \
+ { \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ failure = __GCONV_ILLEGAL_INPUT; \
+ } \
break; \
} \
\
--- glibc-20010430/iconvdata/iso8859-1.c.bak Mon Nov 27 23:28:38 2000
+++ glibc-20010430/iconvdata/iso8859-1.c Mon May 28 03:43:14 2001
@@ -1,5 +1,5 @@
/* Conversion to and from ISO 8859-1.
- Copyright (C) 1997, 1998, 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1997, 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
@@ -48,6 +48,8 @@
uint32_t ch = *((const uint32_t *) inptr); \
if (__builtin_expect (ch, 0) > 0xff) \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ \
/* We have an illegal character. */ \
STANDARD_ERR_HANDLER (4); \
} \
--- glibc-20010430/iconvdata/iso_6937.c.bak Mon Dec 4 19:53:44 2000
+++ glibc-20010430/iconvdata/iso_6937.c Mon May 28 03:43:01 2001
@@ -1,5 +1,5 @@
/* Generic conversion to and from ISO 6937.
- Copyright (C) 1997, 1998, 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1997, 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
@@ -536,6 +536,7 @@
cp = "\xd5"; \
break; \
default: \
+ UNICODE_TAG_HANDLER (ch, 4); \
cp = NULL; \
fail = 1; \
} \
--- glibc-20010430/iconvdata/iso_6937-2.c.bak Mon Dec 4 19:53:44 2000
+++ glibc-20010430/iconvdata/iso_6937-2.c Mon May 28 03:42:52 2001
@@ -1,5 +1,5 @@
/* Generic conversion to and from ISO 6937-2.
- Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
@@ -563,6 +563,7 @@
cp = "\xd5"; \
break; \
default: \
+ UNICODE_TAG_HANDLER (ch, 4); \
/* Illegal characters. */ \
cp = NULL; \
break; \
--- glibc-20010430/iconvdata/iso-2022-cn.c.bak Mon May 14 13:07:10 2001
+++ glibc-20010430/iconvdata/iso-2022-cn.c Mon Jun 4 03:50:46 2001
@@ -1,5 +1,5 @@
/* Conversion module for ISO-2022-CN.
- Copyright (C) 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1999.
@@ -320,6 +320,8 @@
used = GB2312_set + CNS11643_1_set - used; \
else \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ \
/* Even this does not work. Error. */ \
STANDARD_ERR_HANDLER (4); \
} \
--- glibc-20010430/iconvdata/iso-2022-cn-ext.c.bak Mon Jun 4 16:31:00 2001
+++ glibc-20010430/iconvdata/iso-2022-cn-ext.c Mon Jun 4 03:52:41 2001
@@ -560,6 +560,7 @@
} \
if (used == ASCII_set) \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
STANDARD_ERR_HANDLER (4); \
} \
} \
--- glibc-20010430/iconvdata/iso-2022-kr.c.bak Mon May 14 13:08:44 2001
+++ glibc-20010430/iconvdata/iso-2022-kr.c Mon May 28 03:42:11 2001
@@ -1,5 +1,5 @@
/* Conversion module for ISO-2022-KR.
- Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
@@ -221,7 +221,6 @@
#define BODY \
{ \
uint32_t ch; \
- size_t written = 0; \
\
ch = get32 (inptr); \
\
@@ -241,16 +240,18 @@
} \
\
*outptr++ = ch; \
- written = 1; \
} \
else \
{ \
char buf[2]; \
+ size_t written; \
\
written = ucs4_to_ksc5601 (ch, buf, 2); \
\
if (__builtin_expect (written, 0) == __UNKNOWN_10646_CHAR) \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ \
/* Illegal character. */ \
STANDARD_ERR_HANDLER (4); \
} \
--- glibc-20010430/iconvdata/johab.c.bak Tue Sep 26 14:39:33 2000
+++ glibc-20010430/iconvdata/johab.c Mon May 28 03:43:33 2001
@@ -1,5 +1,5 @@
/* Mapping tables for JOHAB handling.
- Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Jungshik Shin <jshin@pantheon.yale.edu>
and Ulrich Drepper <drepper@cygnus.com>, 1998.
@@ -426,6 +426,7 @@
if (__builtin_expect (written, 1) == __UNKNOWN_10646_CHAR \
|| (outptr[0] == 0x22 && outptr[1] > 0x65)) \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
STANDARD_ERR_HANDLER (4); \
} \
\
--- glibc-20010430/iconvdata/sjis.c.bak Mon Dec 4 19:53:44 2000
+++ glibc-20010430/iconvdata/sjis.c Mon May 28 03:44:30 2001
@@ -1,5 +1,5 @@
/* Mapping tables for SJIS handling.
- Copyright (C) 1997, 1998, 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1997, 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
@@ -4471,8 +4471,11 @@
&& __builtin_expect (ch, 0xff01) <= 0xffef) \
cp = from_ucs4_extra[ch - 0xff00]; \
else \
- /* Illegal character. */ \
- cp = ""; \
+ { \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ /* Illegal character. */ \
+ cp = ""; \
+ } \
} \
else \
cp = from_ucs4_lat1[ch]; \
--- glibc-20010430/iconvdata/t.61.c.bak Mon Dec 4 19:53:44 2000
+++ glibc-20010430/iconvdata/t.61.c Mon May 28 03:44:44 2001
@@ -1,5 +1,5 @@
/* Generic conversion to and from T.61.
- Copyright (C) 1997, 1998, 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1997, 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1997.
@@ -468,6 +468,8 @@
|| __builtin_expect (ch, 0x2d8) > 0x2dd \
|| __builtin_expect (ch, 0x2d8) == 0x02dc) \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
+ \
/* Illegal characters. */ \
STANDARD_ERR_HANDLER (4); \
} \
--- glibc-20010430/iconvdata/uhc.c.bak Tue Aug 1 13:52:59 2000
+++ glibc-20010430/iconvdata/uhc.c Mon May 28 03:44:59 2001
@@ -1,5 +1,5 @@
/* Mapping tables for UHC handling.
- Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Jungshik Shin <jshin@pantheon.yale.edu>, 1998.
@@ -3243,6 +3243,7 @@
} \
if (__builtin_expect (written, 0) == __UNKNOWN_10646_CHAR) \
{ \
+ UNICODE_TAG_HANDLER (ch, 4); \
STANDARD_ERR_HANDLER (4); \
} \
\
--- glibc-20010430/iconvdata/unicode.c.bak Tue Apr 17 19:07:33 2001
+++ glibc-20010430/iconvdata/unicode.c Mon May 28 03:45:19 2001
@@ -1,5 +1,5 @@
/* Conversion module for Unicode
- Copyright (C) 1999, 2000 Free Software Foundation, Inc.
+ Copyright (C) 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1999.
@@ -154,6 +154,7 @@
\
if (__builtin_expect (c, 0) >= 0x10000) \
{ \
+ UNICODE_TAG_HANDLER (c, 4); \
STANDARD_ERR_HANDLER (4); \
} \
else if (__builtin_expect (c >= 0xd800 && c < 0xe000, 0)) \
--- glibc-20010430/iconvdata/iso-2022-jp.c.bak Mon May 14 13:59:53 2001
+++ glibc-20010430/iconvdata/iso-2022-jp.c Mon Jun 4 15:53:21 2001
@@ -1,5 +1,5 @@
-/* Conversion module for ISO-2022-JP.
- Copyright (C) 1998, 1999, 2000 Free Software Foundation, Inc.
+/* Conversion module for ISO-2022-JP and ISO-2022-JP-2.
+ Copyright (C) 1998, 1999, 2000, 2001 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@cygnus.com>, 1998.
@@ -90,14 +90,14 @@
enum
{
ASCII_set = 0,
- JISX0208_1978_set = 8,
- JISX0208_1983_set = 16,
- JISX0201_Roman_set = 24,
- JISX0201_Kana_set = 32,
- GB2312_set = 40,
- KSC5601_set = 48,
- JISX0212_set = 56,
- CURRENT_SEL_MASK = 56
+ JISX0208_1978_set = 1 << 3,
+ JISX0208_1983_set = 2 << 3,
+ JISX0201_Roman_set = 3 << 3,
+ JISX0201_Kana_set = 4 << 3,
+ GB2312_set = 5 << 3,
+ KSC5601_set = 6 << 3,
+ JISX0212_set = 7 << 3,
+ CURRENT_SEL_MASK = 7 << 3
};
/* The second value stored is the designation of the G2 set. The following
@@ -105,9 +105,25 @@
enum
{
UNSPECIFIED_set = 0,
- ISO88591_set = 64,
- ISO88597_set = 128,
- CURRENT_ASSIGN_MASK = 192
+ ISO88591_set = 1 << 6,
+ ISO88597_set = 2 << 6,
+ CURRENT_ASSIGN_MASK = 3 << 6
+};
+
+/* The third value, only used during conversion from Unicode to ISO-2022-JP-2,
+ describes the language tag parsing status. The possible values are as
+ follows. Values >= TAG_language are temporary tag parsing states. */
+enum
+{
+ TAG_none = 0,
+ TAG_language = 4 << 8,
+ TAG_language_j = 5 << 8,
+ TAG_language_ja = 1 << 8,
+ TAG_language_k = 6 << 8,
+ TAG_language_ko = 2 << 8,
+ TAG_language_z = 7 << 8,
+ TAG_language_zh = 3 << 8,
+ CURRENT_TAG_MASK = 7 << 8
};
@@ -198,7 +214,8 @@
\
if ((data->__statep->__count & ~7) != ASCII_set) \
{ \
- if (dir == from_iso2022jp) \
+ if (dir == from_iso2022jp \
+ || (data->__statep->__count & CURRENT_SEL_MASK) == ASCII_set) \
{ \
/* It's easy, we don't have to emit anything, we just reset the \
state for the input. Note that this also clears the G2 \
@@ -503,6 +520,23 @@
/* Next, define the other direction. */
+
+enum conversion { none = 0, european, japanese, chinese, korean, other };
+
+/* A datatype for conversion lists. */
+typedef unsigned int cvlist_t;
+#define CVLIST(cv1, cv2, cv3, cv4, cv5) \
+ ((cv1) + ((cv2) << 3) + ((cv3) << 6) + ((cv4) << 9) + ((cv5) << 12))
+#define CVLIST_FIRST(cvl) ((cvl) & ((1 << 3) - 1))
+#define CVLIST_REST(cvl) ((cvl) >> 3)
+static const cvlist_t conversion_lists[4] =
+ {
+ /* TAG_none */ CVLIST (japanese, european, chinese, korean, other),
+ /* TAG_language_ja */ CVLIST (japanese, european, chinese, korean, other),
+ /* TAG_language_ko */ CVLIST (korean, european, japanese, chinese, other),
+ /* TAG_language_zh */ CVLIST (chinese, european, japanese, korean, other)
+ };
+
#define MIN_NEEDED_INPUT MIN_NEEDED_TO
#define MIN_NEEDED_OUTPUT MIN_NEEDED_FROM
#define MAX_NEEDED_OUTPUT (MAX_NEEDED_FROM + 2)
@@ -510,12 +544,56 @@
#define BODY \
{ \
uint32_t ch; \
- size_t written = 0; \
+ size_t written; \
\
ch = get32 (inptr); \
\
+ if (var == iso2022jp2) \
+ { \
+ /* Handle Unicode tag characters (range U+E0000..U+E007F). */ \
+ if (__builtin_expect ((ch >> 7) == (0xe0000 >> 7), 0)) \
+ { \
+ ch &= 0x7f; \
+ if (ch >= 'A' && ch <= 'Z') \
+ ch += 'a' - 'A'; \
+ if (ch == 0x01) \
+ tag = TAG_language; \
+ else if (ch == 'j' && tag == TAG_language) \
+ tag = TAG_language_j; \
+ else if (ch == 'a' && tag == TAG_language_j) \
+ tag = TAG_language_ja; \
+ else if (ch == 'k' && tag == TAG_language) \
+ tag = TAG_language_k; \
+ else if (ch == 'o' && tag == TAG_language_k) \
+ tag = TAG_language_ko; \
+ else if (ch == 'z' && tag == TAG_language) \
+ tag = TAG_language_z; \
+ else if (ch == 'h' && tag == TAG_language_z) \
+ tag = TAG_language_zh; \
+ else if (ch == 0x7f) \
+ tag = TAG_none; \
+ else \
+ { \
+ /* Other tag characters reset the tag parsing state (if the \
+ current state is a temporary state) or are ignored (if \
+ the current state is a stable one). */ \
+ if (tag >= TAG_language) \
+ tag = TAG_none; \
+ } \
+ \
+ inptr += 4; \
+ continue; \
+ } \
+ \
+ /* Non-tag characters reset the tag parsing state, if the current \
+ state is a temporary state. */ \
+ if (__builtin_expect (tag >= TAG_language, 0)) \
+ tag = TAG_none; \
+ } \
+ \
/* First see whether we can write the character using the currently \
- selected character set. */ \
+ selected character set. But ignore the selected character set if \
+ the current language tag shows different preferences. */ \
if (set == ASCII_set) \
{ \
/* Please note that the NUL byte is *not* matched if we are not \
@@ -525,58 +603,75 @@
{ \
*outptr++ = ch; \
written = 1; \
+ \
+ /* At the beginning of a line, G2 designation is cleared. */ \
+ if (var == iso2022jp2 && ch == 0x0a) \
+ set2 = UNSPECIFIED_set; \
} \
- /* At the beginning of a line, G2 designation is cleared. */ \
- if (var == iso2022jp2 && ch == 0x0a) \
- set2 = UNSPECIFIED_set; \
+ else \
+ written = __UNKNOWN_10646_CHAR; \
} \
/* ISO-2022-JP recommends to encode the newline character always in \
ASCII since this allows a context-free interpretation of the \
characters at the beginning of the next line. Otherwise it would \
have to be known whether the last line ended using ASCII or \
JIS X 0201. */ \
- else if (set == JISX0201_Roman_set) \
+ else if (set == JISX0201_Roman_set \
+ && (__builtin_expect (tag == TAG_none, 1) \
+ || tag == TAG_language_ja)) \
{ \
- unsigned char buf[2]; \
+ unsigned char buf[1]; \
written = ucs4_to_jisx0201 (ch, buf); \
- if (written != __UNKNOWN_10646_CHAR && buf[0] > 0x20 \
- && buf[0] < 0x80) \
+ if (written != __UNKNOWN_10646_CHAR) \
{ \
- *outptr++ = buf[0]; \
- written = 1; \
+ if (buf[0] > 0x20 && buf[0] < 0x80) \
+ { \
+ *outptr++ = buf[0]; \
+ written = 1; \
+ } \
+ else \
+ written = __UNKNOWN_10646_CHAR; \
} \
- else \
- written = __UNKNOWN_10646_CHAR; \
} \
- else if (set == JISX0201_Kana_set) \
+ else if (set == JISX0201_Kana_set \
+ && (__builtin_expect (tag == TAG_none, 1) \
+ || tag == TAG_language_ja)) \
{ \
- unsigned char buf[2]; \
+ unsigned char buf[1]; \
written = ucs4_to_jisx0201 (ch, buf); \
- if (written != __UNKNOWN_10646_CHAR && buf[0] > 0xa0 \
- && buf[0] < 0xe0) \
+ if (written != __UNKNOWN_10646_CHAR) \
{ \
- *outptr++ = buf[0] - 0x80; \
- written = 1; \
+ if (buf[0] > 0xa0 && buf[0] < 0xe0) \
+ { \
+ *outptr++ = buf[0] - 0x80; \
+ written = 1; \
+ } \
+ else \
+ written = __UNKNOWN_10646_CHAR; \
} \
- else \
- written = __UNKNOWN_10646_CHAR; \
} \
else \
{ \
- if (set == JISX0208_1978_set || set == JISX0208_1983_set) \
+ if ((set == JISX0208_1978_set || set == JISX0208_1983_set) \
+ && (__builtin_expect (tag == TAG_none, 1) \
+ || tag == TAG_language_ja)) \
written = ucs4_to_jisx0208 (ch, outptr, outend - outptr); \
- else if (set == JISX0212_set) \
+ else if (set == JISX0212_set \
+ && (__builtin_expect (tag == TAG_none, 1) \
+ || tag == TAG_language_ja)) \
written = ucs4_to_jisx0212 (ch, outptr, outend - outptr); \
- else if (set == GB2312_set) \
+ else if (set == GB2312_set \
+ && (__builtin_expect (tag == TAG_none, 1) \
+ || tag == TAG_language_zh)) \
written = ucs4_to_gb2312 (ch, outptr, outend - outptr); \
+ else if (set == KSC5601_set \
+ && (__builtin_expect (tag == TAG_none, 1) \
+ || tag == TAG_language_ko)) \
+ written = ucs4_to_ksc5601 (ch, outptr, outend - outptr); \
else \
- { \
- assert (set == KSC5601_set); \
- \
- written = ucs4_to_ksc5601 (ch, outptr, outend - outptr); \
- } \
+ written = __UNKNOWN_10646_CHAR; \
\
- if (__builtin_expect (written, 1) == 0) \
+ if (__builtin_expect (written == 0, 0)) \
{ \
result = __GCONV_FULL_OUTPUT; \
break; \
@@ -585,18 +680,19 @@
outptr += written; \
} \
\
- if (written == __UNKNOWN_10646_CHAR || written == 0) \
+ if (written == __UNKNOWN_10646_CHAR \
+ && __builtin_expect (tag == TAG_none, 1)) \
{ \
if (set2 == ISO88591_set) \
{ \
- if (__builtin_expect (outptr + 3 > outend, 0)) \
+ if (ch >= 0x80 && ch <= 0xff) \
{ \
- result = __GCONV_FULL_OUTPUT; \
- break; \
- } \
+ if (__builtin_expect (outptr + 3 > outend, 0)) \
+ { \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
\
- if (ch >= 0x80 && ch <= 0xff) \
- { \
*outptr++ = ESC; \
*outptr++ = 'N'; \
*outptr++ = ch & 0x7f; \
@@ -629,18 +725,15 @@
} \
} \
\
- if (written == __UNKNOWN_10646_CHAR || written == 0) \
+ if (written == __UNKNOWN_10646_CHAR) \
{ \
- /* Either this is an unknown character or we have to switch \
- the currently selected character set. The character sets \
- do not code entirely separate parts of ISO 10646 and \
- therefore there is no single correct result. If we choose \
- the character set to use wrong we might be end up with \
- using yet another character set for the next character \
- though the current and the next could be encoded with one \
- character set. We leave this kind of optimization for \
- later and now simply use a fixed order in which we test for \
- availability */ \
+ /* The attempts to use the currently selected character set \
+ failed, either because the language tag changed, or because \
+ the character requires a different character set, or because \
+ the character is unknown. \
+ The CJK character sets partially overlap when seen as subsets \
+ of ISO 10646; therefore there is no single correct result. \
+ We use a preferrence order which depends on the language tag. */ \
\
if (ch <= 0x7f) \
{ \
@@ -671,128 +764,76 @@
else \
{ \
/* Now it becomes difficult. We must search the other \
- character sets one by one and we cannot use simple \
- arithmetic to determine whether the character can be \
- encoded using this set. */ \
- size_t written; \
+ character sets one by one. Use an ordered conversion \
+ list that depends on the current language tag. */ \
+ cvlist_t conversion_list; \
unsigned char buf[2]; \
\
- written = ucs4_to_jisx0201 (ch, buf); \
- if (written != __UNKNOWN_10646_CHAR && buf[0] < 0x80) \
- { \
- /* We use JIS X 0201. */ \
- if (__builtin_expect (outptr + 3 > outend, 0)) \
- { \
- result = __GCONV_FULL_OUTPUT; \
- break; \
- } \
- \
- *outptr++ = ESC; \
- *outptr++ = '('; \
- *outptr++ = 'J'; \
- set = JISX0201_Roman_set; \
+ result = __GCONV_ILLEGAL_INPUT; \
\
- if (__builtin_expect (outptr + 1 > outend, 0)) \
- { \
- result = __GCONV_FULL_OUTPUT; \
- break; \
- } \
- *outptr++ = buf[0]; \
- } \
+ if (var == iso2022jp2) \
+ conversion_list = conversion_lists[tag >> 8]; \
else \
- { \
- written = ucs4_to_jisx0208 (ch, buf, 2); \
- if (written != __UNKNOWN_10646_CHAR) \
- { \
- /* We use JIS X 0208. */ \
- if (__builtin_expect (outptr + 3 > outend, 0)) \
- { \
- result = __GCONV_FULL_OUTPUT; \
- break; \
- } \
+ conversion_list = CVLIST (japanese, 0, 0, 0, 0); \
\
- *outptr++ = ESC; \
- *outptr++ = '$'; \
- *outptr++ = 'B'; \
- set = JISX0208_1983_set; \
+ do \
+ switch (CVLIST_FIRST (conversion_list)) \
+ { \
+ case european: \
+ \
+ /* Try ISO 8859-1 upper half. */ \
+ if (ch >= 0x80 && ch <= 0xff) \
+ { \
+ if (set2 != ISO88591_set) \
+ { \
+ if (__builtin_expect (outptr + 3 > outend, 0)) \
+ { \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
+ *outptr++ = ESC; \
+ *outptr++ = '.'; \
+ *outptr++ = 'A'; \
+ set2 = ISO88591_set; \
+ } \
+ \
+ if (__builtin_expect (outptr + 3 > outend, 0)) \
+ { \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
+ *outptr++ = ESC; \
+ *outptr++ = 'N'; \
+ *outptr++ = ch - 0x80; \
+ result = __GCONV_OK; \
+ break; \
+ } \
\
- if (__builtin_expect (outptr + 2 > outend, 0)) \
- { \
- result = __GCONV_FULL_OUTPUT; \
- break; \
- } \
- *outptr++ = buf[0]; \
- *outptr++ = buf[1]; \
- } \
- else if (__builtin_expect (var, iso2022jp2) == iso2022jp) \
+ /* Try ISO 8859-7 upper half. */ \
{ \
- /* We have no other choice. */ \
- STANDARD_ERR_HANDLER (4); \
- } \
- else \
- { \
- written = ucs4_to_jisx0212 (ch, buf, 2); \
- if (written != __UNKNOWN_10646_CHAR) \
- { \
- /* We use JIS X 0212. */ \
- if (__builtin_expect (outptr + 4 > outend, 0)) \
- { \
- result = __GCONV_FULL_OUTPUT; \
- break; \
- } \
- *outptr++ = ESC; \
- *outptr++ = '$'; \
- *outptr++ = '('; \
- *outptr++ = 'D'; \
- set = JISX0212_set; \
+ const struct gap *rp = from_idx; \
\
- if (__builtin_expect (outptr + 2 > outend, 0)) \
- { \
- result = __GCONV_FULL_OUTPUT; \
- break; \
- } \
- *outptr++ = buf[0]; \
- *outptr++ = buf[1]; \
- } \
- else \
+ while (ch > rp->end) \
+ ++rp; \
+ if (ch >= rp->start) \
{ \
- written = ucs4_to_jisx0201 (ch, buf); \
- if (written != __UNKNOWN_10646_CHAR \
- && buf[0] >= 0x80) \
+ unsigned char res = \
+ iso88597_from_ucs4[ch - 0xa0 + rp->idx]; \
+ if (res != '\0') \
{ \
- /* We use JIS X 0201. */ \
- if (__builtin_expect (outptr + 3 > outend, 0)) \
- { \
- result = __GCONV_FULL_OUTPUT; \
- break; \
- } \
- \
- *outptr++ = ESC; \
- *outptr++ = '('; \
- *outptr++ = 'I'; \
- set = JISX0201_Kana_set; \
- \
- if (__builtin_expect (outptr + 1 > outend, 0)) \
+ if (set2 != ISO88597_set) \
{ \
- result = __GCONV_FULL_OUTPUT; \
- break; \
- } \
- *outptr++ = buf[0] - 0x80; \
- } \
- else if (ch != 0xa5 && ch >= 0x80 && ch <= 0xff) \
- { \
- /* ISO 8859-1 upper half. */ \
- if (__builtin_expect (outptr + 3 > outend, 0)) \
- { \
- result = __GCONV_FULL_OUTPUT; \
- break; \
+ if (__builtin_expect (outptr + 3 > outend, 0))\
+ { \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
+ *outptr++ = ESC; \
+ *outptr++ = '.'; \
+ *outptr++ = 'F'; \
+ set2 = ISO88597_set; \
} \
\
- *outptr++ = ESC; \
- *outptr++ = '.'; \
- *outptr++ = 'A'; \
- set2 = ISO88591_set; \
- \
if (__builtin_expect (outptr + 3 > outend, 0)) \
{ \
result = __GCONV_FULL_OUTPUT; \
@@ -800,106 +841,221 @@
} \
*outptr++ = ESC; \
*outptr++ = 'N'; \
- *outptr++ = ch - 0x80; \
+ *outptr++ = res; \
+ result = __GCONV_OK; \
+ break; \
} \
- else \
- { \
- written = ucs4_to_gb2312 (ch, buf, 2); \
- if (written != __UNKNOWN_10646_CHAR) \
- { \
- /* We use GB 2312. */ \
- if (__builtin_expect (outptr + 3 > outend, 0))\
- { \
- result = __GCONV_FULL_OUTPUT; \
- break; \
- } \
+ } \
+ } \
\
- *outptr++ = ESC; \
- *outptr++ = '$'; \
- *outptr++ = 'A'; \
- set = GB2312_set; \
+ break; \
\
- if (__builtin_expect (outptr + 2 > outend, 0))\
- { \
- result = __GCONV_FULL_OUTPUT; \
- break; \
- } \
- *outptr++ = buf[0]; \
- *outptr++ = buf[1]; \
- } \
- else \
- { \
- written = ucs4_to_ksc5601 (ch, buf, 2); \
- if (written != __UNKNOWN_10646_CHAR) \
- { \
- /* We use KSC 5601. */ \
- if (__builtin_expect (outptr + 4 > outend,\
- 0)) \
- { \
- result = __GCONV_FULL_OUTPUT; \
- break; \
- } \
- *outptr++ = ESC; \
- *outptr++ = '$'; \
- *outptr++ = '('; \
- *outptr++ = 'C'; \
- set = KSC5601_set; \
- \
- if (__builtin_expect (outptr + 2 > outend,\
- 0)) \
- { \
- result = __GCONV_FULL_OUTPUT; \
- break; \
- } \
- *outptr++ = buf[0]; \
- *outptr++ = buf[1]; \
- } \
- else \
- { \
- const struct gap *rp = from_idx; \
- unsigned char gch = 0; \
+ case japanese: \
\
- while (ch > rp->end) \
- ++rp; \
- if (ch >= rp->start) \
- { \
- ch = ch - 0xa0 + rp->idx; \
- gch = iso88597_from_ucs4[ch]; \
- } \
- \
- if (__builtin_expect (gch, 1) != 0) \
- { \
- /* We use ISO 8859-7 greek. */ \
- if (__builtin_expect (outptr + 3 \
- > outend, 0)) \
- { \
- result = __GCONV_FULL_OUTPUT; \
- break; \
- } \
- *outptr++ = ESC; \
- *outptr++ = '.'; \
- *outptr++ = 'F'; \
- set2 = ISO88597_set; \
- \
- if (__builtin_expect (outptr + 3 \
- > outend, 0)) \
- { \
- result = __GCONV_FULL_OUTPUT; \
- break; \
- } \
- *outptr++ = ESC; \
- *outptr++ = 'N'; \
- *outptr++ = gch; \
- } \
- else \
- { \
- STANDARD_ERR_HANDLER (4); \
- } \
- } \
- } \
- } \
- } \
- } \
+ /* Try JIS X 0201 Roman. */ \
+ written = ucs4_to_jisx0201 (ch, buf); \
+ if (written != __UNKNOWN_10646_CHAR \
+ && buf[0] > 0x20 && buf[0] < 0x80) \
+ { \
+ if (set != JISX0201_Roman_set) \
+ { \
+ if (__builtin_expect (outptr + 3 > outend, 0)) \
+ { \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
+ *outptr++ = ESC; \
+ *outptr++ = '('; \
+ *outptr++ = 'J'; \
+ set = JISX0201_Roman_set; \
+ } \
+ \
+ if (__builtin_expect (outptr + 1 > outend, 0)) \
+ { \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
+ *outptr++ = buf[0]; \
+ result = __GCONV_OK; \
+ break; \
+ } \
+ \
+ /* Try JIS X 0208. */ \
+ written = ucs4_to_jisx0208 (ch, buf, 2); \
+ if (written != __UNKNOWN_10646_CHAR) \
+ { \
+ if (set != JISX0208_1983_set) \
+ { \
+ if (__builtin_expect (outptr + 3 > outend, 0)) \
+ { \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
+ *outptr++ = ESC; \
+ *outptr++ = '$'; \
+ *outptr++ = 'B'; \
+ set = JISX0208_1983_set; \
+ } \
+ \
+ if (__builtin_expect (outptr + 2 > outend, 0)) \
+ { \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
+ *outptr++ = buf[0]; \
+ *outptr++ = buf[1]; \
+ result = __GCONV_OK; \
+ break; \
+ } \
+ \
+ if (__builtin_expect (var == iso2022jp, 0)) \
+ /* Don't use the other Japanese character sets. */ \
+ break; \
+ \
+ /* Try JIS X 0212. */ \
+ written = ucs4_to_jisx0212 (ch, buf, 2); \
+ if (written != __UNKNOWN_10646_CHAR) \
+ { \
+ if (set != JISX0212_set) \
+ { \
+ if (__builtin_expect (outptr + 4 > outend, 0)) \
+ { \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
+ *outptr++ = ESC; \
+ *outptr++ = '$'; \
+ *outptr++ = '('; \
+ *outptr++ = 'D'; \
+ set = JISX0212_set; \
+ } \
+ \
+ if (__builtin_expect (outptr + 2 > outend, 0)) \
+ { \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
+ *outptr++ = buf[0]; \
+ *outptr++ = buf[1]; \
+ result = __GCONV_OK; \
+ break; \
+ } \
+ \
+ break; \
+ \
+ case chinese: \
+ assert (var == iso2022jp2); \
+ \
+ /* Try GB 2312. */ \
+ written = ucs4_to_gb2312 (ch, buf, 2); \
+ if (written != __UNKNOWN_10646_CHAR) \
+ { \
+ if (set != GB2312_set) \
+ { \
+ if (__builtin_expect (outptr + 3 > outend, 0)) \
+ { \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
+ *outptr++ = ESC; \
+ *outptr++ = '$'; \
+ *outptr++ = 'A'; \
+ set = GB2312_set; \
+ } \
+ \
+ if (__builtin_expect (outptr + 2 > outend, 0)) \
+ { \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
+ *outptr++ = buf[0]; \
+ *outptr++ = buf[1]; \
+ result = __GCONV_OK; \
+ break; \
+ } \
+ \
+ break; \
+ \
+ case korean: \
+ assert (var == iso2022jp2); \
+ \
+ /* Try KSC 5601. */ \
+ written = ucs4_to_ksc5601 (ch, buf, 2); \
+ if (written != __UNKNOWN_10646_CHAR) \
+ { \
+ if (set != KSC5601_set) \
+ { \
+ if (__builtin_expect (outptr + 4 > outend, 0)) \
+ { \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
+ *outptr++ = ESC; \
+ *outptr++ = '$'; \
+ *outptr++ = '('; \
+ *outptr++ = 'C'; \
+ set = KSC5601_set; \
+ } \
+ \
+ if (__builtin_expect (outptr + 2 > outend, 0)) \
+ { \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
+ *outptr++ = buf[0]; \
+ *outptr++ = buf[1]; \
+ result = __GCONV_OK; \
+ break; \
+ } \
+ \
+ break; \
+ \
+ case other: \
+ assert (var == iso2022jp2); \
+ \
+ /* Try JIS X 0201 Kana. This is not officially part \
+ of ISO-2022-JP-2, according to RFC 1554. Therefore \
+ we try this only after all other attempts. */ \
+ written = ucs4_to_jisx0201 (ch, buf); \
+ if (written != __UNKNOWN_10646_CHAR && buf[0] >= 0x80) \
+ { \
+ if (set != JISX0201_Kana_set) \
+ { \
+ if (__builtin_expect (outptr + 3 > outend, 0)) \
+ { \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
+ *outptr++ = ESC; \
+ *outptr++ = '('; \
+ *outptr++ = 'I'; \
+ set = JISX0201_Kana_set; \
+ } \
+ \
+ if (__builtin_expect (outptr + 1 > outend, 0)) \
+ { \
+ result = __GCONV_FULL_OUTPUT; \
+ break; \
+ } \
+ *outptr++ = buf[0] - 0x80; \
+ result = __GCONV_OK; \
+ break; \
+ } \
+ \
+ break; \
+ \
+ default: \
+ abort (); \
+ } \
+ while (result == __GCONV_ILLEGAL_INPUT \
+ && (conversion_list = CVLIST_REST (conversion_list)) != 0);\
+ \
+ if (result == __GCONV_FULL_OUTPUT) \
+ break; \
+ \
+ if (result == __GCONV_ILLEGAL_INPUT) \
+ { \
+ STANDARD_ERR_HANDLER (4); \
} \
} \
} \
@@ -910,8 +1066,9 @@
#define LOOP_NEED_FLAGS
#define EXTRA_LOOP_DECLS , enum variant var, int *setp
#define INIT_PARAMS int set = *setp & CURRENT_SEL_MASK; \
- int set2 = *setp & CURRENT_ASSIGN_MASK
-#define UPDATE_PARAMS *setp = set | set2
+ int set2 = *setp & CURRENT_ASSIGN_MASK; \
+ int tag = *setp & CURRENT_TAG_MASK;
+#define UPDATE_PARAMS *setp = set | set2 | tag
#include <iconv/loop.c>
More information about the Libc-alpha
mailing list