This is the mail archive of the libc-hacker@sources.redhat.com mailing list for the glibc project.

Note that libc-hacker is a closed list. You may look at the archives of this list, but subscription and posting are not open.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

[PATCH] Fix regex multi-byte handling bug


Hi!

The following patch fixes attached bug-regex17.c.
Included is also bug-regex18.c which fails (well, tests commented out)
and I'm afraid needs big regex changes, see
http://sources.redhat.com/ml/libc-hacker/2002-11/msg00052.html

2003-11-11  Jakub Jelinek  <jakub@redhat.com>

	* posix/regex_internal.c (build_wcs_buffer): Fix comment typo.
	(build_wcs_upper_buffer): Likewise.  Use towupper for wchar_t instead
	of toupper.
	* posix/Makefile (tests): Add bug-regex17 and bug-regex18.
	(bug-regex17-ENV, bug-regex18-ENV): Add LOCPATH.
	* posix/bug-regex17.c: New test.
localedata/
	* Makefile (LOCALES): Add tr_TR.UTF-8.

--- libc/posix/regex_internal.c.jj	2003-02-21 02:21:57.000000000 +0100
+++ libc/posix/regex_internal.c	2003-11-11 17:35:49.000000000 +0100
@@ -220,7 +220,7 @@ build_wcs_buffer (pstr)
 	  pstr->cur_state = prev_st;
 	}
 
-      /* Apply the translateion if we need.  */
+      /* Apply the translation if we need.  */
       if (pstr->trans != NULL && mbclen == 1)
 	{
 	  int ch = pstr->trans[pstr->raw_mbs[pstr->raw_mbs_idx + byte_idx]];
@@ -264,13 +264,13 @@ build_wcs_upper_buffer (pstr)
 	{
 	  /* In case of a singlebyte character.  */
 	  int ch = pstr->raw_mbs[pstr->raw_mbs_idx + byte_idx];
-	  /* Apply the translateion if we need.  */
+	  /* Apply the translation if we need.  */
 	  if (pstr->trans != NULL && mbclen == 1)
 	    {
 	      ch = pstr->trans[ch];
 	      pstr->mbs_case[byte_idx] = ch;
 	    }
-	  pstr->wcs[byte_idx] = iswlower (wc) ? toupper (wc) : wc;
+	  pstr->wcs[byte_idx] = iswlower (wc) ? towupper (wc) : wc;
 	  pstr->mbs[byte_idx++] = islower (ch) ? toupper (ch) : ch;
 	  if (BE (mbclen == (size_t) -1, 0))
 	    pstr->cur_state = prev_st;
@@ -282,7 +282,7 @@ build_wcs_upper_buffer (pstr)
 	  else
 	    memcpy (pstr->mbs + byte_idx,
 		    pstr->raw_mbs + pstr->raw_mbs_idx + byte_idx, mbclen);
-	  pstr->wcs[byte_idx++] = iswlower (wc) ? toupper (wc) : wc;
+	  pstr->wcs[byte_idx++] = iswlower (wc) ? towupper (wc) : wc;
 	  /* Write paddings.  */
 	  for (remain_len = byte_idx + mbclen - 1; byte_idx < remain_len ;)
 	    pstr->wcs[byte_idx++] = WEOF;
@@ -342,7 +342,7 @@ build_upper_buffer (pstr)
       int ch = pstr->raw_mbs[pstr->raw_mbs_idx + char_idx];
       if (pstr->trans != NULL)
 	{
-	  ch =  pstr->trans[ch];
+	  ch = pstr->trans[ch];
 	  pstr->mbs_case[char_idx] = ch;
 	}
       if (islower (ch))
--- libc/posix/Makefile.jj	2003-09-17 13:42:32.000000000 +0200
+++ libc/posix/Makefile	2003-11-11 18:08:22.000000000 +0100
@@ -76,8 +76,8 @@ tests		:= tstgetopt testfnm runtests run
 		   tst-chmod bug-regex1 bug-regex2 bug-regex3 bug-regex4 \
 		   tst-gnuglob tst-regex bug-regex5 bug-regex6 bug-regex7 \
 		   bug-regex8 bug-regex9 bug-regex10 bug-regex11 bug-regex12 \
-		   bug-regex13 bug-regex14 bug-regex15 bug-regex16 tst-nice \
-		   tst-nanosleep transbug
+		   bug-regex13 bug-regex14 bug-regex15 bug-regex16 \
+		   bug-regex17 bug-regex18 tst-nice tst-nanosleep transbug
 ifeq (yes,$(build-shared))
 test-srcs	:= globtest
 tests           += wordexp-test tst-exec tst-spawn
@@ -153,6 +153,8 @@ bug-regex1-ENV = LOCPATH=$(common-objpfx
 tst-regex-ENV = LOCPATH=$(common-objpfx)localedata
 bug-regex5-ENV = LOCPATH=$(common-objpfx)localedata
 bug-regex6-ENV = LOCPATH=$(common-objpfx)localedata
+bug-regex17-ENV = LOCPATH=$(common-objpfx)localedata
+bug-regex18-ENV = LOCPATH=$(common-objpfx)localedata
 
 testcases.h: TESTS TESTS2C.sed
 	sed -f TESTS2C.sed < $< > $@T
--- libc/posix/bug-regex17.c.jj	2003-11-11 17:57:14.000000000 +0100
+++ libc/posix/bug-regex17.c	2003-11-11 17:59:17.000000000 +0100
@@ -0,0 +1,95 @@
+/* Turkish regular expression tests.
+   Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Contributed by Jakub Jelinek <jakub@redhat.com>, 2002.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+#include <sys/types.h>
+#include <mcheck.h>
+#include <regex.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <locale.h>
+
+/* Tests supposed to match.  */
+struct
+{
+  const char *pattern;
+  const char *string;
+  int flags, nmatch;
+  regmatch_t rm[5];
+} tests[] = {
+  /* \xc3\x84	LATIN CAPITAL LETTER A WITH DIAERESIS
+     \xc3\x96	LATIN CAPITAL LETTER O WITH DIAERESIS
+     \xc3\xa4	LATIN SMALL LETTER A WITH DIAERESIS
+     \xc3\xb6	LATIN SMALL LETTER O WITH DIAERESIS  */
+  { "\xc3\x84\xc3\x96*\xc3\xb6$", "aB\xc3\xa4\xc3\xb6\xc3\xb6\xc3\x96", REG_ICASE, 2,
+    { { 2, 10 }, { -1, -1 } } },
+  { "[\xc3\x84x]\xc3\x96*\xc3\xb6$", "aB\xc3\x84\xc3\xb6\xc3\xb6\xc3\x96", REG_ICASE, 2,
+    { { 2, 10 }, { -1, -1 } } },
+  { "[\xc3\x84x]\xc3\x96*\xc3\xb6$", "aB\xc3\xa4\xc3\xb6\xc3\xb6\xc3\x96", REG_ICASE, 2,
+    { { 2, 10 }, { -1, -1 } } },
+  { "[^x]\xc3\x96*\xc3\xb6$", "aB\xc3\xa4\xc3\xb6\xc3\xb6\xc3\x96", REG_ICASE, 2,
+    { { 2, 10 }, { -1, -1 } } },
+};
+
+int
+main (void)
+{
+  regex_t re;
+  regmatch_t rm[5];
+  size_t i;
+  int n, ret = 0;
+
+  setlocale (LC_ALL, "de_DE.UTF-8");
+  for (i = 0; i < sizeof (tests) / sizeof (tests[0]); ++i)
+    {
+      n = regcomp (&re, tests[i].pattern, tests[i].flags);
+      if (n != 0)
+	{
+	  char buf[500];
+	  regerror (n, &re, buf, sizeof (buf));
+	  printf ("regcomp %zd failed: %s\n", i, buf);
+	  ret = 1;
+	  continue;
+	}
+
+      if (regexec (&re, tests[i].string, tests[i].nmatch, rm, 0))
+	{
+	  printf ("regexec %zd failed\n", i);
+	  ret = 1;
+	  regfree (&re);
+	  continue;
+	}
+
+      for (n = 0; n < tests[i].nmatch; ++n)
+	if (rm[n].rm_so != tests[i].rm[n].rm_so
+              || rm[n].rm_eo != tests[i].rm[n].rm_eo)
+	  {
+	    if (tests[i].rm[n].rm_so == -1 && tests[i].rm[n].rm_eo == -1)
+	      break;
+	    printf ("regexec match failure rm[%d] %d..%d\n",
+		    n, rm[n].rm_so, rm[n].rm_eo);
+	    ret = 1;
+	    break;
+	  }
+
+      regfree (&re);
+    }
+
+  return ret;
+}
--- libc/posix/bug-regex18.c.jj	2003-11-07 15:03:41.000000000 +0100
+++ libc/posix/bug-regex18.c	2003-11-11 18:07:48.000000000 +0100
@@ -0,0 +1,94 @@
+/* Turkish regular expression tests.
+   Copyright (C) 2002, 2003 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+   Contributed by Jakub Jelinek <jakub@redhat.com>, 2002.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, write to the Free
+   Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
+   02111-1307 USA.  */
+
+#include <sys/types.h>
+#include <mcheck.h>
+#include <regex.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <locale.h>
+
+/* Tests supposed to match.  */
+struct
+{
+  const char *pattern;
+  const char *string;
+  int flags, nmatch;
+  regmatch_t rm[5];
+} tests[] = {
+  /* \xc4\xb0	LATIN CAPITAL LETTER I WITH DOT ABOVE
+     \xc4\xb1	LATIN SMALL LETTER DOTLESS I  */
+#if 0
+  /* XXX Not used since they fail so far.  */
+  { "\xc4\xb0I*\xc4\xb1$", "aBi\xc4\xb1\xc4\xb1I", REG_ICASE, 2,
+    { { 2, 8 }, { -1, -1 } } },
+  { "[\xc4\xb0x]I*\xc4\xb1$", "aBi\xc4\xb1\xc4\xb1I", REG_ICASE, 2,
+    { { 2, 8 }, { -1, -1 } } },
+  { "[^x]I*\xc4\xb1$", "aBi\xc4\xb1\xc4\xb1I", REG_ICASE, 2,
+    { { 2, 8 }, { -1, -1 } } }
+#endif
+};
+
+int
+main (void)
+{
+  regex_t re;
+  regmatch_t rm[5];
+  size_t i;
+  int n, ret = 0;
+
+  setlocale (LC_ALL, "tr_TR.UTF-8");
+  for (i = 0; i < sizeof (tests) / sizeof (tests[0]); ++i)
+    {
+      n = regcomp (&re, tests[i].pattern, tests[i].flags);
+      if (n != 0)
+	{
+	  char buf[500];
+	  regerror (n, &re, buf, sizeof (buf));
+	  printf ("regcomp %zd failed: %s\n", i, buf);
+	  ret = 1;
+	  continue;
+	}
+
+      if (regexec (&re, tests[i].string, tests[i].nmatch, rm, 0))
+	{
+	  printf ("regexec %zd failed\n", i);
+	  ret = 1;
+	  regfree (&re);
+	  continue;
+	}
+
+      for (n = 0; n < tests[i].nmatch; ++n)
+	if (rm[n].rm_so != tests[i].rm[n].rm_so
+              || rm[n].rm_eo != tests[i].rm[n].rm_eo)
+	  {
+	    if (tests[i].rm[n].rm_so == -1 && tests[i].rm[n].rm_eo == -1)
+	      break;
+	    printf ("regexec match failure rm[%d] %d..%d\n",
+		    n, rm[n].rm_so, rm[n].rm_eo);
+	    ret = 1;
+	    break;
+	  }
+
+      regfree (&re);
+    }
+
+  return ret;
+}
--- libc/localedata/Makefile.jj	2003-11-11 14:37:54.000000000 +0100
+++ libc/localedata/Makefile	2003-11-11 14:37:44.000000000 +0100
@@ -131,7 +131,8 @@ ifeq (no,$(cross-compiling))
 LOCALES := de_DE.ISO-8859-1 de_DE.UTF-8 en_US.ANSI_X3.4-1968 \
 	   en_US.ISO-8859-1 ja_JP.EUC-JP da_DK.ISO-8859-1 \
 	   hr_HR.ISO-8859-2 sv_SE.ISO-8859-1 ja_JP.SJIS fr_FR.ISO-8859-1 \
-	   vi_VN.TCVN5712-1 nb_NO.ISO-8859-1 nn_NO.ISO-8859-1
+	   vi_VN.TCVN5712-1 nb_NO.ISO-8859-1 nn_NO.ISO-8859-1 \
+	   tr_TR.UTF-8
 LOCALE_SRCS := $(shell echo "$(LOCALES)"|sed 's/\([^ .]*\)[^ ]*/\1/g')
 CHARMAPS := $(shell echo "$(LOCALES)" | \
 		    sed -e 's/[^ .]*[.]\([^ ]*\)/\1/g' -e s/SJIS/SHIFT_JIS/g)


	Jakub


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]