This is the mail archive of the
glibc-cvs@sourceware.org
mailing list for the glibc project.
GNU C Library master sources branch release/2.20/master updated. glibc-2.20-18-g4d54424
- From: vapier at sourceware dot org
- To: glibc-cvs at sourceware dot org
- Date: 17 Feb 2015 07:24:49 -0000
- Subject: GNU C Library master sources branch release/2.20/master updated. glibc-2.20-18-g4d54424
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "GNU C Library master sources".
The branch, release/2.20/master has been updated
via 4d54424420c6300efbf57a7b9aa8635a8b8c1942 (commit)
via 1bf9d48aec087062e2a14b77cb5ee1fa81be334c (commit)
via f9e0f439b72e0b2fb035be1bc60aaceeed7f6ed0 (commit)
via b0694b9e98ee64cb25490de0921ce307f3872749 (commit)
from f80af76648ed97a76745fad6caa3315a79cb1c7c (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://sourceware.org/git/gitweb.cgi?p=glibc.git;a=commitdiff;h=4d54424420c6300efbf57a7b9aa8635a8b8c1942
commit 4d54424420c6300efbf57a7b9aa8635a8b8c1942
Author: Paul Pluzhnikov <ppluzhnikov@google.com>
Date: Fri Feb 6 00:30:42 2015 -0500
CVE-2015-1472: wscanf allocates too little memory
BZ #16618
Under certain conditions wscanf can allocate too little memory for the
to-be-scanned arguments and overflow the allocated buffer. The
implementation now correctly computes the required buffer size when
using malloc.
A regression test was added to tst-sscanf.
(cherry picked from commit 5bd80bfe9ca0d955bfbbc002781bc7b01b6bcb06)
Conflicts:
ChangeLog
NEWS
diff --git a/ChangeLog b/ChangeLog
index a6461e6..ccce486 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,11 @@
+2015-02-16 Paul Pluzhnikov <ppluzhnikov@google.com>
+
+ [BZ #16618]
+ * stdio-common/tst-sscanf.c (main): Test for buffer overflow.
+ * stdio-common/vfscanf.c (_IO_vfscanf_internal): Compute needed
+ size in bytes. Store needed elements in wpmax. Use needed size
+ in bytes for extend_alloca.
+
2015-02-16 H.J. Lu <hongjiu.lu@intel.com>
[BZ #17801]
diff --git a/NEWS b/NEWS
index 0eb3fb3..976f2ea 100644
--- a/NEWS
+++ b/NEWS
@@ -9,8 +9,13 @@ Version 2.20.1
* The following bugs are resolved with this release:
- 16009, 16617, 17266, 17370, 17371, 17460, 17485, 17555, 17625, 17630,
- 17801.
+ 16009, 16617, 16618, 17266, 17370, 17371, 17460, 17485, 17555, 17625,
+ 17630, 17801.
+
+* CVE-2015-1472 Under certain conditions wscanf can allocate too little
+ memory for the to-be-scanned arguments and overflow the allocated
+ buffer. The implementation now correctly computes the required buffer
+ size when using malloc.
* CVE-2104-7817 The wordexp function could ignore the WRDE_NOCMD flag
under certain input conditions resulting in the execution of a shell for
diff --git a/stdio-common/tst-sscanf.c b/stdio-common/tst-sscanf.c
index 9fef93a..6394fe1 100644
--- a/stdio-common/tst-sscanf.c
+++ b/stdio-common/tst-sscanf.c
@@ -233,5 +233,38 @@ main (void)
}
}
+ /* BZ #16618
+ The test will segfault during SSCANF if the buffer overflow
+ is not fixed. The size of `s` is such that it forces the use
+ of malloc internally and this triggers the incorrect computation.
+ Thus the value for SIZE is arbitrariy high enough that malloc
+ is used. */
+ {
+#define SIZE 131072
+ CHAR *s = malloc ((SIZE + 1) * sizeof (*s));
+ if (s == NULL)
+ abort ();
+ for (size_t i = 0; i < SIZE; i++)
+ s[i] = L('0');
+ s[SIZE] = L('\0');
+ int i = 42;
+ /* Scan multi-digit zero into `i`. */
+ if (SSCANF (s, L("%d"), &i) != 1)
+ {
+ printf ("FAIL: bug16618: SSCANF did not read one input item.\n");
+ result = 1;
+ }
+ if (i != 0)
+ {
+ printf ("FAIL: bug16618: Value of `i` was not zero as expected.\n");
+ result = 1;
+ }
+ free (s);
+ if (result != 1)
+ printf ("PASS: bug16618: Did not crash.\n");
+#undef SIZE
+ }
+
+
return result;
}
diff --git a/stdio-common/vfscanf.c b/stdio-common/vfscanf.c
index e0d2245..a4f06b4 100644
--- a/stdio-common/vfscanf.c
+++ b/stdio-common/vfscanf.c
@@ -272,9 +272,10 @@ _IO_vfscanf_internal (_IO_FILE *s, const char *format, _IO_va_list argptr,
if (__glibc_unlikely (wpsize == wpmax)) \
{ \
CHAR_T *old = wp; \
- size_t newsize = (UCHAR_MAX + 1 > 2 * wpmax \
- ? UCHAR_MAX + 1 : 2 * wpmax); \
- if (use_malloc || !__libc_use_alloca (newsize)) \
+ bool fits = __glibc_likely (wpmax <= SIZE_MAX / sizeof (CHAR_T) / 2); \
+ size_t wpneed = MAX (UCHAR_MAX + 1, 2 * wpmax); \
+ size_t newsize = fits ? wpneed * sizeof (CHAR_T) : SIZE_MAX; \
+ if (!__libc_use_alloca (newsize)) \
{ \
wp = realloc (use_malloc ? wp : NULL, newsize); \
if (wp == NULL) \
@@ -286,14 +287,13 @@ _IO_vfscanf_internal (_IO_FILE *s, const char *format, _IO_va_list argptr,
} \
if (! use_malloc) \
MEMCPY (wp, old, wpsize); \
- wpmax = newsize; \
+ wpmax = wpneed; \
use_malloc = true; \
} \
else \
{ \
size_t s = wpmax * sizeof (CHAR_T); \
- wp = (CHAR_T *) extend_alloca (wp, s, \
- newsize * sizeof (CHAR_T)); \
+ wp = (CHAR_T *) extend_alloca (wp, s, newsize); \
wpmax = s / sizeof (CHAR_T); \
if (old != NULL) \
MEMCPY (wp, old, wpsize); \
http://sourceware.org/git/gitweb.cgi?p=glibc.git;a=commitdiff;h=1bf9d48aec087062e2a14b77cb5ee1fa81be334c
commit 1bf9d48aec087062e2a14b77cb5ee1fa81be334c
Author: H.J. Lu <hjl.tools@gmail.com>
Date: Fri Jan 30 06:50:20 2015 -0800
Use AVX unaligned memcpy only if AVX2 is available
memcpy with unaligned 256-bit AVX register loads/stores are slow on older
processorsl like Sandy Bridge. This patch adds bit_AVX_Fast_Unaligned_Load
and sets it only when AVX2 is available.
[BZ #17801]
* sysdeps/x86_64/multiarch/init-arch.c (__init_cpu_features):
Set the bit_AVX_Fast_Unaligned_Load bit for AVX2.
* sysdeps/x86_64/multiarch/init-arch.h (bit_AVX_Fast_Unaligned_Load):
New.
(index_AVX_Fast_Unaligned_Load): Likewise.
(HAS_AVX_FAST_UNALIGNED_LOAD): Likewise.
* sysdeps/x86_64/multiarch/memcpy.S (__new_memcpy): Check the
bit_AVX_Fast_Unaligned_Load bit instead of the bit_AVX_Usable bit.
* sysdeps/x86_64/multiarch/memcpy_chk.S (__memcpy_chk): Likewise.
* sysdeps/x86_64/multiarch/mempcpy.S (__mempcpy): Likewise.
* sysdeps/x86_64/multiarch/mempcpy_chk.S (__mempcpy_chk): Likewise.
* sysdeps/x86_64/multiarch/memmove.c (__libc_memmove): Replace
HAS_AVX with HAS_AVX_FAST_UNALIGNED_LOAD.
* sysdeps/x86_64/multiarch/memmove_chk.c (__memmove_chk): Likewise.
(cherry picked from commit 5f3d0b78e011d2a72f9e88b0e9ef5bc081d18f97)
Conflicts:
ChangeLog
NEWS
diff --git a/ChangeLog b/ChangeLog
index 7a2e6c9..a6461e6 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,21 @@
+2015-02-16 H.J. Lu <hongjiu.lu@intel.com>
+
+ [BZ #17801]
+ * sysdeps/x86_64/multiarch/init-arch.c (__init_cpu_features):
+ Set the bit_AVX_Fast_Unaligned_Load bit for AVX2.
+ * sysdeps/x86_64/multiarch/init-arch.h (bit_AVX_Fast_Unaligned_Load):
+ New.
+ (index_AVX_Fast_Unaligned_Load): Likewise.
+ (HAS_AVX_FAST_UNALIGNED_LOAD): Likewise.
+ * sysdeps/x86_64/multiarch/memcpy.S (__new_memcpy): Check the
+ bit_AVX_Fast_Unaligned_Load bit instead of the bit_AVX_Usable bit.
+ * sysdeps/x86_64/multiarch/memcpy_chk.S (__memcpy_chk): Likewise.
+ * sysdeps/x86_64/multiarch/mempcpy.S (__mempcpy): Likewise.
+ * sysdeps/x86_64/multiarch/mempcpy_chk.S (__mempcpy_chk): Likewise.
+ * sysdeps/x86_64/multiarch/memmove.c (__libc_memmove): Replace
+ HAS_AVX with HAS_AVX_FAST_UNALIGNED_LOAD.
+ * sysdeps/x86_64/multiarch/memmove_chk.c (__memmove_chk): Likewise.
+
2015-02-16 Leonhard Holz <leonhard.holz@web.de>
[BZ #16009]
diff --git a/NEWS b/NEWS
index f578805..0eb3fb3 100644
--- a/NEWS
+++ b/NEWS
@@ -9,7 +9,8 @@ Version 2.20.1
* The following bugs are resolved with this release:
- 16009, 16617, 17266, 17370, 17371, 17460, 17485, 17555, 17625, 17630.
+ 16009, 16617, 17266, 17370, 17371, 17460, 17485, 17555, 17625, 17630,
+ 17801.
* CVE-2104-7817 The wordexp function could ignore the WRDE_NOCMD flag
under certain input conditions resulting in the execution of a shell for
diff --git a/sysdeps/x86_64/multiarch/init-arch.c b/sysdeps/x86_64/multiarch/init-arch.c
index 2a6dcb7..f7c1bbe 100644
--- a/sysdeps/x86_64/multiarch/init-arch.c
+++ b/sysdeps/x86_64/multiarch/init-arch.c
@@ -167,9 +167,14 @@ __init_cpu_features (void)
/* Determine if AVX is usable. */
if (CPUID_AVX)
__cpu_features.feature[index_AVX_Usable] |= bit_AVX_Usable;
- /* Determine if AVX2 is usable. */
+#if index_AVX2_Usable != index_AVX_Fast_Unaligned_Load
+# error index_AVX2_Usable != index_AVX_Fast_Unaligned_Load
+#endif
+ /* Determine if AVX2 is usable. Unaligned load with 256-bit
+ AVX registers are faster on processors with AVX2. */
if (CPUID_AVX2)
- __cpu_features.feature[index_AVX2_Usable] |= bit_AVX2_Usable;
+ __cpu_features.feature[index_AVX2_Usable]
+ |= bit_AVX2_Usable | bit_AVX_Fast_Unaligned_Load;
/* Determine if FMA is usable. */
if (CPUID_FMA)
__cpu_features.feature[index_FMA_Usable] |= bit_FMA_Usable;
diff --git a/sysdeps/x86_64/multiarch/init-arch.h b/sysdeps/x86_64/multiarch/init-arch.h
index ef0abbd..2fc7c7c 100644
--- a/sysdeps/x86_64/multiarch/init-arch.h
+++ b/sysdeps/x86_64/multiarch/init-arch.h
@@ -25,6 +25,7 @@
#define bit_FMA4_Usable (1 << 8)
#define bit_Slow_SSE4_2 (1 << 9)
#define bit_AVX2_Usable (1 << 10)
+#define bit_AVX_Fast_Unaligned_Load (1 << 11)
/* CPUID Feature flags. */
@@ -74,6 +75,7 @@
# define index_FMA4_Usable FEATURE_INDEX_1*FEATURE_SIZE
# define index_Slow_SSE4_2 FEATURE_INDEX_1*FEATURE_SIZE
# define index_AVX2_Usable FEATURE_INDEX_1*FEATURE_SIZE
+# define index_AVX_Fast_Unaligned_Load FEATURE_INDEX_1*FEATURE_SIZE
#else /* __ASSEMBLER__ */
@@ -169,6 +171,7 @@ extern const struct cpu_features *__get_cpu_features (void)
# define index_FMA4_Usable FEATURE_INDEX_1
# define index_Slow_SSE4_2 FEATURE_INDEX_1
# define index_AVX2_Usable FEATURE_INDEX_1
+# define index_AVX_Fast_Unaligned_Load FEATURE_INDEX_1
# define HAS_ARCH_FEATURE(name) \
((__get_cpu_features ()->feature[index_##name] & (bit_##name)) != 0)
@@ -181,5 +184,6 @@ extern const struct cpu_features *__get_cpu_features (void)
# define HAS_AVX2 HAS_ARCH_FEATURE (AVX2_Usable)
# define HAS_FMA HAS_ARCH_FEATURE (FMA_Usable)
# define HAS_FMA4 HAS_ARCH_FEATURE (FMA4_Usable)
+# define HAS_AVX_FAST_UNALIGNED_LOAD HAS_ARCH_FEATURE (AVX_Fast_Unaligned_Load)
#endif /* __ASSEMBLER__ */
diff --git a/sysdeps/x86_64/multiarch/memcpy.S b/sysdeps/x86_64/multiarch/memcpy.S
index e666695..10bbd39 100644
--- a/sysdeps/x86_64/multiarch/memcpy.S
+++ b/sysdeps/x86_64/multiarch/memcpy.S
@@ -33,7 +33,7 @@ ENTRY(__new_memcpy)
jne 1f
call __init_cpu_features
1: leaq __memcpy_avx_unaligned(%rip), %rax
- testl $bit_AVX_Usable, __cpu_features+FEATURE_OFFSET+index_AVX_Usable(%rip)
+ testl $bit_AVX_Fast_Unaligned_Load, __cpu_features+FEATURE_OFFSET+index_AVX_Fast_Unaligned_Load(%rip)
jz 1f
ret
1: leaq __memcpy_sse2(%rip), %rax
diff --git a/sysdeps/x86_64/multiarch/memcpy_chk.S b/sysdeps/x86_64/multiarch/memcpy_chk.S
index 076b19a..30cca20 100644
--- a/sysdeps/x86_64/multiarch/memcpy_chk.S
+++ b/sysdeps/x86_64/multiarch/memcpy_chk.S
@@ -39,7 +39,7 @@ ENTRY(__memcpy_chk)
testl $bit_Fast_Copy_Backward, __cpu_features+FEATURE_OFFSET+index_Fast_Copy_Backward(%rip)
jz 2f
leaq __memcpy_chk_ssse3_back(%rip), %rax
- testl $bit_AVX_Usable, __cpu_features+FEATURE_OFFSET+index_AVX_Usable(%rip)
+ testl $bit_AVX_Fast_Unaligned_Load, __cpu_features+FEATURE_OFFSET+index_AVX_Fast_Unaligned_Load(%rip)
jz 2f
leaq __memcpy_chk_avx_unaligned(%rip), %rax
2: ret
diff --git a/sysdeps/x86_64/multiarch/memmove.c b/sysdeps/x86_64/multiarch/memmove.c
index 0c9af7e..2c86a4a 100644
--- a/sysdeps/x86_64/multiarch/memmove.c
+++ b/sysdeps/x86_64/multiarch/memmove.c
@@ -49,7 +49,7 @@ extern __typeof (__redirect_memmove) __memmove_avx_unaligned attribute_hidden;
ifunc symbol properly. */
extern __typeof (__redirect_memmove) __libc_memmove;
libc_ifunc (__libc_memmove,
- HAS_AVX
+ HAS_AVX_FAST_UNALIGNED_LOAD
? __memmove_avx_unaligned
: (HAS_SSSE3
? (HAS_FAST_COPY_BACKWARD
diff --git a/sysdeps/x86_64/multiarch/memmove_chk.c b/sysdeps/x86_64/multiarch/memmove_chk.c
index 44344f2..5ffcaec 100644
--- a/sysdeps/x86_64/multiarch/memmove_chk.c
+++ b/sysdeps/x86_64/multiarch/memmove_chk.c
@@ -30,7 +30,7 @@ extern __typeof (__memmove_chk) __memmove_chk_avx_unaligned attribute_hidden;
#include "debug/memmove_chk.c"
libc_ifunc (__memmove_chk,
- HAS_AVX ? __memmove_chk_avx_unaligned :
+ HAS_AVX_FAST_UNALIGNED_LOAD ? __memmove_chk_avx_unaligned :
(HAS_SSSE3
? (HAS_FAST_COPY_BACKWARD
? __memmove_chk_ssse3_back : __memmove_chk_ssse3)
diff --git a/sysdeps/x86_64/multiarch/mempcpy.S b/sysdeps/x86_64/multiarch/mempcpy.S
index 7589d8c..e205ef5 100644
--- a/sysdeps/x86_64/multiarch/mempcpy.S
+++ b/sysdeps/x86_64/multiarch/mempcpy.S
@@ -37,7 +37,7 @@ ENTRY(__mempcpy)
testl $bit_Fast_Copy_Backward, __cpu_features+FEATURE_OFFSET+index_Fast_Copy_Backward(%rip)
jz 2f
leaq __mempcpy_ssse3_back(%rip), %rax
- testl $bit_AVX_Usable, __cpu_features+FEATURE_OFFSET+index_AVX_Usable(%rip)
+ testl $bit_AVX_Fast_Unaligned_Load, __cpu_features+FEATURE_OFFSET+index_AVX_Fast_Unaligned_Load(%rip)
jz 2f
leaq __mempcpy_avx_unaligned(%rip), %rax
2: ret
diff --git a/sysdeps/x86_64/multiarch/mempcpy_chk.S b/sysdeps/x86_64/multiarch/mempcpy_chk.S
index 88e0b74..dd777df 100644
--- a/sysdeps/x86_64/multiarch/mempcpy_chk.S
+++ b/sysdeps/x86_64/multiarch/mempcpy_chk.S
@@ -39,7 +39,7 @@ ENTRY(__mempcpy_chk)
testl $bit_Fast_Copy_Backward, __cpu_features+FEATURE_OFFSET+index_Fast_Copy_Backward(%rip)
jz 2f
leaq __mempcpy_chk_ssse3_back(%rip), %rax
- testl $bit_AVX_Usable, __cpu_features+FEATURE_OFFSET+index_AVX_Usable(%rip)
+ testl $bit_AVX_Fast_Unaligned_Load, __cpu_features+FEATURE_OFFSET+index_AVX_Fast_Unaligned_Load(%rip)
jz 2f
leaq __mempcpy_chk_avx_unaligned(%rip), %rax
2: ret
http://sourceware.org/git/gitweb.cgi?p=glibc.git;a=commitdiff;h=f9e0f439b72e0b2fb035be1bc60aaceeed7f6ed0
commit f9e0f439b72e0b2fb035be1bc60aaceeed7f6ed0
Author: Leonhard Holz <leonhard.holz@web.de>
Date: Tue Jan 13 11:33:56 2015 +0530
Fix memory handling in strxfrm_l [BZ #16009]
[Modified from the original email by Siddhesh Poyarekar]
This patch solves bug #16009 by implementing an additional path in
strxfrm that does not depend on caching the weight and rule indices.
In detail the following changed:
* The old main loop was factored out of strxfrm_l into the function
do_xfrm_cached to be able to alternativly use the non-caching version
do_xfrm.
* strxfrm_l allocates a a fixed size array on the stack. If this is not
sufficiant to store the weight and rule indices, the non-caching path is
taken. As the cache size is not dependent on the input there can be no
problems with integer overflows or stack allocations greater than
__MAX_ALLOCA_CUTOFF. Note that malloc-ing is not possible because the
definition of strxfrm does not allow an oom errorhandling.
* The uncached path determines the weight and rule index for every char
and for every pass again.
* Passing all the locale data array by array resulted in very long
parameter lists, so I introduced a structure that holds them.
* Checking for zero src string has been moved a bit upwards, it is
before the locale data initialization now.
* To verify that the non-caching path works correct I added a test run
to localedata/sort-test.sh & localedata/xfrm-test.c where all strings
are patched up with spaces so that they are too large for the caching path.
(cherry picked from commit 0f9e585480edcdf1e30dc3d79e24b84aeee516fa)
Conflicts:
ChangeLog
NEWS
diff --git a/ChangeLog b/ChangeLog
index dbf7e86..7a2e6c9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,19 @@
+2015-02-16 Leonhard Holz <leonhard.holz@web.de>
+
+ [BZ #16009]
+ * string/strxfrm_l.c (STRXFRM): Allocate fixed size cache for
+ weights and rules. Use do_xfrm_cached if data fits in cache,
+ do_xfrm otherwise. Moved former main loop to...
+ * (do_xfrm_cached): New function.
+ * (do_xfrm): Non-caching version of do_xfrm_cached. Uses
+ find_idx, find_position and stack_push.
+ * (find_idx): New function.
+ * (find_position): Likewise.
+ * localedata/sort-test.sh: Added test run for do_xfrm.
+ * localedata/xfrm-test.c (main): Added command line option
+ -nocache to run the test with strings that are too large for
+ the STRXFRM cache.
+
2015-02-16 Kostya Serebryany <konstantin.s.serebryany@gmail.com>
Roland McGrath <roland@hack.frob.com>
diff --git a/NEWS b/NEWS
index 9bc835c..f578805 100644
--- a/NEWS
+++ b/NEWS
@@ -9,7 +9,7 @@ Version 2.20.1
* The following bugs are resolved with this release:
- 16617, 17266, 17370, 17371, 17460, 17485, 17555, 17625, 17630.
+ 16009, 16617, 17266, 17370, 17371, 17460, 17485, 17555, 17625, 17630.
* CVE-2104-7817 The wordexp function could ignore the WRDE_NOCMD flag
under certain input conditions resulting in the execution of a shell for
diff --git a/localedata/sort-test.sh b/localedata/sort-test.sh
index e37129a..3cb57fb 100644
--- a/localedata/sort-test.sh
+++ b/localedata/sort-test.sh
@@ -53,11 +53,18 @@ for l in $lang; do
${common_objpfx}localedata/xfrm-test $id < $cns.in \
> ${common_objpfx}localedata/$cns.xout || here=1
cmp -s $cns.in ${common_objpfx}localedata/$cns.xout || here=1
+ ${test_program_prefix_before_env} \
+ ${run_program_env} \
+ LC_ALL=$l ${test_program_prefix_after_env} \
+ ${common_objpfx}localedata/xfrm-test $id -nocache < $cns.in \
+ > ${common_objpfx}localedata/$cns.nocache.xout || here=1
+ cmp -s $cns.in ${common_objpfx}localedata/$cns.nocache.xout || here=1
if test $here -eq 0; then
echo "$l xfrm-test OK"
else
echo "$l xfrm-test FAIL"
diff -u $cns.in ${common_objpfx}localedata/$cns.xout | sed 's/^/ /'
+ diff -u $cns.in ${common_objpfx}localedata/$cns.nocache.xout | sed 's/^/ /'
status=1
fi
done
diff --git a/localedata/xfrm-test.c b/localedata/xfrm-test.c
index d2aba7d..5cf29f6 100644
--- a/localedata/xfrm-test.c
+++ b/localedata/xfrm-test.c
@@ -23,7 +23,10 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <stdbool.h>
+/* Keep in sync with string/strxfrm_l.c. */
+#define SMALL_STR_SIZE 4095
struct lines
{
@@ -37,6 +40,7 @@ int
main (int argc, char *argv[])
{
int result = 0;
+ bool nocache = false;
size_t nstrings, nstrings_max;
struct lines *strings;
char *line = NULL;
@@ -44,7 +48,18 @@ main (int argc, char *argv[])
size_t n;
if (argc < 2)
- error (1, 0, "usage: %s <random seed>", argv[0]);
+ error (1, 0, "usage: %s <random seed> [-nocache]", argv[0]);
+
+ if (argc == 3)
+ {
+ if (strcmp (argv[2], "-nocache") == 0)
+ nocache = true;
+ else
+ {
+ printf ("Unknown option %s!\n", argv[2]);
+ exit (1);
+ }
+ }
setlocale (LC_ALL, "");
@@ -59,9 +74,9 @@ main (int argc, char *argv[])
while (1)
{
- char saved, *newp;
- int needed;
- int l;
+ char saved, *word, *newp;
+ size_t l, line_len, needed;
+
if (getline (&line, &len, stdin) < 0)
break;
@@ -83,10 +98,35 @@ main (int argc, char *argv[])
saved = line[l];
line[l] = '\0';
- needed = strxfrm (NULL, line, 0);
+
+ if (nocache)
+ {
+ line_len = strlen (line);
+ word = malloc (line_len + SMALL_STR_SIZE + 1);
+ if (word == NULL)
+ {
+ printf ("malloc failed: %m\n");
+ exit (1);
+ }
+ memset (word, ' ', SMALL_STR_SIZE);
+ memcpy (word + SMALL_STR_SIZE, line, line_len);
+ word[line_len + SMALL_STR_SIZE] = '\0';
+ }
+ else
+ word = line;
+
+ needed = strxfrm (NULL, word, 0);
newp = malloc (needed + 1);
- strxfrm (newp, line, needed + 1);
+ if (newp == NULL)
+ {
+ printf ("malloc failed: %m\n");
+ exit (1);
+ }
+ strxfrm (newp, word, needed + 1);
strings[nstrings].xfrm = newp;
+
+ if (nocache)
+ free (word);
line[l] = saved;
++nstrings;
}
diff --git a/string/strxfrm_l.c b/string/strxfrm_l.c
index 2d3f1bd..95ffd6f 100644
--- a/string/strxfrm_l.c
+++ b/string/strxfrm_l.c
@@ -40,9 +40,24 @@
#define CONCAT(a,b) CONCAT1(a,b)
#define CONCAT1(a,b) a##b
+/* Maximum string size that is calculated with cached indices. Right now this
+ is an arbitrary value open to optimizations. SMALL_STR_SIZE * 4 has to be
+ lower than __MAX_ALLOCA_CUTOFF. Keep localedata/xfrm-test.c in sync. */
+#define SMALL_STR_SIZE 4095
+
#include "../locale/localeinfo.h"
#include WEIGHT_H
+/* Group locale data for shorter parameter lists. */
+typedef struct
+{
+ uint_fast32_t nrules;
+ unsigned char *rulesets;
+ USTRING_TYPE *weights;
+ int32_t *table;
+ USTRING_TYPE *extra;
+ int32_t *indirect;
+} locale_data_t;
#ifndef WIDE_CHAR_VERSION
@@ -81,113 +96,325 @@ utf8_encode (char *buf, int val)
}
#endif
+/* Find next weight and rule index. Inlined since called for every char. */
+static __always_inline size_t
+find_idx (const USTRING_TYPE **us, int32_t *weight_idx,
+ unsigned char *rule_idx, const locale_data_t *l_data, const int pass)
+{
+ int32_t tmp = findidx (l_data->table, l_data->indirect, l_data->extra, us,
+ -1);
+ *rule_idx = tmp >> 24;
+ int32_t idx = tmp & 0xffffff;
+ size_t len = l_data->weights[idx++];
+
+ /* Skip over indices of previous levels. */
+ for (int i = 0; i < pass; i++)
+ {
+ idx += len;
+ len = l_data->weights[idx++];
+ }
-size_t
-STRXFRM (STRING_TYPE *dest, const STRING_TYPE *src, size_t n, __locale_t l)
+ *weight_idx = idx;
+ return len;
+}
+
+static int
+find_position (const USTRING_TYPE *us, const locale_data_t *l_data,
+ const int pass)
{
- struct __locale_data *current = l->__locales[LC_COLLATE];
- uint_fast32_t nrules = current->values[_NL_ITEM_INDEX (_NL_COLLATE_NRULES)].word;
- /* We don't assign the following values right away since it might be
- unnecessary in case there are no rules. */
- const unsigned char *rulesets;
- const int32_t *table;
- const USTRING_TYPE *weights;
- const USTRING_TYPE *extra;
- const int32_t *indirect;
+ int32_t weight_idx;
+ unsigned char rule_idx;
+ const USTRING_TYPE *usrc = us;
+
+ find_idx (&usrc, &weight_idx, &rule_idx, l_data, pass);
+ return l_data->rulesets[rule_idx * l_data->nrules + pass] & sort_position;
+}
+
+/* Do the transformation. */
+static size_t
+do_xfrm (const USTRING_TYPE *usrc, STRING_TYPE *dest, size_t n,
+ const locale_data_t *l_data)
+{
+ int32_t weight_idx;
+ unsigned char rule_idx;
uint_fast32_t pass;
- size_t needed;
+ size_t needed = 0;
size_t last_needed;
- const USTRING_TYPE *usrc;
- size_t srclen = STRLEN (src);
- int32_t *idxarr;
- unsigned char *rulearr;
- size_t idxmax;
- size_t idxcnt;
- int use_malloc;
- if (nrules == 0)
+ /* Now the passes over the weights. */
+ for (pass = 0; pass < l_data->nrules; ++pass)
{
- if (n != 0)
- STPNCPY (dest, src, MIN (srclen + 1, n));
+ size_t backw_len = 0;
+ last_needed = needed;
+ const USTRING_TYPE *cur = usrc;
+ const USTRING_TYPE *backw_start = NULL;
- return srclen;
- }
+ /* We assume that if a rule has defined `position' in one section
+ this is true for all of them. */
+ int position = find_position (cur, l_data, pass);
- rulesets = (const unsigned char *)
- current->values[_NL_ITEM_INDEX (_NL_COLLATE_RULESETS)].string;
- table = (const int32_t *)
- current->values[_NL_ITEM_INDEX (CONCAT(_NL_COLLATE_TABLE,SUFFIX))].string;
- weights = (const USTRING_TYPE *)
- current->values[_NL_ITEM_INDEX (CONCAT(_NL_COLLATE_WEIGHT,SUFFIX))].string;
- extra = (const USTRING_TYPE *)
- current->values[_NL_ITEM_INDEX (CONCAT(_NL_COLLATE_EXTRA,SUFFIX))].string;
- indirect = (const int32_t *)
- current->values[_NL_ITEM_INDEX (CONCAT(_NL_COLLATE_INDIRECT,SUFFIX))].string;
- use_malloc = 0;
+ if (position == 0)
+ {
+ while (*cur != L('\0'))
+ {
+ const USTRING_TYPE *pos = cur;
+ size_t len = find_idx (&cur, &weight_idx, &rule_idx, l_data,
+ pass);
+ int rule = l_data->rulesets[rule_idx * l_data->nrules + pass];
- assert (((uintptr_t) table) % __alignof__ (table[0]) == 0);
- assert (((uintptr_t) weights) % __alignof__ (weights[0]) == 0);
- assert (((uintptr_t) extra) % __alignof__ (extra[0]) == 0);
- assert (((uintptr_t) indirect) % __alignof__ (indirect[0]) == 0);
+ if ((rule & sort_forward) != 0)
+ {
+ /* Handle the pushed backward sequence. */
+ if (backw_start != NULL)
+ {
+ for (size_t i = backw_len; i > 0; )
+ {
+ int32_t weight_idx;
+ unsigned char rule_idx;
+ size_t len = find_idx (&backw_start, &weight_idx,
+ &rule_idx, l_data, pass);
+ if (needed + i < n)
+ for (size_t j = len; j > 0; j--)
+ dest[needed + i - j] =
+ l_data->weights[weight_idx++];
+
+ i -= len;
+ }
- /* Handle an empty string as a special case. */
- if (srclen == 0)
- {
- if (n != 0)
- *dest = L('\0');
- return 0;
- }
+ needed += backw_len;
+ backw_start = NULL;
+ backw_len = 0;
+ }
- /* We need the elements of the string as unsigned values since they
- are used as indeces. */
- usrc = (const USTRING_TYPE *) src;
-
- /* Perform the first pass over the string and while doing this find
- and store the weights for each character. Since we want this to
- be as fast as possible we are using `alloca' to store the temporary
- values. But since there is no limit on the length of the string
- we have to use `malloc' if the string is too long. We should be
- very conservative here. */
- if (! __libc_use_alloca ((srclen + 1) * (sizeof (int32_t) + 1)))
- {
- idxarr = (int32_t *) malloc ((srclen + 1) * (sizeof (int32_t) + 1));
- rulearr = (unsigned char *) &idxarr[srclen];
-
- if (idxarr == NULL)
- /* No memory. Well, go with the stack then.
-
- XXX Once this implementation is stable we will handle this
- differently. Instead of precomputing the indeces we will
- do this in time. This means, though, that this happens for
- every pass again. */
- goto try_stack;
- use_malloc = 1;
- }
- else
- {
- try_stack:
- idxarr = (int32_t *) alloca (srclen * sizeof (int32_t));
- rulearr = (unsigned char *) alloca (srclen + 1);
+ /* Now handle the forward element. */
+ if (needed + len < n)
+ while (len-- > 0)
+ dest[needed++] = l_data->weights[weight_idx++];
+ else
+ /* No more characters fit into the buffer. */
+ needed += len;
+ }
+ else
+ {
+ /* Remember start of the backward sequence & track length. */
+ if (backw_start == NULL)
+ backw_start = pos;
+ backw_len += len;
+ }
+ }
+
+
+ /* Handle the pushed backward sequence. */
+ if (backw_start != NULL)
+ {
+ for (size_t i = backw_len; i > 0; )
+ {
+ size_t len = find_idx (&backw_start, &weight_idx, &rule_idx,
+ l_data, pass);
+ if (needed + i < n)
+ for (size_t j = len; j > 0; j--)
+ dest[needed + i - j] =
+ l_data->weights[weight_idx++];
+
+ i -= len;
+ }
+
+ needed += backw_len;
+ }
+ }
+ else
+ {
+ int val = 1;
+#ifndef WIDE_CHAR_VERSION
+ char buf[7];
+ size_t buflen;
+#endif
+ size_t i;
+
+ while (*cur != L('\0'))
+ {
+ const USTRING_TYPE *pos = cur;
+ size_t len = find_idx (&cur, &weight_idx, &rule_idx, l_data,
+ pass);
+ int rule = l_data->rulesets[rule_idx * l_data->nrules + pass];
+
+ if ((rule & sort_forward) != 0)
+ {
+ /* Handle the pushed backward sequence. */
+ if (backw_start != NULL)
+ {
+ for (size_t p = backw_len; p > 0; p--)
+ {
+ size_t len;
+ int32_t weight_idx;
+ unsigned char rule_idx;
+ const USTRING_TYPE *backw_cur = backw_start;
+
+ /* To prevent a warning init the used vars. */
+ len = find_idx (&backw_cur, &weight_idx,
+ &rule_idx, l_data, pass);
+
+ for (i = 1; i < p; i++)
+ len = find_idx (&backw_cur, &weight_idx,
+ &rule_idx, l_data, pass);
+
+ if (len != 0)
+ {
+#ifdef WIDE_CHAR_VERSION
+ if (needed + 1 + len < n)
+ {
+ dest[needed] = val;
+ for (i = 0; i < len; ++i)
+ dest[needed + 1 + i] =
+ l_data->weights[weight_idx + i];
+ }
+ needed += 1 + len;
+#else
+ buflen = utf8_encode (buf, val);
+ if (needed + buflen + len < n)
+ {
+ for (i = 0; i < buflen; ++i)
+ dest[needed + i] = buf[i];
+ for (i = 0; i < len; ++i)
+ dest[needed + buflen + i] =
+ l_data->weights[weight_idx + i];
+ }
+ needed += buflen + len;
+#endif
+ val = 1;
+ }
+ else
+ ++val;
+ }
+
+ backw_start = NULL;
+ backw_len = 0;
+ }
+
+ /* Now handle the forward element. */
+ if (len != 0)
+ {
+#ifdef WIDE_CHAR_VERSION
+ if (needed + 1 + len < n)
+ {
+ dest[needed] = val;
+ for (i = 0; i < len; ++i)
+ dest[needed + 1 + i] =
+ l_data->weights[weight_idx + i];
+ }
+ needed += 1 + len;
+#else
+ buflen = utf8_encode (buf, val);
+ if (needed + buflen + len < n)
+ {
+ for (i = 0; i < buflen; ++i)
+ dest[needed + i] = buf[i];
+ for (i = 0; i < len; ++i)
+ dest[needed + buflen + i] =
+ l_data->weights[weight_idx + i];
+ }
+ needed += buflen + len;
+#endif
+ val = 1;
+ }
+ else
+ ++val;
+ }
+ else
+ {
+ /* Remember start of the backward sequence & track length. */
+ if (backw_start == NULL)
+ backw_start = pos;
+ backw_len++;
+ }
+ }
+
+ /* Handle the pushed backward sequence. */
+ if (backw_start != NULL)
+ {
+ for (size_t p = backw_len; p > 0; p--)
+ {
+ size_t len;
+ int32_t weight_idx;
+ unsigned char rule_idx;
+ const USTRING_TYPE *backw_cur = backw_start;
+
+ /* To prevent a warning init the used vars. */
+ len = find_idx (&backw_cur, &weight_idx,
+ &rule_idx, l_data, pass);
+
+ for (i = 1; i < p; i++)
+ len = find_idx (&backw_cur, &weight_idx,
+ &rule_idx, l_data, pass);
+
+ if (len != 0)
+ {
+#ifdef WIDE_CHAR_VERSION
+ if (needed + 1 + len < n)
+ {
+ dest[needed] = val;
+ for (i = 0; i < len; ++i)
+ dest[needed + 1 + i] =
+ l_data->weights[weight_idx + i];
+ }
+ needed += 1 + len;
+#else
+ buflen = utf8_encode (buf, val);
+ if (needed + buflen + len < n)
+ {
+ for (i = 0; i < buflen; ++i)
+ dest[needed + i] = buf[i];
+ for (i = 0; i < len; ++i)
+ dest[needed + buflen + i] =
+ l_data->weights[weight_idx + i];
+ }
+ needed += buflen + len;
+#endif
+ val = 1;
+ }
+ else
+ ++val;
+ }
+ }
+ }
+
+ /* Finally store the byte to separate the passes or terminate
+ the string. */
+ if (needed < n)
+ dest[needed] = pass + 1 < l_data->nrules ? L('\1') : L('\0');
+ ++needed;
}
- idxmax = 0;
- do
+ /* This is a little optimization: many collation specifications have
+ a `position' rule at the end and if no non-ignored character
+ is found the last \1 byte is immediately followed by a \0 byte
+ signalling this. We can avoid the \1 byte(s). */
+ if (needed > 2 && needed == last_needed + 1)
{
- int32_t tmp = findidx (table, indirect, extra, &usrc, -1);
- rulearr[idxmax] = tmp >> 24;
- idxarr[idxmax] = tmp & 0xffffff;
-
- ++idxmax;
+ /* Remove the \1 byte. */
+ if (--needed <= n)
+ dest[needed - 1] = L('\0');
}
- while (*usrc != L('\0'));
- /* This element is only read, the value never used but to determine
- another value which then is ignored. */
- rulearr[idxmax] = '\0';
+ /* Return the number of bytes/words we need, but don't count the NUL
+ byte/word at the end. */
+ return needed - 1;
+}
+
+/* Do the transformation using weight-index and rule cache. */
+static size_t
+do_xfrm_cached (STRING_TYPE *dest, size_t n, const locale_data_t *l_data,
+ size_t idxmax, int32_t *idxarr, const unsigned char *rulearr)
+{
+ uint_fast32_t nrules = l_data->nrules;
+ unsigned char *rulesets = l_data->rulesets;
+ USTRING_TYPE *weights = l_data->weights;
+ uint_fast32_t pass;
+ size_t needed = 0;
+ size_t last_needed;
+ size_t idxcnt;
- /* Now the passes over the weights. We now use the indeces we found
- before. */
- needed = 0;
+ /* Now the passes over the weights. */
for (pass = 0; pass < nrules; ++pass)
{
size_t backw_stop = ~0ul;
@@ -433,14 +660,87 @@ STRXFRM (STRING_TYPE *dest, const STRING_TYPE *src, size_t n, __locale_t l)
dest[needed - 1] = L('\0');
}
- /* Free the memory if needed. */
- if (use_malloc)
- free (idxarr);
-
/* Return the number of bytes/words we need, but don't count the NUL
byte/word at the end. */
return needed - 1;
}
+
+size_t
+STRXFRM (STRING_TYPE *dest, const STRING_TYPE *src, size_t n, __locale_t l)
+{
+ locale_data_t l_data;
+ struct __locale_data *current = l->__locales[LC_COLLATE];
+ l_data.nrules = current->values[_NL_ITEM_INDEX (_NL_COLLATE_NRULES)].word;
+
+ /* Handle byte comparison case. */
+ if (l_data.nrules == 0)
+ {
+ size_t srclen = STRLEN (src);
+
+ if (n != 0)
+ STPNCPY (dest, src, MIN (srclen + 1, n));
+
+ return srclen;
+ }
+
+ /* Handle an empty string, code hereafter relies on strlen (src) > 0. */
+ if (*src == L('\0'))
+ {
+ if (n != 0)
+ *dest = L('\0');
+ return 0;
+ }
+
+ /* Get the locale data. */
+ l_data.rulesets = (unsigned char *)
+ current->values[_NL_ITEM_INDEX (_NL_COLLATE_RULESETS)].string;
+ l_data.table = (int32_t *)
+ current->values[_NL_ITEM_INDEX (CONCAT(_NL_COLLATE_TABLE,SUFFIX))].string;
+ l_data.weights = (USTRING_TYPE *)
+ current->values[_NL_ITEM_INDEX (CONCAT(_NL_COLLATE_WEIGHT,SUFFIX))].string;
+ l_data.extra = (USTRING_TYPE *)
+ current->values[_NL_ITEM_INDEX (CONCAT(_NL_COLLATE_EXTRA,SUFFIX))].string;
+ l_data.indirect = (int32_t *)
+ current->values[_NL_ITEM_INDEX (CONCAT(_NL_COLLATE_INDIRECT,SUFFIX))].string;
+
+ assert (((uintptr_t) l_data.table) % __alignof__ (l_data.table[0]) == 0);
+ assert (((uintptr_t) l_data.weights) % __alignof__ (l_data.weights[0]) == 0);
+ assert (((uintptr_t) l_data.extra) % __alignof__ (l_data.extra[0]) == 0);
+ assert (((uintptr_t) l_data.indirect) % __alignof__ (l_data.indirect[0]) == 0);
+
+ /* We need the elements of the string as unsigned values since they
+ are used as indeces. */
+ const USTRING_TYPE *usrc = (const USTRING_TYPE *) src;
+
+ /* Allocate cache for small strings on the stack and fill it with weight and
+ rule indices. If the cache size is not sufficient, continue with the
+ uncached xfrm version. */
+ size_t idxmax = 0;
+ const USTRING_TYPE *cur = usrc;
+ int32_t *idxarr = alloca (SMALL_STR_SIZE * sizeof (int32_t));
+ unsigned char *rulearr = alloca (SMALL_STR_SIZE + 1);
+
+ do
+ {
+ int32_t tmp = findidx (l_data.table, l_data.indirect, l_data.extra, &cur,
+ -1);
+ rulearr[idxmax] = tmp >> 24;
+ idxarr[idxmax] = tmp & 0xffffff;
+
+ ++idxmax;
+ }
+ while (*cur != L('\0') && idxmax < SMALL_STR_SIZE);
+
+ /* This element is only read, the value never used but to determine
+ another value which then is ignored. */
+ rulearr[idxmax] = '\0';
+
+ /* Do the transformation. */
+ if (*cur == L('\0'))
+ return do_xfrm_cached (dest, n, &l_data, idxmax, idxarr, rulearr);
+ else
+ return do_xfrm (usrc, dest, n, &l_data);
+}
libc_hidden_def (STRXFRM)
#ifndef WIDE_CHAR_VERSION
http://sourceware.org/git/gitweb.cgi?p=glibc.git;a=commitdiff;h=b0694b9e98ee64cb25490de0921ce307f3872749
commit b0694b9e98ee64cb25490de0921ce307f3872749
Author: Roland McGrath <roland@hack.frob.com>
Date: Thu Sep 11 16:02:17 2014 -0700
Move findidx nested functions to top-level.
Needed in order to backport strxfrm_l security fix cleanly.
(cherry picked from commit 8c0ab919f63dc03a420751172602a52d2bea59a8)
Conflicts:
ChangeLog
diff --git a/ChangeLog b/ChangeLog
index e416cd0..dbf7e86 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,38 @@
+2015-02-16 Kostya Serebryany <konstantin.s.serebryany@gmail.com>
+ Roland McGrath <roland@hack.frob.com>
+
+ * locale/weight.h: Add include guard.
+ (findidx): Make static rather than auto; take new parameters
+ TABLE, INDIRECT, and EXTRA instead of getting them as outer locals.
+ * locale/weightwc.h: Likewise.
+ * posix/fnmatch_loop.c
+ (FCT): Change type of EXTRA from int32_t to wint_t.
+ Don't include either header inside the function.
+ Call FINDIDX rather than findidx, and pass new arguments.
+ #undef FINDIDX at the end of the file.
+ * posix/fnmatch.c [_LIBC]: #include <locale/weight.h> and define
+ FINDIDX before including fnmatch_loop.c for the non-wide version.
+ [_LIBC] [HANDLE_MULTIBYTE]: #define findidx to findidxwc around
+ #include <locale/weightwc.h>, and define FINDIDX to findidxwc
+ for the wide version.
+ * posix/regcomp.c [_LIBC]: #include <locale/weight.h>.
+ (build_equiv_class) [_LIBC]: Don't #include it inside the function.
+ Pass new arguments to findidx.
+ * posix/regexec.c [RE_ENABLE_I18N] [_LIBC]: #include <locale/weight.h>.
+ [RE_ENABLE_I18N] (check_node_accept_bytes) [_LIBC]:
+ Don't #include it inside the function. Pass new arguments to findidx.
+ * posix/regex_internal.h
+ [!NOT_IN_libc] [_LIBC]: #include <locale/weight.h>.
+ (re_string_elem_size_at): Don't #include it inside the function.
+ Pass new arguments to findidx.
+ * string/strcoll_l.c: #include WEIGHT_H at top level.
+ (get_next_seq): Don't #include it inside the function.
+ Pass new arguments to findidx.
+ (get_next_seq_nocache): Likewise.
+ * string/strxfrm_l.c: #include WEIGHT_H at top level.
+ (STRXFRM): Don't #include it inside the function.
+ Pass new arguments to findidx.
+
2014-12-16 Florian Weimer <fweimer@redhat.com>
[BZ #17630]
diff --git a/locale/weight.h b/locale/weight.h
index 9eb8ac6..9d93fdc 100644
--- a/locale/weight.h
+++ b/locale/weight.h
@@ -16,10 +16,15 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
+#ifndef _WEIGHT_H_
+#define _WEIGHT_H_ 1
+
/* Find index of weight. */
-auto inline int32_t
-__attribute ((always_inline))
-findidx (const unsigned char **cpp, size_t len)
+static inline int32_t __attribute__ ((always_inline))
+findidx (const int32_t *table,
+ const int32_t *indirect,
+ const unsigned char *extra,
+ const unsigned char **cpp, size_t len)
{
int_fast32_t i = table[*(*cpp)++];
const unsigned char *cp;
@@ -130,3 +135,5 @@ findidx (const unsigned char **cpp, size_t len)
/* NOTREACHED */
return 0x43219876;
}
+
+#endif /* weight.h */
diff --git a/locale/weightwc.h b/locale/weightwc.h
index 8f047e3..0f70b00 100644
--- a/locale/weightwc.h
+++ b/locale/weightwc.h
@@ -16,10 +16,15 @@
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
+#ifndef _WEIGHTWC_H_
+#define _WEIGHTWC_H_ 1
+
/* Find index of weight. */
-auto inline int32_t
-__attribute ((always_inline))
-findidx (const wint_t **cpp, size_t len)
+static inline int32_t __attribute__ ((always_inline))
+findidx (const int32_t *table,
+ const int32_t *indirect,
+ const wint_t *extra,
+ const wint_t **cpp, size_t len)
{
wint_t ch = *(*cpp)++;
int32_t i = __collidx_table_lookup ((const char *) table, ch);
@@ -109,3 +114,5 @@ findidx (const wint_t **cpp, size_t len)
/* NOTREACHED */
return 0x43219876;
}
+
+#endif /* weightwc.h */
diff --git a/posix/fnmatch.c b/posix/fnmatch.c
index c330a12..85a6ec2 100644
--- a/posix/fnmatch.c
+++ b/posix/fnmatch.c
@@ -221,6 +221,8 @@ __wcschrnul (s, c)
# define MEMCHR(S, C, N) memchr (S, C, N)
# define STRCOLL(S1, S2) strcoll (S1, S2)
# define WIDE_CHAR_VERSION 0
+# include <locale/weight.h>
+# define FINDIDX findidx
# include "fnmatch_loop.c"
@@ -246,6 +248,12 @@ __wcschrnul (s, c)
# define MEMCHR(S, C, N) wmemchr (S, C, N)
# define STRCOLL(S1, S2) wcscoll (S1, S2)
# define WIDE_CHAR_VERSION 1
+/* Change the name the header defines so it doesn't conflict with
+ the <locale/weight.h> version included above. */
+# define findidx findidxwc
+# include <locale/weightwc.h>
+# undef findidx
+# define FINDIDX findidxwc
# undef IS_CHAR_CLASS
/* We have to convert the wide character string in a multibyte string. But
diff --git a/posix/fnmatch_loop.c b/posix/fnmatch_loop.c
index 1957397..db6d9d7 100644
--- a/posix/fnmatch_loop.c
+++ b/posix/fnmatch_loop.c
@@ -376,7 +376,7 @@ FCT (pattern, string, string_end, no_leading_period, flags, ends, alloca_used)
const int32_t *table;
# if WIDE_CHAR_VERSION
const int32_t *weights;
- const int32_t *extra;
+ const wint_t *extra;
# else
const unsigned char *weights;
const unsigned char *extra;
@@ -385,19 +385,12 @@ FCT (pattern, string, string_end, no_leading_period, flags, ends, alloca_used)
int32_t idx;
const UCHAR *cp = (const UCHAR *) str;
- /* This #include defines a local function! */
-# if WIDE_CHAR_VERSION
-# include <locale/weightwc.h>
-# else
-# include <locale/weight.h>
-# endif
-
# if WIDE_CHAR_VERSION
table = (const int32_t *)
_NL_CURRENT (LC_COLLATE, _NL_COLLATE_TABLEWC);
weights = (const int32_t *)
_NL_CURRENT (LC_COLLATE, _NL_COLLATE_WEIGHTWC);
- extra = (const int32_t *)
+ extra = (const wint_t *)
_NL_CURRENT (LC_COLLATE, _NL_COLLATE_EXTRAWC);
indirect = (const int32_t *)
_NL_CURRENT (LC_COLLATE, _NL_COLLATE_INDIRECTWC);
@@ -412,7 +405,7 @@ FCT (pattern, string, string_end, no_leading_period, flags, ends, alloca_used)
_NL_CURRENT (LC_COLLATE, _NL_COLLATE_INDIRECTMB);
# endif
- idx = findidx (&cp, 1);
+ idx = FINDIDX (table, indirect, extra, &cp, 1);
if (idx != 0)
{
/* We found a table entry. Now see whether the
@@ -422,7 +415,8 @@ FCT (pattern, string, string_end, no_leading_period, flags, ends, alloca_used)
int32_t idx2;
const UCHAR *np = (const UCHAR *) n;
- idx2 = findidx (&np, string_end - n);
+ idx2 = FINDIDX (table, indirect, extra,
+ &np, string_end - n);
if (idx2 != 0
&& (idx >> 24) == (idx2 >> 24)
&& len == weights[idx2 & 0xffffff])
@@ -1277,3 +1271,4 @@ EXT (INT opt, const CHAR *pattern, const CHAR *string, const CHAR *string_end,
#undef L
#undef BTOWC
#undef WIDE_CHAR_VERSION
+#undef FINDIDX
diff --git a/posix/regcomp.c b/posix/regcomp.c
index 8f2747b..897fe27 100644
--- a/posix/regcomp.c
+++ b/posix/regcomp.c
@@ -19,6 +19,10 @@
#include <stdint.h>
+#ifdef _LIBC
+# include <locale/weight.h>
+#endif
+
static reg_errcode_t re_compile_internal (regex_t *preg, const char * pattern,
size_t length, reg_syntax_t syntax);
static void re_compile_fastmap_iter (regex_t *bufp,
@@ -3426,8 +3430,6 @@ build_equiv_class (bitset_t sbcset, const unsigned char *name)
int32_t idx1, idx2;
unsigned int ch;
size_t len;
- /* This #include defines a local function! */
-# include <locale/weight.h>
/* Calculate the index for equivalence class. */
cp = name;
table = (const int32_t *) _NL_CURRENT (LC_COLLATE, _NL_COLLATE_TABLEMB);
@@ -3437,7 +3439,7 @@ build_equiv_class (bitset_t sbcset, const unsigned char *name)
_NL_COLLATE_EXTRAMB);
indirect = (const int32_t *) _NL_CURRENT (LC_COLLATE,
_NL_COLLATE_INDIRECTMB);
- idx1 = findidx (&cp, -1);
+ idx1 = findidx (table, indirect, extra, &cp, -1);
if (BE (idx1 == 0 || *cp != '\0', 0))
/* This isn't a valid character. */
return REG_ECOLLATE;
@@ -3448,7 +3450,7 @@ build_equiv_class (bitset_t sbcset, const unsigned char *name)
{
char_buf[0] = ch;
cp = char_buf;
- idx2 = findidx (&cp, 1);
+ idx2 = findidx (table, indirect, extra, &cp, 1);
/*
idx2 = table[ch];
*/
diff --git a/posix/regex_internal.h b/posix/regex_internal.h
index 4bbf6a8..d1ed3dc 100644
--- a/posix/regex_internal.h
+++ b/posix/regex_internal.h
@@ -733,6 +733,10 @@ re_string_wchar_at (const re_string_t *pstr, int idx)
}
# ifndef NOT_IN_libc
+# ifdef _LIBC
+# include <locale/weight.h>
+# endif
+
static int
internal_function __attribute__ ((pure, unused))
re_string_elem_size_at (const re_string_t *pstr, int idx)
@@ -740,7 +744,6 @@ re_string_elem_size_at (const re_string_t *pstr, int idx)
# ifdef _LIBC
const unsigned char *p, *extra;
const int32_t *table, *indirect;
-# include <locale/weight.h>
uint_fast32_t nrules = _NL_CURRENT_WORD (LC_COLLATE, _NL_COLLATE_NRULES);
if (nrules != 0)
@@ -751,7 +754,7 @@ re_string_elem_size_at (const re_string_t *pstr, int idx)
indirect = (const int32_t *) _NL_CURRENT (LC_COLLATE,
_NL_COLLATE_INDIRECTMB);
p = pstr->mbs + idx;
- findidx (&p, pstr->len - idx);
+ findidx (table, indirect, extra, &p, pstr->len - idx);
return p - pstr->mbs - idx;
}
else
diff --git a/posix/regexec.c b/posix/regexec.c
index 7032da7..c840b38 100644
--- a/posix/regexec.c
+++ b/posix/regexec.c
@@ -3749,6 +3749,10 @@ group_nodes_into_DFAstates (const re_dfa_t *dfa, const re_dfastate_t *state,
one collating element like '.', '[a-z]', opposite to the other nodes
can only accept one byte. */
+# ifdef _LIBC
+# include <locale/weight.h>
+# endif
+
static int
internal_function
check_node_accept_bytes (const re_dfa_t *dfa, int node_idx,
@@ -3868,8 +3872,6 @@ check_node_accept_bytes (const re_dfa_t *dfa, int node_idx,
const int32_t *table, *indirect;
const unsigned char *weights, *extra;
const char *collseqwc;
- /* This #include defines a local function! */
-# include <locale/weight.h>
/* match with collating_symbol? */
if (cset->ncoll_syms)
@@ -3925,7 +3927,7 @@ check_node_accept_bytes (const re_dfa_t *dfa, int node_idx,
_NL_CURRENT (LC_COLLATE, _NL_COLLATE_EXTRAMB);
indirect = (const int32_t *)
_NL_CURRENT (LC_COLLATE, _NL_COLLATE_INDIRECTMB);
- int32_t idx = findidx (&cp, elem_len);
+ int32_t idx = findidx (table, indirect, extra, &cp, elem_len);
if (idx > 0)
for (i = 0; i < cset->nequiv_classes; ++i)
{
diff --git a/string/strcoll_l.c b/string/strcoll_l.c
index 10ce4a6..d4f42a3 100644
--- a/string/strcoll_l.c
+++ b/string/strcoll_l.c
@@ -41,6 +41,7 @@
#define CONCAT1(a,b) a##b
#include "../locale/localeinfo.h"
+#include WEIGHT_H
/* Track status while looking for sequences in a string. */
typedef struct
@@ -152,7 +153,6 @@ get_next_seq (coll_seq *seq, int nrules, const unsigned char *rulesets,
const USTRING_TYPE *weights, const int32_t *table,
const USTRING_TYPE *extra, const int32_t *indirect)
{
-#include WEIGHT_H
size_t val = seq->val = 0;
int len = seq->len;
size_t backw_stop = seq->backw_stop;
@@ -194,7 +194,7 @@ get_next_seq (coll_seq *seq, int nrules, const unsigned char *rulesets,
while (*us != L('\0'))
{
- int32_t tmp = findidx (&us, -1);
+ int32_t tmp = findidx (table, indirect, extra, &us, -1);
rulearr[idxmax] = tmp >> 24;
idxarr[idxmax] = tmp & 0xffffff;
idxcnt = idxmax++;
@@ -242,7 +242,6 @@ get_next_seq_nocache (coll_seq *seq, int nrules, const unsigned char *rulesets,
const USTRING_TYPE *extra, const int32_t *indirect,
int pass)
{
-#include WEIGHT_H
size_t val = seq->val = 0;
int len = seq->len;
size_t backw_stop = seq->backw_stop;
@@ -285,7 +284,7 @@ get_next_seq_nocache (coll_seq *seq, int nrules, const unsigned char *rulesets,
us = seq->back_us;
while (i < backw)
{
- int32_t tmp = findidx (&us, -1);
+ int32_t tmp = findidx (table, indirect, extra, &us, -1);
idx = tmp & 0xffffff;
i++;
}
@@ -300,7 +299,7 @@ get_next_seq_nocache (coll_seq *seq, int nrules, const unsigned char *rulesets,
while (*us != L('\0'))
{
- int32_t tmp = findidx (&us, -1);
+ int32_t tmp = findidx (table, indirect, extra, &us, -1);
unsigned char rule = tmp >> 24;
prev_idx = idx;
idx = tmp & 0xffffff;
diff --git a/string/strxfrm_l.c b/string/strxfrm_l.c
index 04b9338..2d3f1bd 100644
--- a/string/strxfrm_l.c
+++ b/string/strxfrm_l.c
@@ -41,6 +41,7 @@
#define CONCAT1(a,b) a##b
#include "../locale/localeinfo.h"
+#include WEIGHT_H
#ifndef WIDE_CHAR_VERSION
@@ -104,8 +105,6 @@ STRXFRM (STRING_TYPE *dest, const STRING_TYPE *src, size_t n, __locale_t l)
size_t idxcnt;
int use_malloc;
-#include WEIGHT_H
-
if (nrules == 0)
{
if (n != 0)
@@ -174,7 +173,7 @@ STRXFRM (STRING_TYPE *dest, const STRING_TYPE *src, size_t n, __locale_t l)
idxmax = 0;
do
{
- int32_t tmp = findidx (&usrc, -1);
+ int32_t tmp = findidx (table, indirect, extra, &usrc, -1);
rulearr[idxmax] = tmp >> 24;
idxarr[idxmax] = tmp & 0xffffff;
-----------------------------------------------------------------------
Summary of changes:
ChangeLog | 77 +++++
NEWS | 8 +-
locale/weight.h | 13 +-
locale/weightwc.h | 13 +-
localedata/sort-test.sh | 7 +
localedata/xfrm-test.c | 52 +++-
posix/fnmatch.c | 8 +
posix/fnmatch_loop.c | 17 +-
posix/regcomp.c | 10 +-
posix/regex_internal.h | 7 +-
posix/regexec.c | 8 +-
stdio-common/tst-sscanf.c | 33 +++
stdio-common/vfscanf.c | 12 +-
string/strcoll_l.c | 9 +-
string/strxfrm_l.c | 491 +++++++++++++++++++++++++-------
sysdeps/x86_64/multiarch/init-arch.c | 9 +-
sysdeps/x86_64/multiarch/init-arch.h | 4 +
sysdeps/x86_64/multiarch/memcpy.S | 2 +-
sysdeps/x86_64/multiarch/memcpy_chk.S | 2 +-
sysdeps/x86_64/multiarch/memmove.c | 2 +-
sysdeps/x86_64/multiarch/memmove_chk.c | 2 +-
sysdeps/x86_64/multiarch/mempcpy.S | 2 +-
sysdeps/x86_64/multiarch/mempcpy_chk.S | 2 +-
23 files changed, 642 insertions(+), 148 deletions(-)
hooks/post-receive
--
GNU C Library master sources