[PATCH] Several fixes for localedef
Denis Barbier
barbier@debian.org
Fri Feb 25 21:37:00 GMT 2005
Hi,
here are several patches against locale/programs/ld-collate.c to fix
bugs or propose enhancements to localedef. They have to be applied
sequentially; I will provide modified patches if only some of them
are applied. All these patches apply against ld-collate.c and thus
are about LC_COLLATE section of locale data files.
They have been tested as best as I could for several weeks now, and
should not produce any regression.
* patch-1-keywords_ordering.diff [BZ #690]
Localedef is currently very strict on its keyword ordering.
As a result, one can not add new scripts (am_ET uses an ugly hack so
that state != 2 after 'copy'), collating-element and collating-symbol
are accepted after 'copy' but not symbol-equivalence, etc. which
sometimes prevent tailoring of iso14651_t1.
This patch is also needed for the following one:
* patch-2-toggles.diff [BZ #686]
Toggle switches are defined in locale/programs/locfile-kw.h
but not implemented. Here is a patch to let localedef handle
'define', 'undef', 'ifdef', 'else' and 'endif' keywords.
This is very useful e.g. for locales which sort uppercase before
lowercase letters, or do not want backward direction in level 2.
Iso14651_t1 can then be customized like
ifdef LATIN_FORWARD
order_start <LATIN>;forward;forward;forward;forward,position
else
order_start <LATIN>;forward;backward;forward;forward,position
endif
so that these locales can define specific keywords and then
'copy "iso14651_t1"' instead of copying its contents.
* patch-3-fix_rules.diff [BZ #645]
Localedef does not respect rules definition (after order_start
keyword), as is explained in depth in #645. This breaks French
collation, but also potentially any locale which has a rule
different from forward;forward;forward;forward.
* patch-4-fix_complex_collation.diff [BZ #368]
With many collation elements, localedef enters an endless loop
because it tries to add an element to a full table and never fails.
The table size was artificially limited to 257 elements, but this
limitation can be removed. This fix is needed for the Dzongkha
language.
2005-02-24 Denis Barbier <barbier@debian.org>
[BZ #690]
* locale/programs/ld-collate.c (collate_read): Relax checks on
keywords ordering.
[BZ #686]
* locale/programs/ld-collate.c (collate_read): Implement keywords
for toggles which were already defined in locfile-kw.h.
[BZ #645]
* locale/programs/ld-collate.c (collate_finish): Fix handling of
section rules.
* locale/programs/ld-collate.c (collate_read): Likewise.
[BZ #368]
* locale/programs/ld-collate.c (collate_output): elem_size does
not need to be truncated at 257.
Denis
-------------- next part --------------
Index: libc/locale/programs/ld-collate.c
===================================================================
--- libc.orig/locale/programs/ld-collate.c
+++ libc/locale/programs/ld-collate.c
@@ -2633,7 +2633,6 @@
if (nowtok == tok_copy)
{
- state = 2;
now = lr_token (ldfile, charmap, result, NULL, verbose);
if (now->tok != tok_string)
{
@@ -2704,14 +2703,23 @@
switch (nowtok)
{
case tok_copy:
- /* Allow copying other locales. */
+ /* Ignore the rest of the line if we don't need the input of
+ this line. */
+ if (ignore_content)
+ {
+ lr_ignore_rest (ldfile, 0);
+ break;
+ }
+
now = lr_token (ldfile, charmap, result, NULL, verbose);
if (now->tok != tok_string)
goto err_label;
- if (! ignore_content)
- load_locale (LC_COLLATE, now->val.str.startmb, repertoire_name,
- charmap, result);
+ if (state == 1 || state == 3 || state == 5)
+ goto err_label;
+
+ load_locale (LC_COLLATE, now->val.str.startmb, repertoire_name,
+ charmap, result);
lr_ignore_rest (ldfile, 1);
break;
@@ -2725,9 +2733,6 @@
break;
}
- if (state != 0)
- goto err_label;
-
arg = lr_token (ldfile, charmap, result, NULL, verbose);
if (arg->tok != tok_number)
goto err_label;
@@ -2748,7 +2753,7 @@
break;
}
- if (state != 0)
+ if (state == 1 || state == 3 || state == 5)
goto err_label;
arg = lr_token (ldfile, charmap, result, repertoire, verbose);
@@ -2795,7 +2800,7 @@
break;
}
- if (state != 0 && state != 2)
+ if (state == 1 || state == 3 || state == 5)
goto err_label;
arg = lr_token (ldfile, charmap, result, repertoire, verbose);
@@ -2864,7 +2869,7 @@
break;
}
- if (state != 0 && state != 2)
+ if (state == 1 || state == 3 || state == 5)
goto err_label;
arg = lr_token (ldfile, charmap, result, repertoire, verbose);
@@ -3012,7 +3017,7 @@
break;
}
- if (state != 0)
+ if (state == 1 || state == 3 || state == 5)
goto err_label;
arg = lr_token (ldfile, charmap, result, repertoire, verbose);
@@ -3128,7 +3133,7 @@
break;
}
- if (state != 0 && state != 1)
+ if (state == 3 || state == 5)
goto err_label;
state = 1;
@@ -3296,8 +3301,12 @@
was_ellipsis = tok_none;
}
}
- else if (state != 2 && state != 3)
- goto err_label;
+ else if (state == 5)
+ {
+ WITH_CUR_LOCALE (error (0, 0, _("\
+%s: missing `reorder-sections-end' keyword"), "LC_COLLATE"));
+ state = 6;
+ }
state = 3;
arg = lr_token (ldfile, charmap, result, repertoire, verbose);
@@ -3429,8 +3438,6 @@
%s: missing `reorder-end' keyword"), "LC_COLLATE"));
state = 4;
}
- else if (state != 2 && state != 4)
- goto err_label;
state = 5;
/* Get the name of the sections we are adding after. */
@@ -3767,7 +3774,7 @@
/* Next we assume `LC_COLLATE'. */
if (!ignore_content)
{
- if (state == 0)
+ if (state == 0 && nrules == 0)
/* We must either see a copy statement or have
ordering values. */
lr_error (ldfile,
-------------- next part --------------
Index: libc/locale/programs/ld-collate.c
===================================================================
--- libc.orig/locale/programs/ld-collate.c
+++ libc/locale/programs/ld-collate.c
@@ -161,6 +161,24 @@
size_t line;
};
+/* Data type for toggles. */
+struct toggle_list_t;
+
+struct toggle_list_t
+{
+ const char *name;
+
+ /* Predecessor in the list. */
+ struct toggle_list_t *last;
+
+ /* This flag is set when a keyword is undefined. */
+ int is_undefined;
+
+ /* Where does the branch come from. */
+ const char *file;
+ size_t line;
+};
+
/* Sparse table of struct element_t *. */
#define TABLE wchead_table
#define ELEMENT struct element_t *
@@ -214,6 +232,12 @@
/* This value is used when handling ellipsis. */
struct element_t ellipsis_weight;
+ /* Known keywords. */
+ struct toggle_list_t *defined_keywords;
+
+ /* This is a stack of toggle branches. */
+ struct toggle_list_t *flow_control;
+
/* Known collating elements. */
hash_table elem_table;
@@ -1456,6 +1480,56 @@
}
+static struct token *
+flow_skip (struct linereader *ldfile, const struct charmap_t *charmap,
+ struct locale_collate_t *collate)
+{
+ int level = 0;
+ struct token *now;
+ enum token_t nowtok;
+ while (1)
+ {
+ lr_ignore_rest (ldfile, 0);
+ now = lr_token (ldfile, charmap, NULL, NULL, 0);
+ nowtok = now->tok;
+ if (nowtok == tok_eof)
+ break;
+ else if (nowtok == tok_ifdef || nowtok == tok_ifndef)
+ ++level ;
+ else if (nowtok == tok_else)
+ {
+ if (strcmp (collate->flow_control->name, "else") == 0)
+ lr_error (ldfile,
+ _("%s: `else' statement at `%s:%d' cannot be followed by another `else' statement"),
+ "LC_COLLATE", collate->flow_control->name, collate->flow_control->line);
+ if (level == 0)
+ {
+ collate->flow_control->name = "else";
+ collate->flow_control->file = ldfile->fname;
+ collate->flow_control->line = ldfile->lineno;
+ break;
+ }
+ }
+ else if (nowtok == tok_endif)
+ {
+ if (level == 0)
+ {
+ collate->flow_control = collate->flow_control->last;
+ break;
+ }
+ --level ;
+ }
+ }
+ if (nowtok == tok_eof)
+ WITH_CUR_LOCALE (error (0, 0, _("\
+%s: unterminated `%s' flow control beginning at %s:%d"),
+ "LC_COLLATE", collate->flow_control->name,
+ collate->flow_control->file,
+ collate->flow_control->line));
+ return now;
+}
+
+
static void
collate_startup (struct linereader *ldfile, struct localedef_t *locale,
struct localedef_t *copy_locale, int ignore_content)
@@ -3770,6 +3844,205 @@
repertoire, result, nowtok);
break;
+ case tok_define:
+ /* Ignore the rest of the line if we don't need the input of
+ this line. */
+ if (ignore_content)
+ {
+ lr_ignore_rest (ldfile, 0);
+ break;
+ }
+
+ arg = lr_token (ldfile, charmap, result, NULL, verbose);
+ if (arg->tok != tok_ident)
+ goto err_label;
+ else
+ {
+ struct toggle_list_t *runp = collate->defined_keywords;
+ char *name;
+
+ while (runp != NULL)
+ if (strncmp (runp->name, arg->val.str.startmb,
+ arg->val.str.lenmb) == 0
+ && runp->name[arg->val.str.lenmb] == '\0')
+ break;
+ else
+ runp = runp->last;
+
+ if (runp != NULL && runp->is_undefined == 0)
+ {
+ lr_ignore_rest (ldfile, 0);
+ break;
+ }
+
+ if (runp == NULL)
+ {
+ runp = (struct toggle_list_t *) xcalloc (1, sizeof (*runp));
+ runp->last = collate->defined_keywords;
+ collate->defined_keywords = runp;
+ }
+ else
+ {
+ free ((char *) runp->name);
+ runp->is_undefined = 0;
+ }
+
+ name = (char *) xmalloc (arg->val.str.lenmb + 1);
+ memcpy (name, arg->val.str.startmb, arg->val.str.lenmb);
+ name[arg->val.str.lenmb] = '\0';
+ runp->name = name;
+ }
+ lr_ignore_rest (ldfile, 1);
+ break;
+
+ case tok_undef:
+ /* Ignore the rest of the line if we don't need the input of
+ this line. */
+ if (ignore_content)
+ {
+ lr_ignore_rest (ldfile, 0);
+ break;
+ }
+
+ arg = lr_token (ldfile, charmap, result, NULL, verbose);
+ if (arg->tok != tok_ident)
+ goto err_label;
+ else
+ {
+ struct toggle_list_t *runp = collate->defined_keywords;
+ while (runp != NULL)
+ if (strncmp (runp->name, arg->val.str.startmb,
+ arg->val.str.lenmb) == 0
+ && runp->name[arg->val.str.lenmb] == '\0')
+ {
+ runp->is_undefined = 1;
+ break;
+ }
+ else
+ runp = runp->last;
+ }
+ lr_ignore_rest (ldfile, 1);
+ break;
+
+ case tok_ifdef:
+ /* Ignore the rest of the line if we don't need the input of
+ this line. */
+ if (ignore_content)
+ {
+ lr_ignore_rest (ldfile, 0);
+ break;
+ }
+
+ arg = lr_token (ldfile, charmap, result, NULL, verbose);
+ if (arg->tok != tok_ident)
+ goto err_label;
+ else
+ {
+ struct toggle_list_t *runp = collate->defined_keywords;
+ struct toggle_list_t *flow = (struct toggle_list_t *) xcalloc (1, sizeof (*runp));
+ flow->name = "ifdef";
+ flow->file = ldfile->fname;
+ flow->line = ldfile->lineno;
+ flow->last = collate->flow_control;
+ collate->flow_control = flow;
+
+ while (runp != NULL)
+ if (strncmp (runp->name, arg->val.str.startmb,
+ arg->val.str.lenmb) == 0
+ && runp->name[arg->val.str.lenmb] == '\0')
+ break;
+ else
+ runp = runp->last;
+
+ if (runp == NULL)
+ {
+ now = flow_skip(ldfile, charmap, collate);
+ if (now->tok == tok_eof)
+ WITH_CUR_LOCALE (error (0, 0, _("\
+%s: unterminated `%s' flow control"), "LC_COLLATE", collate->flow_control->name));
+ }
+ }
+ lr_ignore_rest (ldfile, 1);
+ break;
+
+ case tok_ifndef:
+ /* Ignore the rest of the line if we don't need the input of
+ this line. */
+ if (ignore_content)
+ {
+ lr_ignore_rest (ldfile, 0);
+ break;
+ }
+
+ arg = lr_token (ldfile, charmap, result, NULL, verbose);
+ if (arg->tok != tok_ident)
+ goto err_label;
+ else
+ {
+ struct toggle_list_t *runp = collate->defined_keywords;
+ struct toggle_list_t *flow = (struct toggle_list_t *) xcalloc (1, sizeof (*runp));
+ flow->name = "ifndef";
+ flow->file = ldfile->fname;
+ flow->line = ldfile->lineno;
+ flow->last = collate->flow_control;
+ collate->flow_control = flow;
+
+ while (runp != NULL)
+ if (strncmp (runp->name, arg->val.str.startmb,
+ arg->val.str.lenmb) == 0
+ && runp->name[arg->val.str.lenmb] == '\0')
+ break;
+ else
+ runp = runp->last;
+
+ if (runp != NULL)
+ {
+ now = flow_skip(ldfile, charmap, collate);
+ if (now->tok == tok_eof)
+ WITH_CUR_LOCALE (error (0, 0, _("\
+%s: unterminated `%s' flow control"), "LC_COLLATE", collate->flow_control->name));
+ }
+ }
+ lr_ignore_rest (ldfile, 1);
+ break;
+
+ case tok_else:
+ /* Ignore the rest of the line if we don't need the input of
+ this line. */
+ if (ignore_content)
+ {
+ lr_ignore_rest (ldfile, 0);
+ break;
+ }
+
+ if (strcmp (collate->flow_control->name, "else") == 0)
+ lr_error (ldfile,
+ _("%s: `else' statement at `%s:%d' cannot be followed by another `else' statement"),
+ "LC_COLLATE", collate->flow_control->name, collate->flow_control->line);
+ collate->flow_control->name = "else";
+ collate->flow_control->file = ldfile->fname;
+ collate->flow_control->line = ldfile->lineno;
+ now = flow_skip(ldfile, charmap, collate);
+ if (now->tok == tok_eof)
+ WITH_CUR_LOCALE (error (0, 0, _("\
+%s: unterminated `%s' flow control"), "LC_COLLATE", collate->flow_control->name));
+ break;
+
+ case tok_endif:
+ /* Ignore the rest of the line if we don't need the input of
+ this line. */
+ if (ignore_content)
+ {
+ lr_ignore_rest (ldfile, 0);
+ break;
+ }
+
+ if (collate->flow_control == NULL)
+ goto err_label;
+ else
+ collate->flow_control = collate->flow_control->last;
+ break;
+
case tok_end:
/* Next we assume `LC_COLLATE'. */
if (!ignore_content)
@@ -3799,6 +4072,12 @@
else if (state == 5)
WITH_CUR_LOCALE (error (0, 0, _("\
%s: missing `reorder-sections-end' keyword"), "LC_COLLATE"));
+ if (collate->flow_control != NULL)
+ WITH_CUR_LOCALE (error (0, 0, _("\
+%s: unterminated `%s' flow control beginning at %s:%d"),
+ "LC_COLLATE", collate->flow_control->name,
+ collate->flow_control->file,
+ collate->flow_control->line));
}
arg = lr_token (ldfile, charmap, result, NULL, verbose);
if (arg->tok == tok_eof)
-------------- next part --------------
Index: libc/locale/programs/ld-collate.c
===================================================================
--- libc.orig/locale/programs/ld-collate.c
+++ libc/locale/programs/ld-collate.c
@@ -1524,6 +1524,7 @@
int i;
int need_undefined = 0;
struct section_list *sect;
+ enum coll_sort_rule *orules;
int ruleidx;
int nr_wide_elems = 0;
@@ -1535,17 +1536,28 @@
"LC_COLLATE"));
return;
}
+ if (nrules == 0)
+ {
+ /* An error message has already been printed:
+ empty category description not allowed. */
+ return;
+ }
+
/* If this assertion is hit change the type in `element_t'. */
assert (nrules <= sizeof (runp->used_in_level) * 8);
/* Make sure that the `position' rule is used either in all sections
or in none. */
+ sect = collate->sections;
+ while (sect != NULL && sect->rules == NULL)
+ sect = sect->next;
+ orules = sect->rules;
for (i = 0; i < nrules; ++i)
for (sect = collate->sections; sect != NULL; sect = sect->next)
if (sect->rules != NULL
&& ((sect->rules[i] & sort_position)
- != (collate->sections->rules[i] & sort_position)))
+ != (orules[i] & sort_position)))
{
WITH_CUR_LOCALE (error (0, 0, _("\
%s: `position' must be used for a specific level in all sections or none"),
@@ -1840,7 +1852,7 @@
while (osect != sect)
if (osect->rules != NULL
- && memcmp (osect->rules, sect->rules, nrules) == 0)
+ && memcmp (osect->rules, sect->rules, nrules * sizeof (*osect->rules)) == 0)
break;
else
osect = osect->next;
@@ -3184,12 +3196,15 @@
{
/* Insert sp in the collate->sections list,
right after collate->current_section. */
- if (collate->current_section == NULL)
+ if (collate->sections == NULL)
+ collate->sections = collate->current_section = sp;
+ else if (collate->current_section == NULL)
collate->current_section = sp;
else
{
sp->next = collate->current_section->next;
collate->current_section->next = sp;
+ collate->current_section = sp;
}
}
@@ -3380,6 +3395,9 @@
no_error = 0;
}
}
+ /* Update current section. */
+ if (collate->cursor != NULL)
+ collate->current_section = collate->cursor->section;
lr_ignore_rest (ldfile, no_error);
}
-------------- next part --------------
Index: libc/locale/programs/ld-collate.c
===================================================================
--- libc.orig/locale/programs/ld-collate.c
+++ libc/locale/programs/ld-collate.c
@@ -2558,14 +2558,15 @@
runp = collate->start;
while (runp != NULL)
{
- if (runp->mbs != NULL && runp->weights != NULL)
+ if (runp->mbs != NULL && runp->weights != NULL && !runp->is_character)
/* Yep, the element really counts. */
++elem_size;
runp = runp->next;
}
+rehash:
/* Add 40% and find the next prime number. */
- elem_size = MIN (next_prime (elem_size * 1.4), 257);
+ elem_size = next_prime (elem_size * 1.4);
/* Allocate the table. Each entry consists of two words: the hash
value and an index in a secondary table which provides the index
@@ -2597,6 +2598,10 @@
idx += iter;
if (idx >= elem_size)
idx -= elem_size;
+ if (idx == (hash % elem_size)) {
+ obstack_free (&extrapool, elem_table);
+ goto rehash;
+ }
}
while (elem_table[idx * 2] != 0);
}
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 189 bytes
Desc: Digital signature
URL: <http://sourceware.org/pipermail/libc-alpha/attachments/20050225/7b775edf/attachment.sig>
More information about the Libc-alpha
mailing list