This is the mail archive of the
libc-alpha@sources.redhat.com
mailing list for the glibc project.
Re: libintl patches (5)
- To: libc-alpha at sources dot redhat dot com
- Subject: Re: libintl patches (5)
- From: Bruno Haible <haible at ilog dot fr>
- Date: Wed, 3 Jan 2001 20:45:34 +0100 (CET)
- References: <14931.32376.45363.873429@honolulu.ilog.fr>
Ulrich Drepper writes:
> Gcc of course supports zero-sized structarray elements.
Sorry, I misunderstood one of your ChangeLog entries.
Anyway, the important part is number (5). Here it comes again,
assuming (4) is not applied.
Bruno
2001-01-01 Bruno Haible <haible@clisp.cons.org>
Finish implementation of plural form handling.
* intl/dcigettext.c (known_translation_t): Rename 'domain' field to
'domainname'. Remove 'plindex' field. Add 'domain' and
'translation_length' fields.
(transcmp): Don't compare 'plindex' fields.
(plural_lookup): New function.
(DCIGETTEXT): Change cache handing in the plural case. Don't call
plural_eval before the translation and its catalog file have been
found. Remove plindex from cache key. Add 'translation_length' and
'domain' to cache result.
(_nl_find_msg): Remove index argument, return length of translation
to the caller instead. Weaken comparison of string lengths, to account
for plural entries. Call iconv() on the entire result string, not
only on the portion needed so far.
* intl/loadinfo.h (_nl_find_msg): Remove index argument, add lengthp
argument.
* intl/loadmsgcat.c (_nl_load_domain): Adapt to _nl_find_msg change.
diff -r -c3 glibc-cvs/intl1/dcigettext.c glibc-cvs/intl2/dcigettext.c
*** glibc-cvs/intl/dcigettext.c Wed Jan 3 15:33:40 2001
--- glibc-cvs/intl/dcigettext.c Wed Jan 3 20:42:07 2001
***************
*** 210,219 ****
struct known_translation_t
{
/* Domain in which to search. */
! char *domain;
!
! /* Plural index. */
! unsigned long int plindex;
/* The category. */
int category;
--- 210,216 ----
struct known_translation_t
{
/* Domain in which to search. */
! char *domainname;
/* The category. */
int category;
***************
*** 221,228 ****
--- 218,229 ----
/* State of the catalog counter at the point the string was found. */
int counter;
+ /* Catalog where the string was found. */
+ struct loaded_l10nfile *domain;
+
/* And finally the translation. */
const char *translation;
+ size_t translation_length;
/* Pointer to the string in question. */
char msgid[ZERO];
***************
*** 253,268 ****
result = strcmp (s1->msgid, s2->msgid);
if (result == 0)
{
! result = strcmp (s1->domain, s2->domain);
if (result == 0)
! {
! result = s1->plindex - s2->plindex;
! if (result == 0)
! /* We compare the category last (though this is the cheapest
! operation) since it is hopefully always the same (namely
! LC_MESSAGES). */
! result = s1->category - s2->category;
! }
}
return result;
--- 254,265 ----
result = strcmp (s1->msgid, s2->msgid);
if (result == 0)
{
! result = strcmp (s1->domainname, s2->domainname);
if (result == 0)
! /* We compare the category last (though this is the cheapest
! operation) since it is hopefully always the same (namely
! LC_MESSAGES). */
! result = s1->category - s2->category;
}
return result;
***************
*** 284,291 ****
struct binding *_nl_domain_bindings;
/* Prototypes for local functions. */
! static unsigned long int plural_eval (struct expression *pexp,
! unsigned long int n) internal_function;
static const char *category_to_name PARAMS ((int category)) internal_function;
static const char *guess_category_value PARAMS ((int category,
const char *categoryname))
--- 281,294 ----
struct binding *_nl_domain_bindings;
/* Prototypes for local functions. */
! static char *plural_lookup PARAMS ((struct loaded_l10nfile *domain,
! unsigned long int n,
! const char *translation,
! size_t translation_len))
! internal_function;
! static unsigned long int plural_eval PARAMS ((struct expression *pexp,
! unsigned long int n))
! internal_function;
static const char *category_to_name PARAMS ((int category)) internal_function;
static const char *guess_category_value PARAMS ((int category,
const char *categoryname))
***************
*** 328,339 ****
#endif /* have alloca */
/* List of blocks allocated for translations. */
! static struct transmem_list
{
struct transmem_list *next;
char data[0];
! } *transmem_list;
/* Names for the libintl functions are a problem. They must not clash
--- 331,347 ----
#endif /* have alloca */
+ #ifdef _LIBC
/* List of blocks allocated for translations. */
! typedef struct transmem_list
{
struct transmem_list *next;
char data[0];
! } transmem_block_t;
! static struct transmem_list *transmem_list;
! #else
! typedef char transmem_block_t;
! #endif
/* Names for the libintl functions are a problem. They must not clash
***************
*** 389,394 ****
--- 397,403 ----
char *dirname, *xdomainname;
char *single_locale;
char *retval;
+ size_t retlen;
int saved_errno;
#if defined HAVE_TSEARCH || defined _LIBC
struct known_translation_t *search;
***************
*** 412,434 ****
#if defined HAVE_TSEARCH || defined _LIBC
msgid_len = strlen (msgid1) + 1;
! if (plural == 0)
{
! /* Try to find the translation among those which we found at
! some time. */
! search = (struct known_translation_t *) alloca (sizeof (*search)
! + msgid_len);
! memcpy (search->msgid, msgid1, msgid_len);
! search->domain = (char *) domainname;
! search->plindex = 0;
! search->category = category;
! foundp = (struct known_translation_t **) tfind (search, &root, transcmp);
! if (foundp != NULL && (*foundp)->counter == _nl_msg_cat_cntr)
! {
! __libc_rwlock_unlock (_nl_state_lock);
! return (char *) (*foundp)->translation;
! }
}
#endif
--- 421,446 ----
#if defined HAVE_TSEARCH || defined _LIBC
msgid_len = strlen (msgid1) + 1;
! /* Try to find the translation among those which we found at
! some time. */
! search =
! (struct known_translation_t *) alloca (sizeof (*search) + msgid_len);
! memcpy (search->msgid, msgid1, msgid_len);
! search->domainname = (char *) domainname;
! search->category = category;
!
! foundp = (struct known_translation_t **) tfind (search, &root, transcmp);
! if (foundp != NULL && (*foundp)->counter == _nl_msg_cat_cntr)
{
! /* Now deal with plural. */
! if (plural)
! retval = plural_lookup ((*foundp)->domain, n, (*foundp)->translation,
! (*foundp)->translation_length);
! else
! retval = (char *) (*foundp)->translation;
! __libc_rwlock_unlock (_nl_state_lock);
! return retval;
}
#endif
***************
*** 563,601 ****
if (domain != NULL)
{
! unsigned long int index = 0;
!
! if (plural != 0)
! {
! const struct loaded_domain *domaindata =
! (const struct loaded_domain *) domain->data;
! index = plural_eval (domaindata->plural, n);
! if (index >= domaindata->nplurals)
! /* This should never happen. It means the plural expression
! and the given maximum value do not match. */
! index = 0;
!
! #if defined HAVE_TSEARCH || defined _LIBC
! /* Try to find the translation among those which we
! found at some time. */
! search = (struct known_translation_t *) alloca (sizeof (*search)
! + msgid_len);
! memcpy (search->msgid, msgid1, msgid_len);
! search->domain = (char *) domainname;
! search->plindex = index;
! search->category = category;
!
! foundp = (struct known_translation_t **) tfind (search, &root,
! transcmp);
! if (foundp != NULL && (*foundp)->counter == _nl_msg_cat_cntr)
! {
! __libc_rwlock_unlock (_nl_state_lock);
! return (char *) (*foundp)->translation;
! }
! #endif
! }
!
! retval = _nl_find_msg (domain, msgid1, index);
if (retval == NULL)
{
--- 575,581 ----
if (domain != NULL)
{
! retval = _nl_find_msg (domain, msgid1, &retlen);
if (retval == NULL)
{
***************
*** 604,618 ****
for (cnt = 0; domain->successor[cnt] != NULL; ++cnt)
{
retval = _nl_find_msg (domain->successor[cnt], msgid1,
! index);
if (retval != NULL)
! break;
}
}
if (retval != NULL)
{
FREE_BLOCKS (block_list);
__set_errno (saved_errno);
#if defined HAVE_TSEARCH || defined _LIBC
--- 584,603 ----
for (cnt = 0; domain->successor[cnt] != NULL; ++cnt)
{
retval = _nl_find_msg (domain->successor[cnt], msgid1,
! &retlen);
if (retval != NULL)
! {
! domain = domain->successor[cnt];
! break;
! }
}
}
if (retval != NULL)
{
+ /* Found the translation of MSGID1 in domain DOMAIN:
+ starting at RETVAL, RETLEN bytes. */
FREE_BLOCKS (block_list);
__set_errno (saved_errno);
#if defined HAVE_TSEARCH || defined _LIBC
***************
*** 626,637 ****
+ domainname_len + 1 - ZERO);
if (newp != NULL)
{
! newp->domain = mempcpy (newp->msgid, msgid1, msgid_len);
! memcpy (newp->domain, domainname, domainname_len + 1);
! newp->plindex = index;
newp->category = category;
newp->counter = _nl_msg_cat_cntr;
newp->translation = retval;
/* Insert the entry in the search tree. */
foundp = (struct known_translation_t **)
--- 611,624 ----
+ domainname_len + 1 - ZERO);
if (newp != NULL)
{
! newp->domainname =
! mempcpy (newp->msgid, msgid1, msgid_len);
! memcpy (newp->domainname, domainname, domainname_len + 1);
newp->category = category;
newp->counter = _nl_msg_cat_cntr;
+ newp->domain = domain;
newp->translation = retval;
+ newp->translation_length = retlen;
/* Insert the entry in the search tree. */
foundp = (struct known_translation_t **)
***************
*** 646,654 ****
--- 633,647 ----
{
/* We can update the existing entry. */
(*foundp)->counter = _nl_msg_cat_cntr;
+ (*foundp)->domain = domain;
(*foundp)->translation = retval;
+ (*foundp)->translation_length = retlen;
}
#endif
+ /* Now deal with plural. */
+ if (plural)
+ retval = plural_lookup (domain, n, retval, retlen);
+
__libc_rwlock_unlock (_nl_state_lock);
return retval;
}
***************
*** 660,673 ****
char *
internal_function
! _nl_find_msg (domain_file, msgid, index)
struct loaded_l10nfile *domain_file;
const char *msgid;
! unsigned long int index;
{
const struct loaded_domain *domain;
size_t act;
char *result;
if (domain_file->decided == 0)
_nl_load_domain (domain_file);
--- 653,667 ----
char *
internal_function
! _nl_find_msg (domain_file, msgid, lengthp)
struct loaded_l10nfile *domain_file;
const char *msgid;
! size_t *lengthp;
{
const struct loaded_domain *domain;
size_t act;
char *result;
+ size_t resultlen;
if (domain_file->decided == 0)
_nl_load_domain (domain_file);
***************
*** 691,707 ****
/* Hash table entry is empty. */
return NULL;
- if (W (domain->must_swap, domain->orig_tab[nstr - 1].length) == len
- && strcmp (msgid,
- domain->data + W (domain->must_swap,
- domain->orig_tab[nstr - 1].offset)) == 0)
- {
- act = nstr - 1;
- goto found;
- }
-
while (1)
{
if (idx >= domain->hash_size - incr)
idx -= domain->hash_size - incr;
else
--- 685,705 ----
/* Hash table entry is empty. */
return NULL;
while (1)
{
+ /* Compare msgid with the original string at index nstr-1.
+ We compare the lengths with >=, not ==, because plural entries
+ are represented by strings with an embedded NUL. */
+ if (W (domain->must_swap, domain->orig_tab[nstr - 1].length) >= len
+ && (strcmp (msgid,
+ domain->data + W (domain->must_swap,
+ domain->orig_tab[nstr - 1].offset))
+ == 0))
+ {
+ act = nstr - 1;
+ goto found;
+ }
+
if (idx >= domain->hash_size - incr)
idx -= domain->hash_size - incr;
else
***************
*** 711,726 ****
if (nstr == 0)
/* Hash table entry is empty. */
return NULL;
-
- if (W (domain->must_swap, domain->orig_tab[nstr - 1].length) == len
- && (strcmp (msgid,
- domain->data + W (domain->must_swap,
- domain->orig_tab[nstr - 1].offset))
- == 0))
- {
- act = nstr - 1;
- goto found;
- }
}
/* NOTREACHED */
}
--- 709,714 ----
***************
*** 756,761 ****
--- 744,750 ----
string to use a different character set, this is the time. */
result = ((char *) domain->data
+ W (domain->must_swap, domain->trans_tab[act].offset));
+ resultlen = W (domain->must_swap, domain->trans_tab[act].length) + 1;
#if defined _LIBC || HAVE_ICONV
if (
***************
*** 772,780 ****
appropriate table with the same structure as the table
of translations in the file, where we can put the pointers
to the converted strings in.
! There is a slight complication with the INDEX: We don't know
! a priori which entries are plural entries. Therefore at any
! moment we can only translate the variants 0 .. INDEX. */
if (domain->conv_tab == NULL
&& ((domain->conv_tab = (char **) calloc (domain->nstrings,
--- 761,770 ----
appropriate table with the same structure as the table
of translations in the file, where we can put the pointers
to the converted strings in.
! There is a slight complication with plural entries. They
! are represented by consecutive NUL terminated strings. We
! handle this case by converting RESULTLEN bytes, including
! NULs. */
if (domain->conv_tab == NULL
&& ((domain->conv_tab = (char **) calloc (domain->nstrings,
***************
*** 787,794 ****
/* Nothing we can do, no more memory. */
goto converted;
! if (domain->conv_tab[act] == NULL
! || *(nls_uint32 *) domain->conv_tab[act] < index)
{
/* We haven't used this string so far, so it is not
translated yet. Do this now. */
--- 777,783 ----
/* Nothing we can do, no more memory. */
goto converted;
! if (domain->conv_tab[act] == NULL)
{
/* We haven't used this string so far, so it is not
translated yet. Do this now. */
***************
*** 796,841 ****
We allocate always larger blocks which get used over
time. This is faster than many small allocations. */
__libc_lock_define_initialized (static, lock)
! #define INITIAL_BLOCK_SIZE 4080
static unsigned char *freemem;
static size_t freemem_size;
- size_t resultlen;
const unsigned char *inbuf;
unsigned char *outbuf;
int malloc_count;
!
! /* Note that we translate (index + 1) consecutive strings at
! once, including the final NUL byte. */
! {
! unsigned long int i = index;
! char *p = result;
! do
! p += strlen (p) + 1;
! while (i-- > 0);
! resultlen = p - result;
! }
__libc_lock_lock (lock);
inbuf = result;
! outbuf = freemem + sizeof (nls_uint32);
malloc_count = 0;
while (1)
{
# ifdef _LIBC
- struct transmem_list *newmem;
size_t non_reversible;
int res;
! if (freemem_size < 4)
goto resize_freemem;
res = __gconv (domain->conv,
&inbuf, inbuf + resultlen,
&outbuf,
! outbuf + freemem_size - sizeof (nls_uint32),
&non_reversible);
if (res == __GCONV_OK || res == __GCONV_EMPTY_INPUT)
--- 785,821 ----
We allocate always larger blocks which get used over
time. This is faster than many small allocations. */
__libc_lock_define_initialized (static, lock)
! # define INITIAL_BLOCK_SIZE 4080
static unsigned char *freemem;
static size_t freemem_size;
const unsigned char *inbuf;
unsigned char *outbuf;
int malloc_count;
! # ifndef _LIBC
! transmem_block_t *transmem_list = NULL;
! # endif
__libc_lock_lock (lock);
inbuf = result;
! outbuf = freemem + sizeof (size_t);
malloc_count = 0;
while (1)
{
+ transmem_block_t *newmem;
# ifdef _LIBC
size_t non_reversible;
int res;
! if (freemem_size < sizeof (size_t))
goto resize_freemem;
res = __gconv (domain->conv,
&inbuf, inbuf + resultlen,
&outbuf,
! outbuf + freemem_size - sizeof (size_t),
&non_reversible);
if (res == __GCONV_OK || res == __GCONV_EMPTY_INPUT)
***************
*** 850,865 ****
inbuf = result;
# else
# if HAVE_ICONV
- # define transmem freemem
const char *inptr = (const char *) inbuf;
size_t inleft = resultlen;
char *outptr = (char *) outbuf;
size_t outleft;
! if (freemem_size < 4)
goto resize_freemem;
! outleft = freemem_size - 4;
if (iconv (domain->conv, &inptr, &inleft, &outptr, &outleft)
!= (size_t) (-1))
{
--- 830,844 ----
inbuf = result;
# else
# if HAVE_ICONV
const char *inptr = (const char *) inbuf;
size_t inleft = resultlen;
char *outptr = (char *) outbuf;
size_t outleft;
! if (freemem_size < sizeof (size_t))
goto resize_freemem;
! outleft = freemem_size - sizeof (size_t);
if (iconv (domain->conv, &inptr, &inleft, &outptr, &outleft)
!= (size_t) (-1))
{
***************
*** 871,900 ****
__libc_lock_unlock (lock);
goto converted;
}
- # else
- # define transmem freemem
# endif
# endif
resize_freemem:
! /* We must allocate a new buffer of resize the old one. */
if (malloc_count > 0)
{
- struct transmem_list *next = transmem_list->next;
-
++malloc_count;
freemem_size = malloc_count * INITIAL_BLOCK_SIZE;
! newmem = (struct transmem_list *) realloc (transmem_list,
! freemem_size);
!
if (newmem != NULL)
! transmem_list = next;
}
else
{
malloc_count = 1;
freemem_size = INITIAL_BLOCK_SIZE;
! newmem = (struct transmem_list *) malloc (freemem_size);
}
if (__builtin_expect (newmem == NULL, 0))
{
--- 850,883 ----
__libc_lock_unlock (lock);
goto converted;
}
# endif
# endif
resize_freemem:
! /* We must allocate a new buffer or resize the old one. */
if (malloc_count > 0)
{
++malloc_count;
freemem_size = malloc_count * INITIAL_BLOCK_SIZE;
! newmem = (transmem_block_t *) realloc (transmem_list,
! freemem_size);
! # ifdef _LIBC
if (newmem != NULL)
! transmem_list = transmem_list->next;
! else
! {
! struct transmem_list *old = transmem_list;
!
! transmem_list = transmem_list->next;
! free (old);
! }
! # endif
}
else
{
malloc_count = 1;
freemem_size = INITIAL_BLOCK_SIZE;
! newmem = (transmem_block_t *) malloc (freemem_size);
}
if (__builtin_expect (newmem == NULL, 0))
{
***************
*** 912,938 ****
freemem = newmem->data;
freemem_size -= offsetof (struct transmem_list, data);
# endif
! outbuf = freemem + sizeof (nls_uint32);
}
/* We have now in our buffer a converted string. Put this
into the table of conversions. */
! *(nls_uint32 *) freemem = index;
domain->conv_tab[act] = freemem;
/* Shrink freemem, but keep it aligned. */
freemem_size -= outbuf - freemem;
freemem = outbuf;
! freemem += freemem_size & (alignof (nls_uint32) - 1);
! freemem_size = freemem_size & ~ (alignof (nls_uint32) - 1);
__libc_lock_unlock (lock);
}
! /* Now domain->conv_tab[act] contains the translation of at least
! the variants 0 .. INDEX. */
! result = domain->conv_tab[act] + sizeof (nls_uint32);
}
converted:
--- 895,925 ----
freemem = newmem->data;
freemem_size -= offsetof (struct transmem_list, data);
+ # else
+ transmem_list = newmem;
+ freemem = newmem;
# endif
! outbuf = freemem + sizeof (size_t);
}
/* We have now in our buffer a converted string. Put this
into the table of conversions. */
! *(size_t *) freemem = outbuf - freemem - sizeof (size_t);
domain->conv_tab[act] = freemem;
/* Shrink freemem, but keep it aligned. */
freemem_size -= outbuf - freemem;
freemem = outbuf;
! freemem += freemem_size & (alignof (size_t) - 1);
! freemem_size = freemem_size & ~ (alignof (size_t) - 1);
__libc_lock_unlock (lock);
}
! /* Now domain->conv_tab[act] contains the translation of all
! the plural variants. */
! result = domain->conv_tab[act] + sizeof (size_t);
! resultlen = *(size_t *) domain->conv_tab[act];
}
converted:
***************
*** 940,965 ****
#endif /* _LIBC || HAVE_ICONV */
! /* Now skip some strings. How much depends on the index passed in. */
while (index-- > 0)
{
#ifdef _LIBC
! result = __rawmemchr (result, '\0');
#else
! result = strchr (result, '\0');
#endif
/* And skip over the NUL byte. */
! ++result;
! }
! return result;
}
/* Function to evaluate the plural expression and return an index value. */
static unsigned long int
internal_function
! plural_eval (struct expression *pexp, unsigned long int n)
{
switch (pexp->operation)
{
--- 927,984 ----
#endif /* _LIBC || HAVE_ICONV */
! *lengthp = resultlen;
! return result;
! }
!
!
! /* Look up a plural variant. */
! static char *
! internal_function
! plural_lookup (domain, n, translation, translation_len)
! struct loaded_l10nfile *domain;
! unsigned long int n;
! const char *translation;
! size_t translation_len;
! {
! struct loaded_domain *domaindata = (struct loaded_domain *) domain->data;
! unsigned long int index;
! const char *p;
!
! index = plural_eval (domaindata->plural, n);
! if (index >= domaindata->nplurals)
! /* This should never happen. It means the plural expression and the
! given maximum value do not match. */
! index = 0;
!
! /* Skip INDEX strings at TRANSLATION. */
! p = translation;
while (index-- > 0)
{
#ifdef _LIBC
! p = __rawmemchr (p, '\0');
#else
! p = strchr (p, '\0');
#endif
/* And skip over the NUL byte. */
! p++;
! if (p >= translation + translation_len)
! /* This should never happen. It means the plural expression
! evaluated to a value larger than the number of variants
! available for MSGID1. */
! return (char *) translation;
! }
! return (char *) p;
}
/* Function to evaluate the plural expression and return an index value. */
static unsigned long int
internal_function
! plural_eval (pexp, n)
! struct expression *pexp;
! unsigned long int n;
{
switch (pexp->operation)
{
diff -r -c3 glibc-cvs/intl1/loadinfo.h glibc-cvs/intl2/loadinfo.h
*** glibc-cvs/intl/loadinfo.h Thu May 4 12:53:42 2000
--- glibc-cvs/intl/loadinfo.h Wed Jan 3 20:40:23 2001
***************
*** 101,107 ****
extern char *_nl_find_msg PARAMS ((struct loaded_l10nfile *domain_file,
! const char *msgid, unsigned long int index))
internal_function;
#endif /* loadinfo.h */
--- 101,107 ----
extern char *_nl_find_msg PARAMS ((struct loaded_l10nfile *domain_file,
! const char *msgid, size_t *lengthp))
internal_function;
#endif /* loadinfo.h */
diff -r -c3 glibc-cvs/intl1/loadmsgcat.c glibc-cvs/intl2/loadmsgcat.c
*** glibc-cvs/intl/loadmsgcat.c Wed Jan 3 20:39:16 2001
--- glibc-cvs/intl/loadmsgcat.c Wed Jan 3 20:40:23 2001
***************
*** 297,303 ****
# endif
#endif
domain->conv_tab = NULL;
! nullentry = _nl_find_msg (domain_file, "", 0);
if (nullentry != NULL)
{
#if defined _LIBC || HAVE_ICONV
--- 297,303 ----
# endif
#endif
domain->conv_tab = NULL;
! nullentry = _nl_find_msg (domain_file, "", &nullentrylen);
if (nullentry != NULL)
{
#if defined _LIBC || HAVE_ICONV