blob: 600638c867b722f549328c3cef3cc20a601b07f3 [file] [log] [blame]
/*
* Copyright © 2014 Canonical Limited
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with this library; if not, see <http://www.gnu.org/licenses/>.
*
* Author: Ryan Lortie <desrt@desrt.ca>
*/
#include <config.h>
#include "gstrfuncs.h"
#include <glib.h>
#include <locale.h>
#include <stdlib.h>
#include <string.h>
struct mapping_entry
{
guint16 src;
guint16 ascii;
};
struct mapping_range
{
guint16 start;
guint16 length;
};
struct locale_entry
{
guint8 name_offset;
guint8 item_id;
};
#include "gtranslit-data.h"
#define get_src_char(array, encoded, index) ((encoded & 0x8000) ? (array)[((encoded) & 0xfff) + index] : encoded)
#define get_length(encoded) ((encoded & 0x8000) ? ((encoded & 0x7000) >> 12) : 1)
#if G_BYTE_ORDER == G_BIG_ENDIAN
#define get_ascii_item(array, encoded) ((encoded & 0x8000) ? &(array)[(encoded) & 0xfff] : (gpointer) (((char *) &(encoded)) + 1))
#else
#define get_ascii_item(array, encoded) ((encoded & 0x8000) ? &(array)[(encoded) & 0xfff] : (gpointer) &(encoded))
#endif
static const gchar * lookup_in_item (guint item_id,
const gunichar *key,
gint *result_len,
gint *key_consumed);
static gint
compare_mapping_entry (gconstpointer user_data,
gconstpointer data)
{
const struct mapping_entry *entry = data;
const gunichar *key = user_data;
gunichar src_0;
G_STATIC_ASSERT(MAX_KEY_SIZE == 2);
src_0 = get_src_char (src_table, entry->src, 0);
if (key[0] > src_0)
return 1;
else if (key[0] < src_0)
return -1;
if (get_length (entry->src) > 1)
{
gunichar src_1;
src_1 = get_src_char (src_table, entry->src, 1);
if (key[1] > src_1)
return 1;
else if (key[1] < src_1)
return -1;
}
else if (key[1])
return 1;
return 0;
}
static const gchar *
lookup_in_mapping (const struct mapping_entry *mapping,
gint mapping_size,
const gunichar *key,
gint *result_len,
gint *key_consumed)
{
const struct mapping_entry *hit;
hit = bsearch (key, mapping, mapping_size, sizeof (struct mapping_entry), compare_mapping_entry);
if (hit == NULL)
return NULL;
*key_consumed = get_length (hit->src);
*result_len = get_length (hit->ascii);
return get_ascii_item(ascii_table, hit->ascii);
}
static const gchar *
lookup_in_chain (const guint8 *chain,
const gunichar *key,
gint *result_len,
gint *key_consumed)
{
const gchar *result;
while (*chain != 0xff)
{
result = lookup_in_item (*chain, key, result_len, key_consumed);
if (result)
return result;
chain++;
}
return NULL;
}
static const gchar *
lookup_in_item (guint item_id,
const gunichar *key,
gint *result_len,
gint *key_consumed)
{
if (item_id & 0x80)
{
const guint8 *chain = chains_table + chain_starts[item_id & 0x7f];
return lookup_in_chain (chain, key, result_len, key_consumed);
}
else
{
const struct mapping_range *range = &mapping_ranges[item_id];
return lookup_in_mapping (mappings_table + range->start, range->length, key, result_len, key_consumed);
}
}
static gint
compare_locale_entry (gconstpointer user_data,
gconstpointer data)
{
const struct locale_entry *entry = data;
const gchar *key = user_data;
return strcmp (key, &locale_names[entry->name_offset]);
}
static gboolean
lookup_item_id_for_one_locale (const gchar *key,
guint *item_id)
{
const struct locale_entry *hit;
hit = bsearch (key, locale_index, G_N_ELEMENTS (locale_index), sizeof (struct locale_entry), compare_locale_entry);
if (hit == NULL)
return FALSE;
*item_id = hit->item_id;
return TRUE;
}
static guint
lookup_item_id_for_locale (const gchar *locale)
{
gchar key[MAX_LOCALE_NAME + 1];
const gchar *language;
guint language_len;
const gchar *territory = NULL;
guint territory_len = 0;
const gchar *modifier = NULL;
guint modifier_len = 0;
const gchar *next_char;
guint id;
/* As per POSIX, a valid locale looks like:
*
* language[_territory][.codeset][@modifier]
*/
language = locale;
language_len = strcspn (language, "_.@");
next_char = language + language_len;
if (*next_char == '_')
{
territory = next_char;
territory_len = strcspn (territory + 1, "_.@") + 1;
next_char = territory + territory_len;
}
if (*next_char == '.')
{
const gchar *codeset;
guint codeset_len;
codeset = next_char;
codeset_len = strcspn (codeset + 1, "_.@") + 1;
next_char = codeset + codeset_len;
}
if (*next_char == '@')
{
modifier = next_char;
modifier_len = strcspn (modifier + 1, "_.@") + 1;
next_char = modifier + modifier_len;
}
/* What madness is this? */
if (language_len == 0 || *next_char)
return default_item_id;
/* We are not interested in codeset.
*
* For this locale:
*
* aa_BB@cc
*
* try in this order:
*
* Note: we have no locales of the form aa_BB@cc in the database.
*
* 1. aa@cc
* 2. aa_BB
* 3. aa
*/
/* 1. */
if (modifier_len && language_len + modifier_len <= MAX_LOCALE_NAME)
{
memcpy (key, language, language_len);
memcpy (key + language_len, modifier, modifier_len);
key[language_len + modifier_len] = '\0';
if (lookup_item_id_for_one_locale (key, &id))
return id;
}
/* 2. */
if (territory_len && language_len + territory_len <= MAX_LOCALE_NAME)
{
memcpy (key, language, language_len);
memcpy (key + language_len, territory, territory_len);
key[language_len + territory_len] = '\0';
if (lookup_item_id_for_one_locale (key, &id))
return id;
}
/* 3. */
if (language_len <= MAX_LOCALE_NAME)
{
memcpy (key, language, language_len);
key[language_len] = '\0';
if (lookup_item_id_for_one_locale (key, &id))
return id;
}
return default_item_id;
}
static guint
get_default_item_id (void)
{
static guint item_id;
static gboolean done;
/* Doesn't need to be locked -- no harm in doing it twice. */
if (!done)
{
const gchar *locale;
locale = setlocale (LC_CTYPE, NULL);
item_id = lookup_item_id_for_locale (locale);
done = TRUE;
}
return item_id;
}
/**
* g_str_to_ascii:
* @str: a string, in UTF-8
* @from_locale: (nullable): the source locale, if known
*
* Transliterate @str to plain ASCII.
*
* For best results, @str should be in composed normalised form.
*
* This function performs a reasonably good set of character
* replacements. The particular set of replacements that is done may
* change by version or even by runtime environment.
*
* If the source language of @str is known, it can used to improve the
* accuracy of the translation by passing it as @from_locale. It should
* be a valid POSIX locale string (of the form
* "language[_territory][.codeset][@modifier]").
*
* If @from_locale is %NULL then the current locale is used.
*
* If you want to do translation for no specific locale, and you want it
* to be done independently of the currently locale, specify "C" for
* @from_locale.
*
* Returns: a string in plain ASCII
*
* Since: 2.40
**/
gchar *
g_str_to_ascii (const gchar *str,
const gchar *from_locale)
{
GString *result;
guint item_id;
g_return_val_if_fail (str != NULL, NULL);
if (g_str_is_ascii (str))
return g_strdup (str);
if (from_locale)
item_id = lookup_item_id_for_locale (from_locale);
else
item_id = get_default_item_id ();
result = g_string_sized_new (strlen (str));
while (*str)
{
/* We only need to transliterate non-ASCII values... */
if (*str & 0x80)
{
gunichar key[MAX_KEY_SIZE];
const gchar *r;
gint consumed;
gint r_len;
gunichar c;
G_STATIC_ASSERT(MAX_KEY_SIZE == 2);
c = g_utf8_get_char (str);
/* This is where it gets evil...
*
* We know that MAX_KEY_SIZE is 2. We also know that we
* only want to try another character if it's non-ascii.
*/
str = g_utf8_next_char (str);
key[0] = c;
if (*str & 0x80)
key[1] = g_utf8_get_char (str);
else
key[1] = 0;
r = lookup_in_item (item_id, key, &r_len, &consumed);
/* If we failed to map two characters, try again with one.
*
* gconv behaviour is a bit weird here -- it seems to
* depend in the randomness of the binary search and the
* size of the input buffer as to what result we get here.
*
* Doing it this way is more work, but should be
* more-correct.
*/
if (r == NULL && key[1])
{
key[1] = 0;
r = lookup_in_item (item_id, key, &r_len, &consumed);
}
if (r != NULL)
{
g_string_append_len (result, r, r_len);
if (consumed == 2)
/* If it took both then skip again */
str = g_utf8_next_char (str);
}
else /* no match found */
g_string_append_c (result, '?');
}
else if (*str & 0x80) /* Out-of-range non-ASCII case */
{
g_string_append_c (result, '?');
str = g_utf8_next_char (str);
}
else /* ASCII case */
g_string_append_c (result, *str++);
}
return g_string_free (result, FALSE);
}