| #! /usr/bin/perl -w |
| |
| # Copyright (C) 1998, 1999 Tom Tromey |
| # Copyright (C) 2001 Red Hat Software |
| |
| # This program is free software; you can redistribute it and/or modify |
| # it under the terms of the GNU General Public License as published by |
| # the Free Software Foundation; either version 2, or (at your option) |
| # any later version. |
| |
| # This program is distributed in the hope that it will be useful, |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| # GNU General Public License for more details. |
| |
| # You should have received a copy of the GNU General Public License |
| # along with this program; if not, see <http://www.gnu.org/licenses/>. |
| |
| # Contributor(s): |
| # Andrew Taylor <andrew.taylor@montage.ca> |
| |
| # gen-unicode-tables.pl - Generate tables for libunicode from Unicode data. |
| # See http://www.unicode.org/Public/UNIDATA/UnicodeCharacterDatabase.html |
| # I consider the output of this program to be unrestricted. Use it as |
| # you will. |
| |
| # FIXME: |
| # * For decomp table it might make sense to use a shift count other |
| # than 8. We could easily compute the perfect shift count. |
| |
| # we use some perl unicode features |
| require 5.006; |
| |
| use bytes; |
| |
| use vars qw($CODE $NAME $CATEGORY $COMBINING_CLASSES $BIDI_CATEGORY $DECOMPOSITION $DECIMAL_VALUE $DIGIT_VALUE $NUMERIC_VALUE $MIRRORED $OLD_NAME $COMMENT $UPPER $LOWER $TITLE $BREAK_CODE $BREAK_CATEGORY $BREAK_NAME $CASE_CODE $CASE_LOWER $CASE_TITLE $CASE_UPPER $CASE_CONDITION); |
| |
| |
| # Names of fields in Unicode data table. |
| $CODE = 0; |
| $NAME = 1; |
| $CATEGORY = 2; |
| $COMBINING_CLASSES = 3; |
| $BIDI_CATEGORY = 4; |
| $DECOMPOSITION = 5; |
| $DECIMAL_VALUE = 6; |
| $DIGIT_VALUE = 7; |
| $NUMERIC_VALUE = 8; |
| $MIRRORED = 9; |
| $OLD_NAME = 10; |
| $COMMENT = 11; |
| $UPPER = 12; |
| $LOWER = 13; |
| $TITLE = 14; |
| |
| # Names of fields in the line break table |
| $BREAK_CODE = 0; |
| $BREAK_PROPERTY = 1; |
| |
| # Names of fields in the SpecialCasing table |
| $CASE_CODE = 0; |
| $CASE_LOWER = 1; |
| $CASE_TITLE = 2; |
| $CASE_UPPER = 3; |
| $CASE_CONDITION = 4; |
| |
| # Names of fields in the CaseFolding table |
| $FOLDING_CODE = 0; |
| $FOLDING_STATUS = 1; |
| $FOLDING_MAPPING = 2; |
| |
| # Map general category code onto symbolic name. |
| %mappings = |
| ( |
| # Normative. |
| 'Lu' => "G_UNICODE_UPPERCASE_LETTER", |
| 'Ll' => "G_UNICODE_LOWERCASE_LETTER", |
| 'Lt' => "G_UNICODE_TITLECASE_LETTER", |
| 'Mn' => "G_UNICODE_NON_SPACING_MARK", |
| 'Mc' => "G_UNICODE_SPACING_MARK", |
| 'Me' => "G_UNICODE_ENCLOSING_MARK", |
| 'Nd' => "G_UNICODE_DECIMAL_NUMBER", |
| 'Nl' => "G_UNICODE_LETTER_NUMBER", |
| 'No' => "G_UNICODE_OTHER_NUMBER", |
| 'Zs' => "G_UNICODE_SPACE_SEPARATOR", |
| 'Zl' => "G_UNICODE_LINE_SEPARATOR", |
| 'Zp' => "G_UNICODE_PARAGRAPH_SEPARATOR", |
| 'Cc' => "G_UNICODE_CONTROL", |
| 'Cf' => "G_UNICODE_FORMAT", |
| 'Cs' => "G_UNICODE_SURROGATE", |
| 'Co' => "G_UNICODE_PRIVATE_USE", |
| 'Cn' => "G_UNICODE_UNASSIGNED", |
| |
| # Informative. |
| 'Lm' => "G_UNICODE_MODIFIER_LETTER", |
| 'Lo' => "G_UNICODE_OTHER_LETTER", |
| 'Pc' => "G_UNICODE_CONNECT_PUNCTUATION", |
| 'Pd' => "G_UNICODE_DASH_PUNCTUATION", |
| 'Ps' => "G_UNICODE_OPEN_PUNCTUATION", |
| 'Pe' => "G_UNICODE_CLOSE_PUNCTUATION", |
| 'Pi' => "G_UNICODE_INITIAL_PUNCTUATION", |
| 'Pf' => "G_UNICODE_FINAL_PUNCTUATION", |
| 'Po' => "G_UNICODE_OTHER_PUNCTUATION", |
| 'Sm' => "G_UNICODE_MATH_SYMBOL", |
| 'Sc' => "G_UNICODE_CURRENCY_SYMBOL", |
| 'Sk' => "G_UNICODE_MODIFIER_SYMBOL", |
| 'So' => "G_UNICODE_OTHER_SYMBOL" |
| ); |
| |
| %break_mappings = |
| ( |
| 'AI' => "G_UNICODE_BREAK_AMBIGUOUS", |
| 'AL' => "G_UNICODE_BREAK_ALPHABETIC", |
| 'AK' => "G_UNICODE_BREAK_AKSARA", |
| 'AP' => "G_UNICODE_BREAK_AKSARA_PRE_BASE", |
| 'AS' => "G_UNICODE_BREAK_AKSARA_START", |
| 'B2' => "G_UNICODE_BREAK_BEFORE_AND_AFTER", |
| 'BA' => "G_UNICODE_BREAK_AFTER", |
| 'BB' => "G_UNICODE_BREAK_BEFORE", |
| 'BK' => "G_UNICODE_BREAK_MANDATORY", |
| 'CB' => "G_UNICODE_BREAK_CONTINGENT", |
| 'CJ' => "G_UNICODE_BREAK_CONDITIONAL_JAPANESE_STARTER", |
| 'CL' => "G_UNICODE_BREAK_CLOSE_PUNCTUATION", |
| 'CM' => "G_UNICODE_BREAK_COMBINING_MARK", |
| 'CP' => "G_UNICODE_BREAK_CLOSE_PARENTHESIS", |
| 'CR' => "G_UNICODE_BREAK_CARRIAGE_RETURN", |
| 'EB' => "G_UNICODE_BREAK_EMOJI_BASE", |
| 'EM' => "G_UNICODE_BREAK_EMOJI_MODIFIER", |
| 'EX' => "G_UNICODE_BREAK_EXCLAMATION", |
| 'GL' => "G_UNICODE_BREAK_NON_BREAKING_GLUE", |
| 'H2' => "G_UNICODE_BREAK_HANGUL_LV_SYLLABLE", |
| 'H3' => "G_UNICODE_BREAK_HANGUL_LVT_SYLLABLE", |
| 'HL' => "G_UNICODE_BREAK_HEBREW_LETTER", |
| 'HY' => "G_UNICODE_BREAK_HYPHEN", |
| 'ID' => "G_UNICODE_BREAK_IDEOGRAPHIC", |
| 'IN' => "G_UNICODE_BREAK_INSEPARABLE", |
| 'IS' => "G_UNICODE_BREAK_INFIX_SEPARATOR", |
| 'JL' => "G_UNICODE_BREAK_HANGUL_L_JAMO", |
| 'JT' => "G_UNICODE_BREAK_HANGUL_T_JAMO", |
| 'JV' => "G_UNICODE_BREAK_HANGUL_V_JAMO", |
| 'LF' => "G_UNICODE_BREAK_LINE_FEED", |
| 'NL' => "G_UNICODE_BREAK_NEXT_LINE", |
| 'NS' => "G_UNICODE_BREAK_NON_STARTER", |
| 'NU' => "G_UNICODE_BREAK_NUMERIC", |
| 'OP' => "G_UNICODE_BREAK_OPEN_PUNCTUATION", |
| 'PO' => "G_UNICODE_BREAK_POSTFIX", |
| 'PR' => "G_UNICODE_BREAK_PREFIX", |
| 'QU' => "G_UNICODE_BREAK_QUOTATION", |
| 'RI' => "G_UNICODE_BREAK_REGIONAL_INDICATOR", |
| 'SA' => "G_UNICODE_BREAK_COMPLEX_CONTEXT", |
| 'SG' => "G_UNICODE_BREAK_SURROGATE", |
| 'SP' => "G_UNICODE_BREAK_SPACE", |
| 'SY' => "G_UNICODE_BREAK_SYMBOL", |
| 'VF' => "G_UNICODE_BREAK_VIRAMA_FINAL", |
| 'VI' => "G_UNICODE_BREAK_VIRAMA", |
| 'WJ' => "G_UNICODE_BREAK_WORD_JOINER", |
| 'XX' => "G_UNICODE_BREAK_UNKNOWN", |
| 'ZW' => "G_UNICODE_BREAK_ZERO_WIDTH_SPACE", |
| 'ZWJ' => "G_UNICODE_BREAK_ZERO_WIDTH_JOINER" |
| ); |
| |
| # Title case mappings. |
| %title_to_lower = (); |
| %title_to_upper = (); |
| |
| # Maximum length of special-case strings |
| |
| my @special_cases; |
| my @special_case_offsets; |
| my $special_case_offset = 0; |
| |
| # Scripts |
| |
| my @scripts; |
| |
| # East asian widths |
| |
| my @eawidths; |
| |
| $do_decomp = 0; |
| $do_props = 1; |
| $do_scripts = 1; |
| if (@ARGV && $ARGV[0] eq '-decomp') |
| { |
| $do_decomp = 1; |
| $do_props = 0; |
| shift @ARGV; |
| } |
| elsif (@ARGV && $ARGV[0] eq '-both') |
| { |
| $do_decomp = 1; |
| shift @ARGV; |
| } |
| |
| if (@ARGV != 2) { |
| $0 =~ s@.*/@@; |
| die "\nUsage: $0 [-decomp | -both] UNICODE-VERSION DIRECTORY\n\n DIRECTORY should contain the following Unicode data files:\n UnicodeData.txt, LineBreak.txt, SpecialCasing.txt, CaseFolding.txt,\n CompositionExclusions.txt Scripts.txt extracted/DerivedEastAsianWidth.txt \n\n"; |
| } |
| |
| my ($unicodedatatxt, $linebreaktxt, $specialcasingtxt, $casefoldingtxt, $compositionexclusionstxt, |
| $scriptstxt, $derivedeastasianwidth); |
| |
| my $d = $ARGV[1]; |
| opendir (my $dir, $d) or die "Cannot open Unicode data dir $d: $!\n"; |
| for my $f (readdir ($dir)) |
| { |
| $unicodedatatxt = "$d/$f" if ($f =~ /^UnicodeData.*\.txt/); |
| $linebreaktxt = "$d/$f" if ($f =~ /^LineBreak.*\.txt/); |
| $specialcasingtxt = "$d/$f" if ($f =~ /^SpecialCasing.*\.txt/); |
| $casefoldingtxt = "$d/$f" if ($f =~ /^CaseFolding.*\.txt/); |
| $compositionexclusionstxt = "$d/$f" if ($f =~ /^CompositionExclusions.*\.txt/); |
| $scriptstxt = "$d/$f" if ($f =~ /^Scripts.*\.txt/); |
| } |
| |
| my $extd = $ARGV[1] . "/extracted"; |
| opendir (my $extdir, $extd) or die "Cannot open Unicode/extracted data dir $extd: $!\n"; |
| for my $f (readdir ($extdir)) |
| { |
| $derivedeastasianwidthtxt = "$extd/$f" if ($f =~ /^DerivedEastAsianWidth.*\.txt/); |
| } |
| |
| defined $unicodedatatxt or die "Did not find UnicodeData file"; |
| defined $linebreaktxt or die "Did not find LineBreak file"; |
| defined $specialcasingtxt or die "Did not find SpecialCasing file"; |
| defined $casefoldingtxt or die "Did not find CaseFolding file"; |
| defined $compositionexclusionstxt or die "Did not find CompositionExclusions file"; |
| defined $scriptstxt or die "Did not find Scripts file"; |
| defined $derivedeastasianwidthtxt or die "Did not find DerivedEastAsianWidth file"; |
| |
| print "Creating decomp table\n" if ($do_decomp); |
| print "Creating property table\n" if ($do_props); |
| |
| print "Composition exclusions from $compositionexclusionstxt\n"; |
| |
| open (INPUT, "< $compositionexclusionstxt") || exit 1; |
| |
| while (<INPUT>) { |
| |
| chop; |
| |
| next if /^#/; |
| next if /^\s*$/; |
| |
| s/\s*#.*//; |
| |
| s/^\s*//; |
| s/\s*$//; |
| |
| $composition_exclusions{hex($_)} = 1; |
| } |
| |
| close INPUT; |
| |
| print "Unicode data from $unicodedatatxt\n"; |
| |
| open (INPUT, "< $unicodedatatxt") || exit 1; |
| |
| # we save memory by skipping the huge empty area before U+E0000 |
| my $pages_before_e0000; |
| |
| $last_code = -1; |
| while (<INPUT>) |
| { |
| chop; |
| @fields = split (';', $_, 30); |
| if ($#fields != 14) |
| { |
| printf STDERR ("Entry for $fields[$CODE] has wrong number of fields (%d)\n", $#fields); |
| } |
| |
| $code = hex ($fields[$CODE]); |
| |
| if ($code >= 0xE0000 and $last_code < 0xE0000) |
| { |
| $pages_before_e0000 = ($last_code >> 8) + 1; |
| } |
| |
| if ($code > $last_code + 1) |
| { |
| # Found a gap. |
| if ($fields[$NAME] =~ /Last>/) |
| { |
| # Fill the gap with the last character read, |
| # since this was a range specified in the char database |
| @gfields = @fields; |
| } |
| else |
| { |
| # The gap represents undefined characters. Only the type |
| # matters. |
| @gfields = ('', '', 'Cn', '0', '', '', '', '', '', '', '', |
| '', '', '', ''); |
| } |
| for (++$last_code; $last_code < $code; ++$last_code) |
| { |
| $gfields{$CODE} = sprintf ("%04x", $last_code); |
| &process_one ($last_code, @gfields); |
| } |
| } |
| &process_one ($code, @fields); |
| $last_code = $code; |
| } |
| |
| close INPUT; |
| |
| @gfields = ('', '', 'Cn', '0', '', '', '', '', '', '', '', |
| '', '', '', ''); |
| for (++$last_code; $last_code <= 0x10FFFF; ++$last_code) |
| { |
| $gfields{$CODE} = sprintf ("%04x", $last_code); |
| &process_one ($last_code, @gfields); |
| } |
| --$last_code; # Want last to be 0x10FFFF. |
| |
| print "Creating line break table\n"; |
| |
| print "Line break data from $linebreaktxt\n"; |
| |
| open (INPUT, "< $linebreaktxt") || exit 1; |
| |
| $last_code = -1; |
| while (<INPUT>) |
| { |
| my ($start_code, $end_code); |
| |
| chop; |
| |
| next if /^#/; |
| next if /^$/; |
| |
| s/\s*#.*//; |
| |
| @fields = split (';', $_, 30); |
| if ($#fields != 1) |
| { |
| printf STDERR ("Entry for $fields[$CODE] has wrong number of fields (%d)\n", $#fields); |
| next; |
| } |
| |
| # Trim leading and trailing whitespace |
| $fields[$CODE] =~ s/^\s+|\s+$//; |
| $fields[$BREAK_PROPERTY] =~ s/^\s+|\s+$//; |
| |
| if ($fields[$CODE] =~ /([A-F0-9]{4,6})\.\.([A-F0-9]{4,6})/) |
| { |
| $start_code = hex ($1); |
| $end_code = hex ($2); |
| } else { |
| $start_code = $end_code = hex ($fields[$CODE]); |
| |
| } |
| |
| if ($start_code > $last_code + 1) |
| { |
| # The gap represents undefined characters. If assigned, |
| # they are AL, if not assigned, XX |
| for (++$last_code; $last_code < $start_code; ++$last_code) |
| { |
| if ($type[$last_code] eq 'Cn') |
| { |
| $break_props[$last_code] = 'XX'; |
| } |
| else |
| { |
| $break_props[$last_code] = 'AL'; |
| } |
| } |
| } |
| |
| for ($last_code = $start_code; $last_code <= $end_code; $last_code++) |
| { |
| $break_props[$last_code] = $fields[$BREAK_PROPERTY]; |
| } |
| |
| $last_code = $end_code; |
| } |
| |
| close INPUT; |
| |
| for (++$last_code; $last_code <= 0x10FFFF; ++$last_code) |
| { |
| if ($type[$last_code] eq 'Cn') |
| { |
| $break_props[$last_code] = 'XX'; |
| } |
| else |
| { |
| $break_props[$last_code] = 'AL'; |
| } |
| } |
| --$last_code; # Want last to be 0x10FFFF. |
| |
| print STDERR "Last code is not 0x10FFFF" if ($last_code != 0x10FFFF); |
| |
| print "Reading special-casing table for case conversion\n"; |
| |
| open (INPUT, "< $specialcasingtxt") || exit 1; |
| |
| while (<INPUT>) |
| { |
| my $code; |
| |
| chop; |
| |
| next if /^#/; |
| next if /^\s*$/; |
| |
| s/\s*#.*//; |
| |
| @fields = split ('\s*;\s*', $_, 30); |
| |
| $raw_code = $fields[$CASE_CODE]; |
| $code = hex ($raw_code); |
| |
| if ($#fields != 4 && $#fields != 5) |
| { |
| printf STDERR ("Entry for $raw_code has wrong number of fields (%d)\n", $#fields); |
| next; |
| } |
| |
| if (!defined $type[$code]) |
| { |
| printf STDERR "Special case for code point: $code, which has no defined type\n"; |
| next; |
| } |
| |
| if (defined $fields[5]) { |
| # Ignore conditional special cases - we'll handle them in code |
| next; |
| } |
| |
| if ($type[$code] eq 'Lu') |
| { |
| (hex $fields[$CASE_UPPER] == $code) || die "$raw_code is Lu and UCD_Upper($raw_code) != $raw_code"; |
| |
| &add_special_case ($code, $value[$code], $fields[$CASE_LOWER], $fields[$CASE_TITLE]); |
| |
| } elsif ($type[$code] eq 'Lt') |
| { |
| (hex $fields[$CASE_TITLE] == $code) || die "$raw_code is Lt and UCD_Title($raw_code) != $raw_code"; |
| |
| &add_special_case ($code, undef, $fields[$CASE_LOWER], $fields[$CASE_UPPER]); |
| } elsif ($type[$code] eq 'Ll') |
| { |
| (hex $fields[$CASE_LOWER] == $code) || die "$raw_code is Ll and UCD_Lower($raw_code) != $raw_code"; |
| |
| &add_special_case ($code, $value[$code], $fields[$CASE_UPPER], $fields[$CASE_TITLE]); |
| } else { |
| printf STDERR "Special case for non-alphabetic code point: $raw_code\n"; |
| next; |
| } |
| } |
| |
| close INPUT; |
| |
| open (INPUT, "< $casefoldingtxt") || exit 1; |
| |
| my $casefoldlen = 0; |
| my @casefold; |
| |
| while (<INPUT>) |
| { |
| my $code; |
| |
| chop; |
| |
| next if /^#/; |
| next if /^\s*$/; |
| |
| s/\s*#.*//; |
| |
| @fields = split ('\s*;\s*', $_, 30); |
| |
| $raw_code = $fields[$FOLDING_CODE]; |
| $code = hex ($raw_code); |
| |
| if ($#fields != 3) |
| { |
| printf STDERR ("Entry for $raw_code has wrong number of fields (%d)\n", $#fields); |
| next; |
| } |
| |
| # we don't use Simple or Turkic rules here |
| next if ($fields[$FOLDING_STATUS] =~ /^[ST]$/); |
| |
| @values = map { hex ($_) } split /\s+/, $fields[$FOLDING_MAPPING]; |
| |
| # Check simple case |
| |
| if (@values == 1 && |
| !(defined $value[$code] && $value[$code] >= 0x1000000) && |
| defined $type[$code]) { |
| |
| my $lower; |
| if ($type[$code] eq 'Ll') |
| { |
| $lower = $code; |
| } elsif ($type[$code] eq 'Lt') |
| { |
| $lower = $title_to_lower{$code}; |
| } elsif ($type[$code] eq 'Lu') |
| { |
| $lower = $value[$code]; |
| } else { |
| $lower = $code; |
| } |
| |
| if ($lower == $values[0]) { |
| next; |
| } |
| } |
| |
| my $string = pack ("U*", @values); |
| |
| if (1 + &length_in_bytes ($string) > $casefoldlen) { |
| $casefoldlen = 1 + &length_in_bytes ($string); |
| } |
| |
| push @casefold, [ $code, &escape ($string) ]; |
| } |
| |
| close INPUT; |
| |
| print "Reading scripts\n"; |
| |
| open (INPUT, "< $scriptstxt") || exit 1; |
| |
| while (<INPUT>) { |
| s/#.*//; |
| next if /^\s*$/; |
| if (!/^([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s*;\s*([A-Za-z_]+)\s*$/) { |
| die "Cannot parse line: '$_'\n"; |
| } |
| |
| if (defined $2) { |
| push @scripts, [ hex $1, hex $2, uc $3 ]; |
| } else { |
| push @scripts, [ hex $1, hex $1, uc $3 ]; |
| } |
| } |
| |
| close INPUT; |
| |
| print "Reading derived east asian widths\n"; |
| |
| open (INPUT, "< $derivedeastasianwidthtxt") || exit 1; |
| |
| while (<INPUT>) |
| { |
| my ($start_code, $end_code); |
| |
| chop; |
| |
| s/#.*//; |
| next if /^\s*$/; |
| if (!/^([0-9A-F]+)(?:\.\.([0-9A-F]+))?\s*;\s*([A-Za-z_]+)\s*$/) { |
| die "Cannot parse line: '$_'\n"; |
| } |
| |
| if (defined $2) { |
| push @eawidths, [ hex $1, hex $2, $3 ]; |
| } else { |
| push @eawidths, [ hex $1, hex $1, $3 ]; |
| } |
| } |
| |
| close INPUT; |
| |
| if ($do_props) { |
| &print_tables ($last_code) |
| } |
| if ($do_decomp) { |
| &print_decomp ($last_code); |
| &output_composition_table; |
| } |
| &print_line_break ($last_code); |
| |
| if ($do_scripts) { |
| &print_scripts |
| } |
| |
| exit 0; |
| |
| |
| # perl "length" returns the length in characters |
| sub length_in_bytes |
| { |
| my ($string) = @_; |
| |
| return length $string; |
| } |
| |
| # Process a single character. |
| sub process_one |
| { |
| my ($code, @fields) = @_; |
| |
| $type[$code] = $fields[$CATEGORY]; |
| if ($type[$code] eq 'Nd') |
| { |
| $value[$code] = int ($fields[$DECIMAL_VALUE]); |
| } |
| elsif ($type[$code] eq 'Ll') |
| { |
| $value[$code] = hex ($fields[$UPPER]); |
| } |
| elsif ($type[$code] eq 'Lu') |
| { |
| $value[$code] = hex ($fields[$LOWER]); |
| } |
| |
| if ($type[$code] eq 'Lt') |
| { |
| $title_to_lower{$code} = hex ($fields[$LOWER]); |
| $title_to_upper{$code} = hex ($fields[$UPPER]); |
| } |
| |
| $cclass[$code] = $fields[$COMBINING_CLASSES]; |
| |
| # Handle decompositions. |
| if ($fields[$DECOMPOSITION] ne '') |
| { |
| if ($fields[$DECOMPOSITION] =~ s/\<.*\>\s*//) { |
| $decompose_compat[$code] = 1; |
| } else { |
| $decompose_compat[$code] = 0; |
| |
| if (!exists $composition_exclusions{$code}) { |
| $compositions{$code} = $fields[$DECOMPOSITION]; |
| } |
| } |
| $decompositions[$code] = $fields[$DECOMPOSITION]; |
| } |
| } |
| |
| sub print_tables |
| { |
| my ($last) = @_; |
| my ($outfile) = "gunichartables.h"; |
| |
| local ($bytes_out) = 0; |
| |
| print "Writing $outfile...\n"; |
| |
| open (OUT, "> $outfile"); |
| |
| print OUT "/* This file is automatically generated. DO NOT EDIT!\n"; |
| print OUT " Instead, edit gen-unicode-tables.pl and re-run. */\n\n"; |
| |
| print OUT "#ifndef CHARTABLES_H\n"; |
| print OUT "#define CHARTABLES_H\n\n"; |
| |
| print OUT "#define G_UNICODE_DATA_VERSION \"$ARGV[0]\"\n\n"; |
| |
| printf OUT "#define G_UNICODE_LAST_CHAR 0x%04x\n\n", $last; |
| |
| printf OUT "#define G_UNICODE_MAX_TABLE_INDEX 10000\n\n"; |
| |
| my $last_part1 = ($pages_before_e0000 * 256) - 1; |
| printf OUT "#define G_UNICODE_LAST_CHAR_PART1 0x%04X\n\n", $last_part1; |
| printf OUT "#define G_UNICODE_LAST_PAGE_PART1 %d\n\n", $pages_before_e0000 - 1; |
| |
| $table_index = 0; |
| printf OUT "static const char type_data[][256] = {\n"; |
| for ($count = 0; $count <= $last; $count += 256) |
| { |
| $row[$count / 256] = &print_row ($count, 1, \&fetch_type); |
| } |
| printf OUT "\n};\n\n"; |
| |
| printf OUT "/* U+0000 through U+%04X */\n", $last_part1; |
| print OUT "static const gint16 type_table_part1[$pages_before_e0000] = {\n"; |
| for ($count = 0; $count <= $last_part1; $count += 256) |
| { |
| print OUT ",\n" if $count > 0; |
| print OUT " ", $row[$count / 256]; |
| $bytes_out += 2; |
| } |
| print OUT "\n};\n\n"; |
| |
| printf OUT "/* U+E0000 through U+%04X */\n", $last; |
| print OUT "static const gint16 type_table_part2[768] = {\n"; |
| for ($count = 0xE0000; $count <= $last; $count += 256) |
| { |
| print OUT ",\n" if $count > 0xE0000; |
| print OUT " ", $row[$count / 256]; |
| $bytes_out += 2; |
| } |
| print OUT "\n};\n\n"; |
| |
| |
| # |
| # Now print attribute table. |
| # |
| |
| $table_index = 0; |
| printf OUT "static const gunichar attr_data[][256] = {\n"; |
| for ($count = 0; $count <= $last; $count += 256) |
| { |
| $row[$count / 256] = &print_row ($count, 4, \&fetch_attr); |
| } |
| printf OUT "\n};\n\n"; |
| |
| printf OUT "/* U+0000 through U+%04X */\n", $last_part1; |
| print OUT "static const gint16 attr_table_part1[$pages_before_e0000] = {\n"; |
| for ($count = 0; $count <= $last_part1; $count += 256) |
| { |
| print OUT ",\n" if $count > 0; |
| print OUT " ", $row[$count / 256]; |
| $bytes_out += 2; |
| } |
| print OUT "\n};\n\n"; |
| |
| printf OUT "/* U+E0000 through U+%04X */\n", $last; |
| print OUT "static const gint16 attr_table_part2[768] = {\n"; |
| for ($count = 0xE0000; $count <= $last; $count += 256) |
| { |
| print OUT ",\n" if $count > 0xE0000; |
| print OUT " ", $row[$count / 256]; |
| $bytes_out += 2; |
| } |
| print OUT "\n};\n\n"; |
| |
| # |
| # print title case table |
| # |
| |
| print OUT "static const gunichar title_table[][3] = {\n"; |
| my ($item); |
| my ($first) = 1; |
| foreach $item (sort keys %title_to_lower) |
| { |
| print OUT ",\n" |
| unless $first; |
| $first = 0; |
| printf OUT " { 0x%04x, 0x%04x, 0x%04x }", $item, $title_to_upper{$item}, $title_to_lower{$item}; |
| $bytes_out += 12; |
| } |
| print OUT "\n};\n\n"; |
| |
| # |
| # And special case conversion table -- conversions that change length |
| # |
| &output_special_case_table (\*OUT); |
| &output_casefold_table (\*OUT); |
| |
| # |
| # And the widths tables |
| # |
| &output_width_tables (\*OUT); |
| |
| print OUT "#endif /* CHARTABLES_H */\n"; |
| |
| close (OUT); |
| |
| printf STDERR "Generated %d bytes in tables\n", $bytes_out; |
| } |
| |
| # A fetch function for the type table. |
| sub fetch_type |
| { |
| my ($index) = @_; |
| return $mappings{$type[$index]}; |
| } |
| |
| # A fetch function for the attribute table. |
| sub fetch_attr |
| { |
| my ($index) = @_; |
| if (defined $value[$index]) |
| { |
| return sprintf ("0x%04x", $value[$index]); |
| } |
| else |
| { |
| return "0x0000"; |
| } |
| } |
| |
| sub print_row |
| { |
| my ($start, $typsize, $fetcher) = @_; |
| |
| my ($i); |
| my (@values); |
| my ($flag) = 1; |
| my ($off); |
| |
| for ($off = 0; $off < 256; ++$off) |
| { |
| $values[$off] = $fetcher->($off + $start); |
| if ($values[$off] ne $values[0]) |
| { |
| $flag = 0; |
| } |
| } |
| if ($flag) |
| { |
| return $values[0] . " + G_UNICODE_MAX_TABLE_INDEX"; |
| } |
| |
| printf OUT ",\n" if ($table_index != 0); |
| printf OUT " { /* page %d, index %d */\n ", $start / 256, $table_index; |
| my ($column) = 4; |
| for ($i = $start; $i < $start + 256; ++$i) |
| { |
| print OUT "," |
| if $i > $start; |
| my ($text) = $values[$i - $start]; |
| if (length ($text) + $column + 2 > 78) |
| { |
| print OUT "\n "; |
| $column = 4; |
| } |
| elsif ($i > $start) |
| { |
| print OUT " " |
| } |
| print OUT $text; |
| $column += length ($text) + 2; |
| } |
| print OUT "\n }"; |
| |
| $bytes_out += 256 * $typsize; |
| |
| return sprintf "%d /* page %d */", $table_index++, $start / 256; |
| } |
| |
| sub escape |
| { |
| my ($string) = @_; |
| |
| my $escaped = unpack("H*", $string); |
| $escaped =~ s/(.{2})/\\x$1/g; |
| |
| return $escaped; |
| } |
| |
| # Returns the offset of $decomp in the offset string. Updates the |
| # referenced variables as appropriate. |
| sub handle_decomp ($$$$) |
| { |
| my ($decomp, $decomp_offsets_ref, $decomp_string_ref, $decomp_string_offset_ref) = @_; |
| my $offset = "G_UNICODE_NOT_PRESENT_OFFSET"; |
| |
| if (defined $decomp) |
| { |
| if (defined $decomp_offsets_ref->{$decomp}) |
| { |
| $offset = $decomp_offsets_ref->{$decomp}; |
| } |
| else |
| { |
| $offset = ${$decomp_string_offset_ref}; |
| $decomp_offsets_ref->{$decomp} = $offset; |
| ${$decomp_string_ref} .= "\n \"" . &escape ($decomp) . "\\0\" /* offset ${$decomp_string_offset_ref} */"; |
| ${$decomp_string_offset_ref} += &length_in_bytes ($decomp) + 1; |
| } |
| } |
| |
| return $offset; |
| } |
| |
| # Generate the character decomposition header. |
| sub print_decomp |
| { |
| my ($last) = @_; |
| my ($outfile) = "gunidecomp.h"; |
| |
| local ($bytes_out) = 0; |
| |
| print "Writing $outfile...\n"; |
| |
| open (OUT, "> $outfile") || exit 1; |
| |
| print OUT "/* This file is automatically generated. DO NOT EDIT! */\n\n"; |
| print OUT "#ifndef DECOMP_H\n"; |
| print OUT "#define DECOMP_H\n\n"; |
| |
| printf OUT "#define G_UNICODE_LAST_CHAR 0x%04x\n\n", $last; |
| |
| printf OUT "#define G_UNICODE_MAX_TABLE_INDEX (0x110000 / 256)\n\n"; |
| |
| my $last_part1 = ($pages_before_e0000 * 256) - 1; |
| printf OUT "#define G_UNICODE_LAST_CHAR_PART1 0x%04X\n\n", $last_part1; |
| printf OUT "#define G_UNICODE_LAST_PAGE_PART1 %d\n\n", $pages_before_e0000 - 1; |
| |
| $NOT_PRESENT_OFFSET = 65535; |
| print OUT "#define G_UNICODE_NOT_PRESENT_OFFSET $NOT_PRESENT_OFFSET\n\n"; |
| |
| my ($count, @row); |
| $table_index = 0; |
| printf OUT "static const guchar cclass_data[][256] = {\n"; |
| for ($count = 0; $count <= $last; $count += 256) |
| { |
| $row[$count / 256] = &print_row ($count, 1, \&fetch_cclass); |
| } |
| printf OUT "\n};\n\n"; |
| |
| print OUT "static const gint16 combining_class_table_part1[$pages_before_e0000] = {\n"; |
| for ($count = 0; $count <= $last_part1; $count += 256) |
| { |
| print OUT ",\n" if $count > 0; |
| print OUT " ", $row[$count / 256]; |
| $bytes_out += 2; |
| } |
| print OUT "\n};\n\n"; |
| |
| print OUT "static const gint16 combining_class_table_part2[768] = {\n"; |
| for ($count = 0xE0000; $count <= $last; $count += 256) |
| { |
| print OUT ",\n" if $count > 0xE0000; |
| print OUT " ", $row[$count / 256]; |
| $bytes_out += 2; |
| } |
| print OUT "\n};\n\n"; |
| |
| print OUT "typedef struct\n{\n"; |
| print OUT " gunichar ch;\n"; |
| print OUT " guint16 canon_offset;\n"; |
| print OUT " guint16 compat_offset;\n"; |
| print OUT "} decomposition;\n\n"; |
| |
| print OUT "static const decomposition decomp_table[] =\n{\n"; |
| my ($iter); |
| my ($first) = 1; |
| my ($decomp_string) = ""; |
| my ($decomp_string_offset) = 0; |
| for ($count = 0; $count <= $last; ++$count) |
| { |
| if (defined $decompositions[$count]) |
| { |
| print OUT ",\n" |
| if ! $first; |
| $first = 0; |
| |
| my $canon_decomp; |
| my $compat_decomp; |
| |
| if (!$decompose_compat[$count]) { |
| $canon_decomp = make_decomp ($count, 0); |
| } |
| $compat_decomp = make_decomp ($count, 1); |
| |
| if (defined $canon_decomp && $compat_decomp eq $canon_decomp) { |
| undef $compat_decomp; |
| } |
| |
| my $canon_offset = handle_decomp ($canon_decomp, \%decomp_offsets, \$decomp_string, \$decomp_string_offset); |
| my $compat_offset = handle_decomp ($compat_decomp, \%decomp_offsets, \$decomp_string, \$decomp_string_offset); |
| |
| die if $decomp_string_offset > $NOT_PRESENT_OFFSET; |
| |
| printf OUT qq( { 0x%04x, $canon_offset, $compat_offset }), $count; |
| $bytes_out += 8; |
| } |
| } |
| print OUT "\n};\n\n"; |
| $bytes_out += $decomp_string_offset + 1; |
| |
| printf OUT "static const gchar decomp_expansion_string[] = %s;\n\n", $decomp_string; |
| |
| print OUT "typedef struct\n{\n"; |
| print OUT " gunichar ch;\n"; |
| print OUT " gunichar a;\n"; |
| print OUT " gunichar b;\n"; |
| print OUT "} decomposition_step;\n\n"; |
| |
| # There's lots of room to optimize the following table... |
| print OUT "static const decomposition_step decomp_step_table[] =\n{\n"; |
| $first = 1; |
| my @steps = (); |
| for ($count = 0; $count <= $last; ++$count) |
| { |
| if ((defined $decompositions[$count]) && (!$decompose_compat[$count])) |
| { |
| print OUT ",\n" |
| if ! $first; |
| $first = 0; |
| my @list; |
| @list = (split(' ', $decompositions[$count]), "0"); |
| printf OUT qq( { 0x%05x, 0x%05x, 0x%05x }), $count, hex($list[0]), hex($list[1]); |
| # don't include 1:1 in the compose table |
| push @steps, [ ($count, hex($list[0]), hex($list[1])) ] |
| if hex($list[1]) |
| } |
| } |
| print OUT "\n};\n\n"; |
| |
| print OUT "#endif /* DECOMP_H */\n"; |
| |
| printf STDERR "Generated %d bytes in decomp tables\n", $bytes_out; |
| } |
| |
| sub print_line_break |
| { |
| my ($last) = @_; |
| my ($outfile) = "gunibreak.h"; |
| |
| local ($bytes_out) = 0; |
| |
| print "Writing $outfile...\n"; |
| |
| open (OUT, "> $outfile"); |
| |
| print OUT "/* This file is automatically generated. DO NOT EDIT!\n"; |
| print OUT " Instead, edit gen-unicode-tables.pl and re-run. */\n\n"; |
| |
| print OUT "#ifndef BREAKTABLES_H\n"; |
| print OUT "#define BREAKTABLES_H\n\n"; |
| |
| print OUT "#include <glib/gtypes.h>\n"; |
| print OUT "#include <glib/gunicode.h>\n\n"; |
| |
| print OUT "#define G_UNICODE_DATA_VERSION \"$ARGV[0]\"\n\n"; |
| |
| printf OUT "#define G_UNICODE_LAST_CHAR 0x%04X\n\n", $last; |
| |
| printf OUT "#define G_UNICODE_MAX_TABLE_INDEX 10000\n\n"; |
| |
| my $last_part1 = ($pages_before_e0000 * 256) - 1; |
| printf OUT "/* the last code point that should be looked up in break_property_table_part1 */\n"; |
| printf OUT "#define G_UNICODE_LAST_CHAR_PART1 0x%04X\n\n", $last_part1; |
| |
| $table_index = 0; |
| printf OUT "static const gint8 break_property_data[][256] = {\n"; |
| for ($count = 0; $count <= $last; $count += 256) |
| { |
| $row[$count / 256] = &print_row ($count, 1, \&fetch_break_type); |
| } |
| printf OUT "\n};\n\n"; |
| |
| printf OUT "/* U+0000 through U+%04X */\n", $last_part1; |
| print OUT "static const gint16 break_property_table_part1[$pages_before_e0000] = {\n"; |
| for ($count = 0; $count <= $last_part1; $count += 256) |
| { |
| print OUT ",\n" if $count > 0; |
| print OUT " ", $row[$count / 256]; |
| $bytes_out += 2; |
| } |
| print OUT "\n};\n\n"; |
| |
| printf OUT "/* U+E0000 through U+%04X */\n", $last; |
| print OUT "static const gint16 break_property_table_part2[768] = {\n"; |
| for ($count = 0xE0000; $count <= $last; $count += 256) |
| { |
| print OUT ",\n" if $count > 0xE0000; |
| print OUT " ", $row[$count / 256]; |
| $bytes_out += 2; |
| } |
| print OUT "\n};\n\n"; |
| |
| |
| print OUT "#endif /* BREAKTABLES_H */\n"; |
| |
| close (OUT); |
| |
| printf STDERR "Generated %d bytes in break tables\n", $bytes_out; |
| } |
| |
| |
| # A fetch function for the break properties table. |
| sub fetch_break_type |
| { |
| my ($index) = @_; |
| return $break_mappings{$break_props[$index]}; |
| } |
| |
| # Fetcher for combining class. |
| sub fetch_cclass |
| { |
| my ($i) = @_; |
| return $cclass[$i]; |
| } |
| |
| # Expand a character decomposition recursively. |
| sub expand_decomp |
| { |
| my ($code, $compat) = @_; |
| |
| my ($iter, $val); |
| my (@result) = (); |
| foreach $iter (split (' ', $decompositions[$code])) |
| { |
| $val = hex ($iter); |
| if (defined $decompositions[$val] && |
| ($compat || !$decompose_compat[$val])) |
| { |
| push (@result, &expand_decomp ($val, $compat)); |
| } |
| else |
| { |
| push (@result, $val); |
| } |
| } |
| |
| return @result; |
| } |
| |
| sub make_decomp |
| { |
| my ($code, $compat) = @_; |
| |
| my $result = ""; |
| foreach $iter (&expand_decomp ($code, $compat)) |
| { |
| $result .= pack ("U", $iter); # to utf-8 |
| } |
| |
| $result; |
| } |
| # Generate special case data string from two fields |
| sub add_special_case |
| { |
| my ($code, $single, $field1, $field2) = @_; |
| |
| @values = (defined $single ? $single : (), |
| (map { hex ($_) } split /\s+/, $field1), |
| 0, |
| (map { hex ($_) } split /\s+/, $field2)); |
| |
| $result = ""; |
| |
| for $value (@values) { |
| $result .= pack ("U", $value); # to utf-8 |
| } |
| |
| push @special_case_offsets, $special_case_offset; |
| |
| # We encode special cases up in the 0x1000000 space |
| $value[$code] = 0x1000000 + $special_case_offset; |
| |
| $special_case_offset += 1 + &length_in_bytes ($result); |
| |
| push @special_cases, &escape ($result); |
| } |
| |
| sub output_special_case_table |
| { |
| my $out = shift; |
| |
| print $out <<EOT; |
| |
| /* Table of special cases for case conversion; each record contains |
| * First, the best single character mapping to lowercase if Lu, |
| * and to uppercase if Ll, followed by the output mapping for the two cases |
| * other than the case of the codepoint, in the order [Ll],[Lu],[Lt], |
| * encoded in UTF-8, separated and terminated by a null character. |
| */ |
| static const gchar special_case_table[] = { |
| EOT |
| |
| my $i = 0; |
| for $case (@special_cases) { |
| print $out qq( "$case\\0" /* offset ${special_case_offsets[$i]} */\n); |
| $i++; |
| } |
| |
| print $out <<EOT; |
| }; |
| |
| EOT |
| |
| print STDERR "Generated " . ($special_case_offset + 1) . " bytes in special case table\n"; |
| } |
| |
| sub enumerate_ordered |
| { |
| my ($array) = @_; |
| |
| my $n = 0; |
| for my $code (sort { $a <=> $b } keys %$array) { |
| if ($array->{$code} == 1) { |
| delete $array->{$code}; |
| next; |
| } |
| $array->{$code} = $n++; |
| } |
| |
| return $n; |
| } |
| |
| sub output_composition_table |
| { |
| print STDERR "Generating composition table\n"; |
| |
| local ($bytes_out) = 0; |
| |
| my %first; |
| my %second; |
| |
| # First we need to go through and remove decompositions |
| # starting with a non-starter, and single-character |
| # decompositions. At the same time, record |
| # the first and second character of each decomposition |
| |
| for $code (keys %compositions) |
| { |
| @values = map { hex ($_) } split /\s+/, $compositions{$code}; |
| |
| # non-starters |
| if ($cclass[$code]) { |
| delete $compositions{$code}; |
| next; |
| } |
| if ($cclass[$values[0]]) { |
| delete $compositions{$code}; |
| next; |
| } |
| |
| # single-character decompositions |
| if (@values == 1) { |
| delete $compositions{$code}; |
| next; |
| } |
| |
| if (@values != 2) { |
| die "$code has more than two elements in its decomposition!\n"; |
| } |
| |
| if (exists $first{$values[0]}) { |
| $first{$values[0]}++; |
| } else { |
| $first{$values[0]} = 1; |
| } |
| } |
| |
| # Assign integer indices, removing singletons |
| my $n_first = enumerate_ordered (\%first); |
| |
| # Now record the second character of each (non-singleton) decomposition |
| for $code (keys %compositions) { |
| @values = map { hex ($_) } split /\s+/, $compositions{$code}; |
| |
| if (exists $first{$values[0]}) { |
| if (exists $second{$values[1]}) { |
| $second{$values[1]}++; |
| } else { |
| $second{$values[1]} = 1; |
| } |
| } |
| } |
| |
| # Assign integer indices, removing duplicate |
| my $n_second = enumerate_ordered (\%second); |
| |
| # Build reverse table |
| |
| my @first_singletons; |
| my @second_singletons; |
| my %reverse; |
| for $code (keys %compositions) { |
| @values = map { hex ($_) } split /\s+/, $compositions{$code}; |
| |
| my $first = $first{$values[0]}; |
| my $second = $second{$values[1]}; |
| |
| if (defined $first && defined $second) { |
| $reverse{"$first|$second"} = $code; |
| } elsif (!defined $first) { |
| push @first_singletons, [ $values[0], $values[1], $code ]; |
| } else { |
| push @second_singletons, [ $values[1], $values[0], $code ]; |
| } |
| } |
| |
| @first_singletons = sort { $a->[0] <=> $b->[0] } @first_singletons; |
| @second_singletons = sort { $a->[0] <=> $b->[0] } @second_singletons; |
| |
| my %vals; |
| |
| open OUT, ">gunicomp.h" or die "Cannot open gunicomp.h: $!\n"; |
| |
| # Assign values in lookup table for all code points involved |
| |
| my $total = 1; |
| my $last = 0; |
| printf OUT "#define COMPOSE_FIRST_START %d\n", $total; |
| for $code (keys %first) { |
| $vals{$code} = $first{$code} + $total; |
| $last = $code if $code > $last; |
| } |
| $total += $n_first; |
| $i = 0; |
| printf OUT "#define COMPOSE_FIRST_SINGLE_START %d\n", $total; |
| for $record (@first_singletons) { |
| my $code = $record->[0]; |
| $vals{$code} = $i++ + $total; |
| $last = $code if $code > $last; |
| } |
| $total += @first_singletons; |
| printf OUT "#define COMPOSE_SECOND_START %d\n", $total; |
| for $code (keys %second) { |
| $vals{$code} = $second{$code} + $total; |
| $last = $code if $code > $last; |
| } |
| $total += $n_second; |
| $i = 0; |
| printf OUT "#define COMPOSE_SECOND_SINGLE_START %d\n\n", $total; |
| for $record (@second_singletons) { |
| my $code = $record->[0]; |
| $vals{$code} = $i++ + $total; |
| $last = $code if $code > $last; |
| } |
| |
| printf OUT "#define COMPOSE_TABLE_LAST %d\n\n", $last / 256; |
| |
| # Output lookup table |
| |
| my @row; |
| $table_index = 0; |
| printf OUT "static const guint16 compose_data[][256] = {\n"; |
| for (my $count = 0; $count <= $last; $count += 256) |
| { |
| $row[$count / 256] = &print_row ($count, 2, sub { exists $vals{$_[0]} ? $vals{$_[0]} : 0; }); |
| } |
| printf OUT "\n};\n\n"; |
| |
| print OUT "static const gint16 compose_table[COMPOSE_TABLE_LAST + 1] = {\n"; |
| for (my $count = 0; $count <= $last; $count += 256) |
| { |
| print OUT ",\n" if $count > 0; |
| print OUT " ", $row[$count / 256]; |
| $bytes_out += 2; |
| } |
| print OUT "\n};\n\n"; |
| |
| # Output first singletons |
| |
| print OUT "static const gunichar compose_first_single[][2] = {\n"; |
| $i = 0; |
| for $record (@first_singletons) { |
| print OUT ",\n" if $i++ > 0; |
| printf OUT " { %#06x, %#06x }", $record->[1], $record->[2]; |
| } |
| print OUT "\n};\n"; |
| |
| $bytes_out += @first_singletons * 4; |
| |
| # Output second singletons |
| |
| print OUT "static const gunichar compose_second_single[][2] = {\n"; |
| $i = 0; |
| for $record (@second_singletons) { |
| print OUT ",\n" if $i++ > 0; |
| printf OUT " { %#06x, %#06x }", $record->[1], $record->[2]; |
| } |
| print OUT "\n};\n"; |
| |
| $bytes_out += @second_singletons * 4; |
| |
| # Output array of composition pairs |
| |
| print OUT <<EOT; |
| static const guint16 compose_array[$n_first][$n_second] = { |
| EOT |
| |
| for (my $i = 0; $i < $n_first; $i++) { |
| print OUT ",\n" if $i; |
| print OUT " { "; |
| for (my $j = 0; $j < $n_second; $j++) { |
| print OUT ", " if $j; |
| if (exists $reverse{"$i|$j"}) { |
| if ($reverse{"$i|$j"} > 0xFFFF) { |
| die "time to switch compose_array to gunichar" ; |
| } |
| printf OUT "0x%04x", $reverse{"$i|$j"}; |
| } else { |
| print OUT " 0"; |
| } |
| } |
| print OUT " }"; |
| } |
| print OUT "\n"; |
| |
| print OUT <<EOT; |
| }; |
| EOT |
| |
| $bytes_out += $n_first * $n_second * 2; |
| |
| printf STDERR "Generated %d bytes in compose tables\n", $bytes_out; |
| } |
| |
| sub output_casefold_table |
| { |
| my $out = shift; |
| |
| print $out <<EOT; |
| |
| /* Table of casefolding cases that can't be derived by lowercasing |
| */ |
| static const struct { |
| guint16 ch; |
| gchar data[$casefoldlen]; |
| } casefold_table[] = { |
| EOT |
| |
| @casefold = sort { $a->[0] <=> $b->[0] } @casefold; |
| |
| for $case (@casefold) |
| { |
| $code = $case->[0]; |
| $string = $case->[1]; |
| |
| if ($code > 0xFFFF) { |
| die "time to switch casefold_table to gunichar" ; |
| } |
| |
| print $out sprintf(qq( { 0x%04x, "$string" },\n), $code); |
| |
| } |
| |
| print $out <<EOT; |
| }; |
| |
| EOT |
| |
| my $recordlen = (2+$casefoldlen+1) & ~1; |
| printf "Generated %d bytes for casefold table\n", $recordlen * @casefold; |
| } |
| |
| sub output_one_width_table |
| { |
| my ($out, $name, $wpe) = @_; |
| my $start; |
| my $end; |
| my $wp; |
| my $rex; |
| |
| print $out "static const struct Interval g_unicode_width_table_${name}[] = {\n"; |
| |
| $rex = qr/$wpe/; |
| |
| for (my $i = 0; $i <= $#eawidths; $i++) { |
| $start = $eawidths[$i]->[0]; |
| $end = $eawidths[$i]->[1]; |
| $wp = $eawidths[$i]->[2]; |
| |
| next if ($wp !~ $rex); |
| |
| while ($i <= $#eawidths - 1 && |
| $eawidths[$i + 1]->[0] == $end + 1 && |
| ($eawidths[$i + 1]->[2] =~ $rex)) { |
| $i++; |
| $end = $eawidths[$i]->[1]; |
| } |
| |
| printf $out "{0x%04X, 0x%04X},\n", $start, $end; |
| } |
| |
| printf $out "};\n\n"; |
| } |
| |
| sub output_width_tables |
| { |
| my $out = shift; |
| |
| @eawidths = sort { $a->[0] <=> $b->[0] } @eawidths; |
| |
| print $out <<EOT; |
| |
| struct Interval |
| { |
| gunichar start, end; |
| }; |
| |
| EOT |
| |
| &output_one_width_table ($out,"wide", "[FW]"); |
| &output_one_width_table ($out, "ambiguous", "[A]"); |
| } |
| |
| sub print_scripts |
| { |
| my $start; |
| my $end; |
| my $script; |
| my $easy_range; |
| my $i; |
| |
| print STDERR "Writing gscripttable.h\n"; |
| |
| open OUT, ">gscripttable.h" or die "Cannot open gscripttable.h: $!\n"; |
| |
| print OUT<<EOT; |
| /* This file is automatically generated. DO NOT EDIT! |
| Instead, edit gen-unicode-tables.pl and re-run. */ |
| |
| #ifndef SCRIPTTABLES_H |
| #define SCRIPTTABLES_H |
| |
| EOT |
| |
| @scripts = sort { $a->[0] <=> $b->[0] } @scripts; |
| |
| $easy_range = 0x2000; |
| |
| print OUT<<EOT; |
| #define G_EASY_SCRIPTS_RANGE $easy_range |
| |
| static const guchar g_script_easy_table[$easy_range] = { |
| EOT |
| |
| $i = 0; |
| $end = -1; |
| |
| for (my $c = 0; $c < $easy_range; $c++) { |
| |
| if ($c % 3 == 0) { |
| printf OUT "\n "; |
| } |
| |
| if ($c > $end) { |
| $start = $scripts[$i]->[0]; |
| $end = $scripts[$i]->[1]; |
| $script = $scripts[$i]->[2]; |
| $i++; |
| } |
| |
| if ($c < $start) { |
| printf OUT " G_UNICODE_SCRIPT_UNKNOWN,"; |
| } else { |
| printf OUT " G_UNICODE_SCRIPT_%s,", $script; |
| } |
| } |
| |
| if ($end >= $easy_range) { |
| $i--; |
| $scripts[$i]->[0] = $easy_range; |
| } |
| |
| print OUT<<EOT; |
| |
| }; |
| |
| static const struct { |
| gunichar start; |
| guint16 chars; |
| guint16 script; |
| } g_script_table[] = { |
| EOT |
| |
| for (; $i <= $#scripts; $i++) { |
| $start = $scripts[$i]->[0]; |
| $end = $scripts[$i]->[1]; |
| $script = $scripts[$i]->[2]; |
| |
| while ($i <= $#scripts - 1 && |
| $scripts[$i + 1]->[0] == $end + 1 && |
| $scripts[$i + 1]->[2] eq $script) { |
| $i++; |
| $end = $scripts[$i]->[1]; |
| } |
| printf OUT " { %#06x, %5d, G_UNICODE_SCRIPT_%s },\n", $start, $end - $start + 1, $script; |
| } |
| |
| printf OUT<<EOT; |
| }; |
| |
| #endif /* SCRIPTTABLES_H */ |
| EOT |
| |
| close OUT; |
| } |