language/internal: copy language for refactoring
Change-Id: I04dad1af3f709ecf2f92a7257717b195a832a641
Reviewed-on: https://go-review.googlesource.com/95816
Run-TryBot: Marcel van Lohuizen <mpvl@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Nigel Tao <nigeltao@golang.org>
diff --git a/language/internal/common.go b/language/internal/common.go
new file mode 100644
index 0000000..9d86e18
--- /dev/null
+++ b/language/internal/common.go
@@ -0,0 +1,16 @@
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+package language
+
+// This file contains code common to the maketables.go and the package code.
+
+// langAliasType is the type of an alias in langAliasMap.
+type langAliasType int8
+
+const (
+ langDeprecated langAliasType = iota
+ langMacro
+ langLegacy
+
+ langAliasTypeUnknown langAliasType = -1
+)
diff --git a/language/internal/coverage.go b/language/internal/coverage.go
new file mode 100644
index 0000000..101fd23
--- /dev/null
+++ b/language/internal/coverage.go
@@ -0,0 +1,197 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language
+
+import (
+ "fmt"
+ "sort"
+)
+
+// The Coverage interface is used to define the level of coverage of an
+// internationalization service. Note that not all types are supported by all
+// services. As lists may be generated on the fly, it is recommended that users
+// of a Coverage cache the results.
+type Coverage interface {
+ // Tags returns the list of supported tags.
+ Tags() []Tag
+
+ // BaseLanguages returns the list of supported base languages.
+ BaseLanguages() []Base
+
+ // Scripts returns the list of supported scripts.
+ Scripts() []Script
+
+ // Regions returns the list of supported regions.
+ Regions() []Region
+}
+
+var (
+ // Supported defines a Coverage that lists all supported subtags. Tags
+ // always returns nil.
+ Supported Coverage = allSubtags{}
+)
+
+// TODO:
+// - Support Variants, numbering systems.
+// - CLDR coverage levels.
+// - Set of common tags defined in this package.
+
+type allSubtags struct{}
+
+// Regions returns the list of supported regions. As all regions are in a
+// consecutive range, it simply returns a slice of numbers in increasing order.
+// The "undefined" region is not returned.
+func (s allSubtags) Regions() []Region {
+ reg := make([]Region, numRegions)
+ for i := range reg {
+ reg[i] = Region{regionID(i + 1)}
+ }
+ return reg
+}
+
+// Scripts returns the list of supported scripts. As all scripts are in a
+// consecutive range, it simply returns a slice of numbers in increasing order.
+// The "undefined" script is not returned.
+func (s allSubtags) Scripts() []Script {
+ scr := make([]Script, numScripts)
+ for i := range scr {
+ scr[i] = Script{scriptID(i + 1)}
+ }
+ return scr
+}
+
+// BaseLanguages returns the list of all supported base languages. It generates
+// the list by traversing the internal structures.
+func (s allSubtags) BaseLanguages() []Base {
+ base := make([]Base, 0, numLanguages)
+ for i := 0; i < langNoIndexOffset; i++ {
+ // We included "und" already for the value 0.
+ if i != nonCanonicalUnd {
+ base = append(base, Base{langID(i)})
+ }
+ }
+ i := langNoIndexOffset
+ for _, v := range langNoIndex {
+ for k := 0; k < 8; k++ {
+ if v&1 == 1 {
+ base = append(base, Base{langID(i)})
+ }
+ v >>= 1
+ i++
+ }
+ }
+ return base
+}
+
+// Tags always returns nil.
+func (s allSubtags) Tags() []Tag {
+ return nil
+}
+
+// coverage is used used by NewCoverage which is used as a convenient way for
+// creating Coverage implementations for partially defined data. Very often a
+// package will only need to define a subset of slices. coverage provides a
+// convenient way to do this. Moreover, packages using NewCoverage, instead of
+// their own implementation, will not break if later new slice types are added.
+type coverage struct {
+ tags func() []Tag
+ bases func() []Base
+ scripts func() []Script
+ regions func() []Region
+}
+
+func (s *coverage) Tags() []Tag {
+ if s.tags == nil {
+ return nil
+ }
+ return s.tags()
+}
+
+// bases implements sort.Interface and is used to sort base languages.
+type bases []Base
+
+func (b bases) Len() int {
+ return len(b)
+}
+
+func (b bases) Swap(i, j int) {
+ b[i], b[j] = b[j], b[i]
+}
+
+func (b bases) Less(i, j int) bool {
+ return b[i].langID < b[j].langID
+}
+
+// BaseLanguages returns the result from calling s.bases if it is specified or
+// otherwise derives the set of supported base languages from tags.
+func (s *coverage) BaseLanguages() []Base {
+ if s.bases == nil {
+ tags := s.Tags()
+ if len(tags) == 0 {
+ return nil
+ }
+ a := make([]Base, len(tags))
+ for i, t := range tags {
+ a[i] = Base{langID(t.lang)}
+ }
+ sort.Sort(bases(a))
+ k := 0
+ for i := 1; i < len(a); i++ {
+ if a[k] != a[i] {
+ k++
+ a[k] = a[i]
+ }
+ }
+ return a[:k+1]
+ }
+ return s.bases()
+}
+
+func (s *coverage) Scripts() []Script {
+ if s.scripts == nil {
+ return nil
+ }
+ return s.scripts()
+}
+
+func (s *coverage) Regions() []Region {
+ if s.regions == nil {
+ return nil
+ }
+ return s.regions()
+}
+
+// NewCoverage returns a Coverage for the given lists. It is typically used by
+// packages providing internationalization services to define their level of
+// coverage. A list may be of type []T or func() []T, where T is either Tag,
+// Base, Script or Region. The returned Coverage derives the value for Bases
+// from Tags if no func or slice for []Base is specified. For other unspecified
+// types the returned Coverage will return nil for the respective methods.
+func NewCoverage(list ...interface{}) Coverage {
+ s := &coverage{}
+ for _, x := range list {
+ switch v := x.(type) {
+ case func() []Base:
+ s.bases = v
+ case func() []Script:
+ s.scripts = v
+ case func() []Region:
+ s.regions = v
+ case func() []Tag:
+ s.tags = v
+ case []Base:
+ s.bases = func() []Base { return v }
+ case []Script:
+ s.scripts = func() []Script { return v }
+ case []Region:
+ s.regions = func() []Region { return v }
+ case []Tag:
+ s.tags = func() []Tag { return v }
+ default:
+ panic(fmt.Sprintf("language: unsupported set type %T", v))
+ }
+ }
+ return s
+}
diff --git a/language/internal/coverage_test.go b/language/internal/coverage_test.go
new file mode 100644
index 0000000..8e08e5c
--- /dev/null
+++ b/language/internal/coverage_test.go
@@ -0,0 +1,154 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language
+
+import (
+ "fmt"
+ "reflect"
+ "testing"
+)
+
+func TestSupported(t *testing.T) {
+ // To prove the results are correct for a type, we test that the number of
+ // results is identical to the number of results on record, that all results
+ // are distinct and that all results are valid.
+ tests := map[string]int{
+ "BaseLanguages": numLanguages,
+ "Scripts": numScripts,
+ "Regions": numRegions,
+ "Tags": 0,
+ }
+ sup := reflect.ValueOf(Supported)
+ for name, num := range tests {
+ v := sup.MethodByName(name).Call(nil)[0]
+ if n := v.Len(); n != num {
+ t.Errorf("len(%s()) was %d; want %d", name, n, num)
+ }
+ dup := make(map[string]bool)
+ for i := 0; i < v.Len(); i++ {
+ x := v.Index(i).Interface()
+ // An invalid value will either cause a crash or result in a
+ // duplicate when passed to Sprint.
+ s := fmt.Sprint(x)
+ if dup[s] {
+ t.Errorf("%s: duplicate entry %q", name, s)
+ }
+ dup[s] = true
+ }
+ if len(dup) != v.Len() {
+ t.Errorf("%s: # unique entries was %d; want %d", name, len(dup), v.Len())
+ }
+ }
+}
+
+func TestNewCoverage(t *testing.T) {
+ bases := []Base{Base{0}, Base{3}, Base{7}}
+ scripts := []Script{Script{11}, Script{17}, Script{23}}
+ regions := []Region{Region{101}, Region{103}, Region{107}}
+ tags := []Tag{Make("pt"), Make("en"), Make("en-GB"), Make("en-US"), Make("pt-PT")}
+ fbases := func() []Base { return bases }
+ fscripts := func() []Script { return scripts }
+ fregions := func() []Region { return regions }
+ ftags := func() []Tag { return tags }
+
+ tests := []struct {
+ desc string
+ list []interface{}
+ bases []Base
+ scripts []Script
+ regions []Region
+ tags []Tag
+ }{
+ {
+ desc: "empty",
+ },
+ {
+ desc: "bases",
+ list: []interface{}{bases},
+ bases: bases,
+ },
+ {
+ desc: "scripts",
+ list: []interface{}{scripts},
+ scripts: scripts,
+ },
+ {
+ desc: "regions",
+ list: []interface{}{regions},
+ regions: regions,
+ },
+ {
+ desc: "bases derives from tags",
+ list: []interface{}{tags},
+ bases: []Base{Base{_en}, Base{_pt}},
+ tags: tags,
+ },
+ {
+ desc: "tags and bases",
+ list: []interface{}{tags, bases},
+ bases: bases,
+ tags: tags,
+ },
+ {
+ desc: "fully specified",
+ list: []interface{}{tags, bases, scripts, regions},
+ bases: bases,
+ scripts: scripts,
+ regions: regions,
+ tags: tags,
+ },
+ {
+ desc: "bases func",
+ list: []interface{}{fbases},
+ bases: bases,
+ },
+ {
+ desc: "scripts func",
+ list: []interface{}{fscripts},
+ scripts: scripts,
+ },
+ {
+ desc: "regions func",
+ list: []interface{}{fregions},
+ regions: regions,
+ },
+ {
+ desc: "tags func",
+ list: []interface{}{ftags},
+ bases: []Base{Base{_en}, Base{_pt}},
+ tags: tags,
+ },
+ {
+ desc: "tags and bases",
+ list: []interface{}{ftags, fbases},
+ bases: bases,
+ tags: tags,
+ },
+ {
+ desc: "fully specified",
+ list: []interface{}{ftags, fbases, fscripts, fregions},
+ bases: bases,
+ scripts: scripts,
+ regions: regions,
+ tags: tags,
+ },
+ }
+
+ for i, tt := range tests {
+ l := NewCoverage(tt.list...)
+ if a := l.BaseLanguages(); !reflect.DeepEqual(a, tt.bases) {
+ t.Errorf("%d:%s: BaseLanguages was %v; want %v", i, tt.desc, a, tt.bases)
+ }
+ if a := l.Scripts(); !reflect.DeepEqual(a, tt.scripts) {
+ t.Errorf("%d:%s: Scripts was %v; want %v", i, tt.desc, a, tt.scripts)
+ }
+ if a := l.Regions(); !reflect.DeepEqual(a, tt.regions) {
+ t.Errorf("%d:%s: Regions was %v; want %v", i, tt.desc, a, tt.regions)
+ }
+ if a := l.Tags(); !reflect.DeepEqual(a, tt.tags) {
+ t.Errorf("%d:%s: Tags was %v; want %v", i, tt.desc, a, tt.tags)
+ }
+ }
+}
diff --git a/language/internal/doc.go b/language/internal/doc.go
new file mode 100644
index 0000000..c5bcbaf
--- /dev/null
+++ b/language/internal/doc.go
@@ -0,0 +1,102 @@
+// Copyright 2017 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package language implements BCP 47 language tags and related functionality.
+//
+// The most important function of package language is to match a list of
+// user-preferred languages to a list of supported languages.
+// It alleviates the developer of dealing with the complexity of this process
+// and provides the user with the best experience
+// (see https://blog.golang.org/matchlang).
+//
+//
+// Matching preferred against supported languages
+//
+// A Matcher for an application that supports English, Australian English,
+// Danish, and standard Mandarin can be created as follows:
+//
+// var matcher = language.NewMatcher([]language.Tag{
+// language.English, // The first language is used as fallback.
+// language.MustParse("en-AU"),
+// language.Danish,
+// language.Chinese,
+// })
+//
+// This list of supported languages is typically implied by the languages for
+// which there exists translations of the user interface.
+//
+// User-preferred languages usually come as a comma-separated list of BCP 47
+// language tags.
+// The MatchString finds best matches for such strings:
+//
+// handler(w http.ResponseWriter, r *http.Request) {
+// lang, _ := r.Cookie("lang")
+// accept := r.Header.Get("Accept-Language")
+// tag, _ := language.MatchStrings(matcher, lang.String(), accept)
+//
+// // tag should now be used for the initialization of any
+// // locale-specific service.
+// }
+//
+// The Matcher's Match method can be used to match Tags directly.
+//
+// Matchers are aware of the intricacies of equivalence between languages, such
+// as deprecated subtags, legacy tags, macro languages, mutual
+// intelligibility between scripts and languages, and transparently passing
+// BCP 47 user configuration.
+// For instance, it will know that a reader of Bokmål Danish can read Norwegian
+// and will know that Cantonese ("yue") is a good match for "zh-HK".
+//
+//
+// Using match results
+//
+// To guarantee a consistent user experience to the user it is important to
+// use the same language tag for the selection of any locale-specific services.
+// For example, it is utterly confusing to substitute spelled-out numbers
+// or dates in one language in text of another language.
+// More subtly confusing is using the wrong sorting order or casing
+// algorithm for a certain language.
+//
+// All the packages in x/text that provide locale-specific services
+// (e.g. collate, cases) should be initialized with the tag that was
+// obtained at the start of an interaction with the user.
+//
+// Note that Tag that is returned by Match and MatchString may differ from any
+// of the supported languages, as it may contain carried over settings from
+// the user tags.
+// This may be inconvenient when your application has some additional
+// locale-specific data for your supported languages.
+// Match and MatchString both return the index of the matched supported tag
+// to simplify associating such data with the matched tag.
+//
+//
+// Canonicalization
+//
+// If one uses the Matcher to compare languages one does not need to
+// worry about canonicalization.
+//
+// The meaning of a Tag varies per application. The language package
+// therefore delays canonicalization and preserves information as much
+// as possible. The Matcher, however, will always take into account that
+// two different tags may represent the same language.
+//
+// By default, only legacy and deprecated tags are converted into their
+// canonical equivalent. All other information is preserved. This approach makes
+// the confidence scores more accurate and allows matchers to distinguish
+// between variants that are otherwise lost.
+//
+// As a consequence, two tags that should be treated as identical according to
+// BCP 47 or CLDR, like "en-Latn" and "en", will be represented differently. The
+// Matcher handles such distinctions, though, and is aware of the
+// equivalence relations. The CanonType type can be used to alter the
+// canonicalization form.
+//
+// References
+//
+// BCP 47 - Tags for Identifying Languages http://tools.ietf.org/html/bcp47
+//
+package language // import "golang.org/x/text/language/internal"
+
+// TODO: explanation on how to match languages for your own locale-specific
+// service.
diff --git a/language/internal/gen.go b/language/internal/gen.go
new file mode 100644
index 0000000..302f194
--- /dev/null
+++ b/language/internal/gen.go
@@ -0,0 +1,1712 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// Language tag table generator.
+// Data read from the web.
+
+package main
+
+import (
+ "bufio"
+ "flag"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "log"
+ "math"
+ "reflect"
+ "regexp"
+ "sort"
+ "strconv"
+ "strings"
+
+ "golang.org/x/text/internal/gen"
+ "golang.org/x/text/internal/tag"
+ "golang.org/x/text/unicode/cldr"
+)
+
+var (
+ test = flag.Bool("test",
+ false,
+ "test existing tables; can be used to compare web data with package data.")
+ outputFile = flag.String("output",
+ "tables.go",
+ "output file for generated tables")
+)
+
+var comment = []string{
+ `
+lang holds an alphabetically sorted list of ISO-639 language identifiers.
+All entries are 4 bytes. The index of the identifier (divided by 4) is the language tag.
+For 2-byte language identifiers, the two successive bytes have the following meaning:
+ - if the first letter of the 2- and 3-letter ISO codes are the same:
+ the second and third letter of the 3-letter ISO code.
+ - otherwise: a 0 and a by 2 bits right-shifted index into altLangISO3.
+For 3-byte language identifiers the 4th byte is 0.`,
+ `
+langNoIndex is a bit vector of all 3-letter language codes that are not used as an index
+in lookup tables. The language ids for these language codes are derived directly
+from the letters and are not consecutive.`,
+ `
+altLangISO3 holds an alphabetically sorted list of 3-letter language code alternatives
+to 2-letter language codes that cannot be derived using the method described above.
+Each 3-letter code is followed by its 1-byte langID.`,
+ `
+altLangIndex is used to convert indexes in altLangISO3 to langIDs.`,
+ `
+langAliasMap maps langIDs to their suggested replacements.`,
+ `
+script is an alphabetically sorted list of ISO 15924 codes. The index
+of the script in the string, divided by 4, is the internal scriptID.`,
+ `
+isoRegionOffset needs to be added to the index of regionISO to obtain the regionID
+for 2-letter ISO codes. (The first isoRegionOffset regionIDs are reserved for
+the UN.M49 codes used for groups.)`,
+ `
+regionISO holds a list of alphabetically sorted 2-letter ISO region codes.
+Each 2-letter codes is followed by two bytes with the following meaning:
+ - [A-Z}{2}: the first letter of the 2-letter code plus these two
+ letters form the 3-letter ISO code.
+ - 0, n: index into altRegionISO3.`,
+ `
+regionTypes defines the status of a region for various standards.`,
+ `
+m49 maps regionIDs to UN.M49 codes. The first isoRegionOffset entries are
+codes indicating collections of regions.`,
+ `
+m49Index gives indexes into fromM49 based on the three most significant bits
+of a 10-bit UN.M49 code. To search an UN.M49 code in fromM49, search in
+ fromM49[m49Index[msb39(code)]:m49Index[msb3(code)+1]]
+for an entry where the first 7 bits match the 7 lsb of the UN.M49 code.
+The region code is stored in the 9 lsb of the indexed value.`,
+ `
+fromM49 contains entries to map UN.M49 codes to regions. See m49Index for details.`,
+ `
+altRegionISO3 holds a list of 3-letter region codes that cannot be
+mapped to 2-letter codes using the default algorithm. This is a short list.`,
+ `
+altRegionIDs holds a list of regionIDs the positions of which match those
+of the 3-letter ISO codes in altRegionISO3.`,
+ `
+variantNumSpecialized is the number of specialized variants in variants.`,
+ `
+suppressScript is an index from langID to the dominant script for that language,
+if it exists. If a script is given, it should be suppressed from the language tag.`,
+ `
+likelyLang is a lookup table, indexed by langID, for the most likely
+scripts and regions given incomplete information. If more entries exist for a
+given language, region and script are the index and size respectively
+of the list in likelyLangList.`,
+ `
+likelyLangList holds lists info associated with likelyLang.`,
+ `
+likelyRegion is a lookup table, indexed by regionID, for the most likely
+languages and scripts given incomplete information. If more entries exist
+for a given regionID, lang and script are the index and size respectively
+of the list in likelyRegionList.
+TODO: exclude containers and user-definable regions from the list.`,
+ `
+likelyRegionList holds lists info associated with likelyRegion.`,
+ `
+likelyScript is a lookup table, indexed by scriptID, for the most likely
+languages and regions given a script.`,
+ `
+matchLang holds pairs of langIDs of base languages that are typically
+mutually intelligible. Each pair is associated with a confidence and
+whether the intelligibility goes one or both ways.`,
+ `
+matchScript holds pairs of scriptIDs where readers of one script
+can typically also read the other. Each is associated with a confidence.`,
+ `
+nRegionGroups is the number of region groups.`,
+ `
+regionInclusion maps region identifiers to sets of regions in regionInclusionBits,
+where each set holds all groupings that are directly connected in a region
+containment graph.`,
+ `
+regionInclusionBits is an array of bit vectors where every vector represents
+a set of region groupings. These sets are used to compute the distance
+between two regions for the purpose of language matching.`,
+ `
+regionInclusionNext marks, for each entry in regionInclusionBits, the set of
+all groups that are reachable from the groups set in the respective entry.`,
+}
+
+// TODO: consider changing some of these structures to tries. This can reduce
+// memory, but may increase the need for memory allocations. This could be
+// mitigated if we can piggyback on language tags for common cases.
+
+func failOnError(e error) {
+ if e != nil {
+ log.Panic(e)
+ }
+}
+
+type setType int
+
+const (
+ Indexed setType = 1 + iota // all elements must be of same size
+ Linear
+)
+
+type stringSet struct {
+ s []string
+ sorted, frozen bool
+
+ // We often need to update values after the creation of an index is completed.
+ // We include a convenience map for keeping track of this.
+ update map[string]string
+ typ setType // used for checking.
+}
+
+func (ss *stringSet) clone() stringSet {
+ c := *ss
+ c.s = append([]string(nil), c.s...)
+ return c
+}
+
+func (ss *stringSet) setType(t setType) {
+ if ss.typ != t && ss.typ != 0 {
+ log.Panicf("type %d cannot be assigned as it was already %d", t, ss.typ)
+ }
+}
+
+// parse parses a whitespace-separated string and initializes ss with its
+// components.
+func (ss *stringSet) parse(s string) {
+ scan := bufio.NewScanner(strings.NewReader(s))
+ scan.Split(bufio.ScanWords)
+ for scan.Scan() {
+ ss.add(scan.Text())
+ }
+}
+
+func (ss *stringSet) assertChangeable() {
+ if ss.frozen {
+ log.Panic("attempt to modify a frozen stringSet")
+ }
+}
+
+func (ss *stringSet) add(s string) {
+ ss.assertChangeable()
+ ss.s = append(ss.s, s)
+ ss.sorted = ss.frozen
+}
+
+func (ss *stringSet) freeze() {
+ ss.compact()
+ ss.frozen = true
+}
+
+func (ss *stringSet) compact() {
+ if ss.sorted {
+ return
+ }
+ a := ss.s
+ sort.Strings(a)
+ k := 0
+ for i := 1; i < len(a); i++ {
+ if a[k] != a[i] {
+ a[k+1] = a[i]
+ k++
+ }
+ }
+ ss.s = a[:k+1]
+ ss.sorted = ss.frozen
+}
+
+type funcSorter struct {
+ fn func(a, b string) bool
+ sort.StringSlice
+}
+
+func (s funcSorter) Less(i, j int) bool {
+ return s.fn(s.StringSlice[i], s.StringSlice[j])
+}
+
+func (ss *stringSet) sortFunc(f func(a, b string) bool) {
+ ss.compact()
+ sort.Sort(funcSorter{f, sort.StringSlice(ss.s)})
+}
+
+func (ss *stringSet) remove(s string) {
+ ss.assertChangeable()
+ if i, ok := ss.find(s); ok {
+ copy(ss.s[i:], ss.s[i+1:])
+ ss.s = ss.s[:len(ss.s)-1]
+ }
+}
+
+func (ss *stringSet) replace(ol, nu string) {
+ ss.s[ss.index(ol)] = nu
+ ss.sorted = ss.frozen
+}
+
+func (ss *stringSet) index(s string) int {
+ ss.setType(Indexed)
+ i, ok := ss.find(s)
+ if !ok {
+ if i < len(ss.s) {
+ log.Panicf("find: item %q is not in list. Closest match is %q.", s, ss.s[i])
+ }
+ log.Panicf("find: item %q is not in list", s)
+
+ }
+ return i
+}
+
+func (ss *stringSet) find(s string) (int, bool) {
+ ss.compact()
+ i := sort.SearchStrings(ss.s, s)
+ return i, i != len(ss.s) && ss.s[i] == s
+}
+
+func (ss *stringSet) slice() []string {
+ ss.compact()
+ return ss.s
+}
+
+func (ss *stringSet) updateLater(v, key string) {
+ if ss.update == nil {
+ ss.update = map[string]string{}
+ }
+ ss.update[v] = key
+}
+
+// join joins the string and ensures that all entries are of the same length.
+func (ss *stringSet) join() string {
+ ss.setType(Indexed)
+ n := len(ss.s[0])
+ for _, s := range ss.s {
+ if len(s) != n {
+ log.Panicf("join: not all entries are of the same length: %q", s)
+ }
+ }
+ ss.s = append(ss.s, strings.Repeat("\xff", n))
+ return strings.Join(ss.s, "")
+}
+
+// ianaEntry holds information for an entry in the IANA Language Subtag Repository.
+// All types use the same entry.
+// See http://tools.ietf.org/html/bcp47#section-5.1 for a description of the various
+// fields.
+type ianaEntry struct {
+ typ string
+ description []string
+ scope string
+ added string
+ preferred string
+ deprecated string
+ suppressScript string
+ macro string
+ prefix []string
+}
+
+type builder struct {
+ w *gen.CodeWriter
+ hw io.Writer // MultiWriter for w and w.Hash
+ data *cldr.CLDR
+ supp *cldr.SupplementalData
+
+ // indices
+ locale stringSet // common locales
+ lang stringSet // canonical language ids (2 or 3 letter ISO codes) with data
+ langNoIndex stringSet // 3-letter ISO codes with no associated data
+ script stringSet // 4-letter ISO codes
+ region stringSet // 2-letter ISO or 3-digit UN M49 codes
+ variant stringSet // 4-8-alphanumeric variant code.
+
+ // Region codes that are groups with their corresponding group IDs.
+ groups map[int]index
+
+ // langInfo
+ registry map[string]*ianaEntry
+}
+
+type index uint
+
+func newBuilder(w *gen.CodeWriter) *builder {
+ r := gen.OpenCLDRCoreZip()
+ defer r.Close()
+ d := &cldr.Decoder{}
+ data, err := d.DecodeZip(r)
+ failOnError(err)
+ b := builder{
+ w: w,
+ hw: io.MultiWriter(w, w.Hash),
+ data: data,
+ supp: data.Supplemental(),
+ }
+ b.parseRegistry()
+ return &b
+}
+
+func (b *builder) parseRegistry() {
+ r := gen.OpenIANAFile("assignments/language-subtag-registry")
+ defer r.Close()
+ b.registry = make(map[string]*ianaEntry)
+
+ scan := bufio.NewScanner(r)
+ scan.Split(bufio.ScanWords)
+ var record *ianaEntry
+ for more := scan.Scan(); more; {
+ key := scan.Text()
+ more = scan.Scan()
+ value := scan.Text()
+ switch key {
+ case "Type:":
+ record = &ianaEntry{typ: value}
+ case "Subtag:", "Tag:":
+ if s := strings.SplitN(value, "..", 2); len(s) > 1 {
+ for a := s[0]; a <= s[1]; a = inc(a) {
+ b.addToRegistry(a, record)
+ }
+ } else {
+ b.addToRegistry(value, record)
+ }
+ case "Suppress-Script:":
+ record.suppressScript = value
+ case "Added:":
+ record.added = value
+ case "Deprecated:":
+ record.deprecated = value
+ case "Macrolanguage:":
+ record.macro = value
+ case "Preferred-Value:":
+ record.preferred = value
+ case "Prefix:":
+ record.prefix = append(record.prefix, value)
+ case "Scope:":
+ record.scope = value
+ case "Description:":
+ buf := []byte(value)
+ for more = scan.Scan(); more; more = scan.Scan() {
+ b := scan.Bytes()
+ if b[0] == '%' || b[len(b)-1] == ':' {
+ break
+ }
+ buf = append(buf, ' ')
+ buf = append(buf, b...)
+ }
+ record.description = append(record.description, string(buf))
+ continue
+ default:
+ continue
+ }
+ more = scan.Scan()
+ }
+ if scan.Err() != nil {
+ log.Panic(scan.Err())
+ }
+}
+
+func (b *builder) addToRegistry(key string, entry *ianaEntry) {
+ if info, ok := b.registry[key]; ok {
+ if info.typ != "language" || entry.typ != "extlang" {
+ log.Fatalf("parseRegistry: tag %q already exists", key)
+ }
+ } else {
+ b.registry[key] = entry
+ }
+}
+
+var commentIndex = make(map[string]string)
+
+func init() {
+ for _, s := range comment {
+ key := strings.TrimSpace(strings.SplitN(s, " ", 2)[0])
+ commentIndex[key] = s
+ }
+}
+
+func (b *builder) comment(name string) {
+ if s := commentIndex[name]; len(s) > 0 {
+ b.w.WriteComment(s)
+ } else {
+ fmt.Fprintln(b.w)
+ }
+}
+
+func (b *builder) pf(f string, x ...interface{}) {
+ fmt.Fprintf(b.hw, f, x...)
+ fmt.Fprint(b.hw, "\n")
+}
+
+func (b *builder) p(x ...interface{}) {
+ fmt.Fprintln(b.hw, x...)
+}
+
+func (b *builder) addSize(s int) {
+ b.w.Size += s
+ b.pf("// Size: %d bytes", s)
+}
+
+func (b *builder) writeConst(name string, x interface{}) {
+ b.comment(name)
+ b.w.WriteConst(name, x)
+}
+
+// writeConsts computes f(v) for all v in values and writes the results
+// as constants named _v to a single constant block.
+func (b *builder) writeConsts(f func(string) int, values ...string) {
+ b.pf("const (")
+ for _, v := range values {
+ b.pf("\t_%s = %v", v, f(v))
+ }
+ b.pf(")")
+}
+
+// writeType writes the type of the given value, which must be a struct.
+func (b *builder) writeType(value interface{}) {
+ b.comment(reflect.TypeOf(value).Name())
+ b.w.WriteType(value)
+}
+
+func (b *builder) writeSlice(name string, ss interface{}) {
+ b.writeSliceAddSize(name, 0, ss)
+}
+
+func (b *builder) writeSliceAddSize(name string, extraSize int, ss interface{}) {
+ b.comment(name)
+ b.w.Size += extraSize
+ v := reflect.ValueOf(ss)
+ t := v.Type().Elem()
+ b.pf("// Size: %d bytes, %d elements", v.Len()*int(t.Size())+extraSize, v.Len())
+
+ fmt.Fprintf(b.w, "var %s = ", name)
+ b.w.WriteArray(ss)
+ b.p()
+}
+
+type fromTo struct {
+ from, to uint16
+}
+
+func (b *builder) writeSortedMap(name string, ss *stringSet, index func(s string) uint16) {
+ ss.sortFunc(func(a, b string) bool {
+ return index(a) < index(b)
+ })
+ m := []fromTo{}
+ for _, s := range ss.s {
+ m = append(m, fromTo{index(s), index(ss.update[s])})
+ }
+ b.writeSlice(name, m)
+}
+
+const base = 'z' - 'a' + 1
+
+func strToInt(s string) uint {
+ v := uint(0)
+ for i := 0; i < len(s); i++ {
+ v *= base
+ v += uint(s[i] - 'a')
+ }
+ return v
+}
+
+// converts the given integer to the original ASCII string passed to strToInt.
+// len(s) must match the number of characters obtained.
+func intToStr(v uint, s []byte) {
+ for i := len(s) - 1; i >= 0; i-- {
+ s[i] = byte(v%base) + 'a'
+ v /= base
+ }
+}
+
+func (b *builder) writeBitVector(name string, ss []string) {
+ vec := make([]uint8, int(math.Ceil(math.Pow(base, float64(len(ss[0])))/8)))
+ for _, s := range ss {
+ v := strToInt(s)
+ vec[v/8] |= 1 << (v % 8)
+ }
+ b.writeSlice(name, vec)
+}
+
+// TODO: convert this type into a list or two-stage trie.
+func (b *builder) writeMapFunc(name string, m map[string]string, f func(string) uint16) {
+ b.comment(name)
+ v := reflect.ValueOf(m)
+ sz := v.Len() * (2 + int(v.Type().Key().Size()))
+ for _, k := range m {
+ sz += len(k)
+ }
+ b.addSize(sz)
+ keys := []string{}
+ b.pf(`var %s = map[string]uint16{`, name)
+ for k := range m {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ for _, k := range keys {
+ b.pf("\t%q: %v,", k, f(m[k]))
+ }
+ b.p("}")
+}
+
+func (b *builder) writeMap(name string, m interface{}) {
+ b.comment(name)
+ v := reflect.ValueOf(m)
+ sz := v.Len() * (2 + int(v.Type().Key().Size()) + int(v.Type().Elem().Size()))
+ b.addSize(sz)
+ f := strings.FieldsFunc(fmt.Sprintf("%#v", m), func(r rune) bool {
+ return strings.IndexRune("{}, ", r) != -1
+ })
+ sort.Strings(f[1:])
+ b.pf(`var %s = %s{`, name, f[0])
+ for _, kv := range f[1:] {
+ b.pf("\t%s,", kv)
+ }
+ b.p("}")
+}
+
+func (b *builder) langIndex(s string) uint16 {
+ if s == "und" {
+ return 0
+ }
+ if i, ok := b.lang.find(s); ok {
+ return uint16(i)
+ }
+ return uint16(strToInt(s)) + uint16(len(b.lang.s))
+}
+
+// inc advances the string to its lexicographical successor.
+func inc(s string) string {
+ const maxTagLength = 4
+ var buf [maxTagLength]byte
+ intToStr(strToInt(strings.ToLower(s))+1, buf[:len(s)])
+ for i := 0; i < len(s); i++ {
+ if s[i] <= 'Z' {
+ buf[i] -= 'a' - 'A'
+ }
+ }
+ return string(buf[:len(s)])
+}
+
+func (b *builder) parseIndices() {
+ meta := b.supp.Metadata
+
+ for k, v := range b.registry {
+ var ss *stringSet
+ switch v.typ {
+ case "language":
+ if len(k) == 2 || v.suppressScript != "" || v.scope == "special" {
+ b.lang.add(k)
+ continue
+ } else {
+ ss = &b.langNoIndex
+ }
+ case "region":
+ ss = &b.region
+ case "script":
+ ss = &b.script
+ case "variant":
+ ss = &b.variant
+ default:
+ continue
+ }
+ ss.add(k)
+ }
+ // Include any language for which there is data.
+ for _, lang := range b.data.Locales() {
+ if x := b.data.RawLDML(lang); false ||
+ x.LocaleDisplayNames != nil ||
+ x.Characters != nil ||
+ x.Delimiters != nil ||
+ x.Measurement != nil ||
+ x.Dates != nil ||
+ x.Numbers != nil ||
+ x.Units != nil ||
+ x.ListPatterns != nil ||
+ x.Collations != nil ||
+ x.Segmentations != nil ||
+ x.Rbnf != nil ||
+ x.Annotations != nil ||
+ x.Metadata != nil {
+
+ from := strings.Split(lang, "_")
+ if lang := from[0]; lang != "root" {
+ b.lang.add(lang)
+ }
+ }
+ }
+ // Include locales for plural rules, which uses a different structure.
+ for _, plurals := range b.data.Supplemental().Plurals {
+ for _, rules := range plurals.PluralRules {
+ for _, lang := range strings.Split(rules.Locales, " ") {
+ if lang = strings.Split(lang, "_")[0]; lang != "root" {
+ b.lang.add(lang)
+ }
+ }
+ }
+ }
+ // Include languages in likely subtags.
+ for _, m := range b.supp.LikelySubtags.LikelySubtag {
+ from := strings.Split(m.From, "_")
+ b.lang.add(from[0])
+ }
+ // Include ISO-639 alpha-3 bibliographic entries.
+ for _, a := range meta.Alias.LanguageAlias {
+ if a.Reason == "bibliographic" {
+ b.langNoIndex.add(a.Type)
+ }
+ }
+ // Include regions in territoryAlias (not all are in the IANA registry!)
+ for _, reg := range b.supp.Metadata.Alias.TerritoryAlias {
+ if len(reg.Type) == 2 {
+ b.region.add(reg.Type)
+ }
+ }
+
+ for _, s := range b.lang.s {
+ if len(s) == 3 {
+ b.langNoIndex.remove(s)
+ }
+ }
+ b.writeConst("numLanguages", len(b.lang.slice())+len(b.langNoIndex.slice()))
+ b.writeConst("numScripts", len(b.script.slice()))
+ b.writeConst("numRegions", len(b.region.slice()))
+
+ // Add dummy codes at the start of each list to represent "unspecified".
+ b.lang.add("---")
+ b.script.add("----")
+ b.region.add("---")
+
+ // common locales
+ b.locale.parse(meta.DefaultContent.Locales)
+}
+
+// TODO: region inclusion data will probably not be use used in future matchers.
+
+func (b *builder) computeRegionGroups() {
+ b.groups = make(map[int]index)
+
+ // Create group indices.
+ for i := 1; b.region.s[i][0] < 'A'; i++ { // Base M49 indices on regionID.
+ b.groups[i] = index(len(b.groups))
+ }
+ for _, g := range b.supp.TerritoryContainment.Group {
+ // Skip UN and EURO zone as they are flattening the containment
+ // relationship.
+ if g.Type == "EZ" || g.Type == "UN" {
+ continue
+ }
+ group := b.region.index(g.Type)
+ if _, ok := b.groups[group]; !ok {
+ b.groups[group] = index(len(b.groups))
+ }
+ }
+ if len(b.groups) > 64 {
+ log.Fatalf("only 64 groups supported, found %d", len(b.groups))
+ }
+ b.writeConst("nRegionGroups", len(b.groups))
+}
+
+var langConsts = []string{
+ "af", "am", "ar", "az", "bg", "bn", "ca", "cs", "da", "de", "el", "en", "es",
+ "et", "fa", "fi", "fil", "fr", "gu", "he", "hi", "hr", "hu", "hy", "id", "is",
+ "it", "ja", "ka", "kk", "km", "kn", "ko", "ky", "lo", "lt", "lv", "mk", "ml",
+ "mn", "mo", "mr", "ms", "mul", "my", "nb", "ne", "nl", "no", "pa", "pl", "pt",
+ "ro", "ru", "sh", "si", "sk", "sl", "sq", "sr", "sv", "sw", "ta", "te", "th",
+ "tl", "tn", "tr", "uk", "ur", "uz", "vi", "zh", "zu",
+
+ // constants for grandfathered tags (if not already defined)
+ "jbo", "ami", "bnn", "hak", "tlh", "lb", "nv", "pwn", "tao", "tay", "tsu",
+ "nn", "sfb", "vgt", "sgg", "cmn", "nan", "hsn",
+}
+
+// writeLanguage generates all tables needed for language canonicalization.
+func (b *builder) writeLanguage() {
+ meta := b.supp.Metadata
+
+ b.writeConst("nonCanonicalUnd", b.lang.index("und"))
+ b.writeConsts(func(s string) int { return int(b.langIndex(s)) }, langConsts...)
+ b.writeConst("langPrivateStart", b.langIndex("qaa"))
+ b.writeConst("langPrivateEnd", b.langIndex("qtz"))
+
+ // Get language codes that need to be mapped (overlong 3-letter codes,
+ // deprecated 2-letter codes, legacy and grandfathered tags.)
+ langAliasMap := stringSet{}
+ aliasTypeMap := map[string]langAliasType{}
+
+ // altLangISO3 get the alternative ISO3 names that need to be mapped.
+ altLangISO3 := stringSet{}
+ // Add dummy start to avoid the use of index 0.
+ altLangISO3.add("---")
+ altLangISO3.updateLater("---", "aa")
+
+ lang := b.lang.clone()
+ for _, a := range meta.Alias.LanguageAlias {
+ if a.Replacement == "" {
+ a.Replacement = "und"
+ }
+ // TODO: support mapping to tags
+ repl := strings.SplitN(a.Replacement, "_", 2)[0]
+ if a.Reason == "overlong" {
+ if len(a.Replacement) == 2 && len(a.Type) == 3 {
+ lang.updateLater(a.Replacement, a.Type)
+ }
+ } else if len(a.Type) <= 3 {
+ switch a.Reason {
+ case "macrolanguage":
+ aliasTypeMap[a.Type] = langMacro
+ case "deprecated":
+ // handled elsewhere
+ continue
+ case "bibliographic", "legacy":
+ if a.Type == "no" {
+ continue
+ }
+ aliasTypeMap[a.Type] = langLegacy
+ default:
+ log.Fatalf("new %s alias: %s", a.Reason, a.Type)
+ }
+ langAliasMap.add(a.Type)
+ langAliasMap.updateLater(a.Type, repl)
+ }
+ }
+ // Manually add the mapping of "nb" (Norwegian) to its macro language.
+ // This can be removed if CLDR adopts this change.
+ langAliasMap.add("nb")
+ langAliasMap.updateLater("nb", "no")
+ aliasTypeMap["nb"] = langMacro
+
+ for k, v := range b.registry {
+ // Also add deprecated values for 3-letter ISO codes, which CLDR omits.
+ if v.typ == "language" && v.deprecated != "" && v.preferred != "" {
+ langAliasMap.add(k)
+ langAliasMap.updateLater(k, v.preferred)
+ aliasTypeMap[k] = langDeprecated
+ }
+ }
+ // Fix CLDR mappings.
+ lang.updateLater("tl", "tgl")
+ lang.updateLater("sh", "hbs")
+ lang.updateLater("mo", "mol")
+ lang.updateLater("no", "nor")
+ lang.updateLater("tw", "twi")
+ lang.updateLater("nb", "nob")
+ lang.updateLater("ak", "aka")
+ lang.updateLater("bh", "bih")
+
+ // Ensure that each 2-letter code is matched with a 3-letter code.
+ for _, v := range lang.s[1:] {
+ s, ok := lang.update[v]
+ if !ok {
+ if s, ok = lang.update[langAliasMap.update[v]]; !ok {
+ continue
+ }
+ lang.update[v] = s
+ }
+ if v[0] != s[0] {
+ altLangISO3.add(s)
+ altLangISO3.updateLater(s, v)
+ }
+ }
+
+ // Complete canonicalized language tags.
+ lang.freeze()
+ for i, v := range lang.s {
+ // We can avoid these manual entries by using the IANA registry directly.
+ // Seems easier to update the list manually, as changes are rare.
+ // The panic in this loop will trigger if we miss an entry.
+ add := ""
+ if s, ok := lang.update[v]; ok {
+ if s[0] == v[0] {
+ add = s[1:]
+ } else {
+ add = string([]byte{0, byte(altLangISO3.index(s))})
+ }
+ } else if len(v) == 3 {
+ add = "\x00"
+ } else {
+ log.Panicf("no data for long form of %q", v)
+ }
+ lang.s[i] += add
+ }
+ b.writeConst("lang", tag.Index(lang.join()))
+
+ b.writeConst("langNoIndexOffset", len(b.lang.s))
+
+ // space of all valid 3-letter language identifiers.
+ b.writeBitVector("langNoIndex", b.langNoIndex.slice())
+
+ altLangIndex := []uint16{}
+ for i, s := range altLangISO3.slice() {
+ altLangISO3.s[i] += string([]byte{byte(len(altLangIndex))})
+ if i > 0 {
+ idx := b.lang.index(altLangISO3.update[s])
+ altLangIndex = append(altLangIndex, uint16(idx))
+ }
+ }
+ b.writeConst("altLangISO3", tag.Index(altLangISO3.join()))
+ b.writeSlice("altLangIndex", altLangIndex)
+
+ b.writeSortedMap("langAliasMap", &langAliasMap, b.langIndex)
+ types := make([]langAliasType, len(langAliasMap.s))
+ for i, s := range langAliasMap.s {
+ types[i] = aliasTypeMap[s]
+ }
+ b.writeSlice("langAliasTypes", types)
+}
+
+var scriptConsts = []string{
+ "Latn", "Hani", "Hans", "Hant", "Qaaa", "Qaai", "Qabx", "Zinh", "Zyyy",
+ "Zzzz",
+}
+
+func (b *builder) writeScript() {
+ b.writeConsts(b.script.index, scriptConsts...)
+ b.writeConst("script", tag.Index(b.script.join()))
+
+ supp := make([]uint8, len(b.lang.slice()))
+ for i, v := range b.lang.slice()[1:] {
+ if sc := b.registry[v].suppressScript; sc != "" {
+ supp[i+1] = uint8(b.script.index(sc))
+ }
+ }
+ b.writeSlice("suppressScript", supp)
+
+ // There is only one deprecated script in CLDR. This value is hard-coded.
+ // We check here if the code must be updated.
+ for _, a := range b.supp.Metadata.Alias.ScriptAlias {
+ if a.Type != "Qaai" {
+ log.Panicf("unexpected deprecated stript %q", a.Type)
+ }
+ }
+}
+
+func parseM49(s string) int16 {
+ if len(s) == 0 {
+ return 0
+ }
+ v, err := strconv.ParseUint(s, 10, 10)
+ failOnError(err)
+ return int16(v)
+}
+
+var regionConsts = []string{
+ "001", "419", "BR", "CA", "ES", "GB", "MD", "PT", "UK", "US",
+ "ZZ", "XA", "XC", "XK", // Unofficial tag for Kosovo.
+}
+
+func (b *builder) writeRegion() {
+ b.writeConsts(b.region.index, regionConsts...)
+
+ isoOffset := b.region.index("AA")
+ m49map := make([]int16, len(b.region.slice()))
+ fromM49map := make(map[int16]int)
+ altRegionISO3 := ""
+ altRegionIDs := []uint16{}
+
+ b.writeConst("isoRegionOffset", isoOffset)
+
+ // 2-letter region lookup and mapping to numeric codes.
+ regionISO := b.region.clone()
+ regionISO.s = regionISO.s[isoOffset:]
+ regionISO.sorted = false
+
+ regionTypes := make([]byte, len(b.region.s))
+
+ // Is the region valid BCP 47?
+ for s, e := range b.registry {
+ if len(s) == 2 && s == strings.ToUpper(s) {
+ i := b.region.index(s)
+ for _, d := range e.description {
+ if strings.Contains(d, "Private use") {
+ regionTypes[i] = iso3166UserAssigned
+ }
+ }
+ regionTypes[i] |= bcp47Region
+ }
+ }
+
+ // Is the region a valid ccTLD?
+ r := gen.OpenIANAFile("domains/root/db")
+ defer r.Close()
+
+ buf, err := ioutil.ReadAll(r)
+ failOnError(err)
+ re := regexp.MustCompile(`"/domains/root/db/([a-z]{2}).html"`)
+ for _, m := range re.FindAllSubmatch(buf, -1) {
+ i := b.region.index(strings.ToUpper(string(m[1])))
+ regionTypes[i] |= ccTLD
+ }
+
+ b.writeSlice("regionTypes", regionTypes)
+
+ iso3Set := make(map[string]int)
+ update := func(iso2, iso3 string) {
+ i := regionISO.index(iso2)
+ if j, ok := iso3Set[iso3]; !ok && iso3[0] == iso2[0] {
+ regionISO.s[i] += iso3[1:]
+ iso3Set[iso3] = -1
+ } else {
+ if ok && j >= 0 {
+ regionISO.s[i] += string([]byte{0, byte(j)})
+ } else {
+ iso3Set[iso3] = len(altRegionISO3)
+ regionISO.s[i] += string([]byte{0, byte(len(altRegionISO3))})
+ altRegionISO3 += iso3
+ altRegionIDs = append(altRegionIDs, uint16(isoOffset+i))
+ }
+ }
+ }
+ for _, tc := range b.supp.CodeMappings.TerritoryCodes {
+ i := regionISO.index(tc.Type) + isoOffset
+ if d := m49map[i]; d != 0 {
+ log.Panicf("%s found as a duplicate UN.M49 code of %03d", tc.Numeric, d)
+ }
+ m49 := parseM49(tc.Numeric)
+ m49map[i] = m49
+ if r := fromM49map[m49]; r == 0 {
+ fromM49map[m49] = i
+ } else if r != i {
+ dep := b.registry[regionISO.s[r-isoOffset]].deprecated
+ if t := b.registry[tc.Type]; t != nil && dep != "" && (t.deprecated == "" || t.deprecated > dep) {
+ fromM49map[m49] = i
+ }
+ }
+ }
+ for _, ta := range b.supp.Metadata.Alias.TerritoryAlias {
+ if len(ta.Type) == 3 && ta.Type[0] <= '9' && len(ta.Replacement) == 2 {
+ from := parseM49(ta.Type)
+ if r := fromM49map[from]; r == 0 {
+ fromM49map[from] = regionISO.index(ta.Replacement) + isoOffset
+ }
+ }
+ }
+ for _, tc := range b.supp.CodeMappings.TerritoryCodes {
+ if len(tc.Alpha3) == 3 {
+ update(tc.Type, tc.Alpha3)
+ }
+ }
+ // This entries are not included in territoryCodes. Mostly 3-letter variants
+ // of deleted codes and an entry for QU.
+ for _, m := range []struct{ iso2, iso3 string }{
+ {"CT", "CTE"},
+ {"DY", "DHY"},
+ {"HV", "HVO"},
+ {"JT", "JTN"},
+ {"MI", "MID"},
+ {"NH", "NHB"},
+ {"NQ", "ATN"},
+ {"PC", "PCI"},
+ {"PU", "PUS"},
+ {"PZ", "PCZ"},
+ {"RH", "RHO"},
+ {"VD", "VDR"},
+ {"WK", "WAK"},
+ // These three-letter codes are used for others as well.
+ {"FQ", "ATF"},
+ } {
+ update(m.iso2, m.iso3)
+ }
+ for i, s := range regionISO.s {
+ if len(s) != 4 {
+ regionISO.s[i] = s + " "
+ }
+ }
+ b.writeConst("regionISO", tag.Index(regionISO.join()))
+ b.writeConst("altRegionISO3", altRegionISO3)
+ b.writeSlice("altRegionIDs", altRegionIDs)
+
+ // Create list of deprecated regions.
+ // TODO: consider inserting SF -> FI. Not included by CLDR, but is the only
+ // Transitionally-reserved mapping not included.
+ regionOldMap := stringSet{}
+ // Include regions in territoryAlias (not all are in the IANA registry!)
+ for _, reg := range b.supp.Metadata.Alias.TerritoryAlias {
+ if len(reg.Type) == 2 && reg.Reason == "deprecated" && len(reg.Replacement) == 2 {
+ regionOldMap.add(reg.Type)
+ regionOldMap.updateLater(reg.Type, reg.Replacement)
+ i, _ := regionISO.find(reg.Type)
+ j, _ := regionISO.find(reg.Replacement)
+ if k := m49map[i+isoOffset]; k == 0 {
+ m49map[i+isoOffset] = m49map[j+isoOffset]
+ }
+ }
+ }
+ b.writeSortedMap("regionOldMap", ®ionOldMap, func(s string) uint16 {
+ return uint16(b.region.index(s))
+ })
+ // 3-digit region lookup, groupings.
+ for i := 1; i < isoOffset; i++ {
+ m := parseM49(b.region.s[i])
+ m49map[i] = m
+ fromM49map[m] = i
+ }
+ b.writeSlice("m49", m49map)
+
+ const (
+ searchBits = 7
+ regionBits = 9
+ )
+ if len(m49map) >= 1<<regionBits {
+ log.Fatalf("Maximum number of regions exceeded: %d > %d", len(m49map), 1<<regionBits)
+ }
+ m49Index := [9]int16{}
+ fromM49 := []uint16{}
+ m49 := []int{}
+ for k, _ := range fromM49map {
+ m49 = append(m49, int(k))
+ }
+ sort.Ints(m49)
+ for _, k := range m49[1:] {
+ val := (k & (1<<searchBits - 1)) << regionBits
+ fromM49 = append(fromM49, uint16(val|fromM49map[int16(k)]))
+ m49Index[1:][k>>searchBits] = int16(len(fromM49))
+ }
+ b.writeSlice("m49Index", m49Index)
+ b.writeSlice("fromM49", fromM49)
+}
+
+const (
+ // TODO: put these lists in regionTypes as user data? Could be used for
+ // various optimizations and refinements and could be exposed in the API.
+ iso3166Except = "AC CP DG EA EU FX IC SU TA UK"
+ iso3166Trans = "AN BU CS NT TP YU ZR" // SF is not in our set of Regions.
+ // DY and RH are actually not deleted, but indeterminately reserved.
+ iso3166DelCLDR = "CT DD DY FQ HV JT MI NH NQ PC PU PZ RH VD WK YD"
+)
+
+const (
+ iso3166UserAssigned = 1 << iota
+ ccTLD
+ bcp47Region
+)
+
+func find(list []string, s string) int {
+ for i, t := range list {
+ if t == s {
+ return i
+ }
+ }
+ return -1
+}
+
+// writeVariants generates per-variant information and creates a map from variant
+// name to index value. We assign index values such that sorting multiple
+// variants by index value will result in the correct order.
+// There are two types of variants: specialized and general. Specialized variants
+// are only applicable to certain language or language-script pairs. Generalized
+// variants apply to any language. Generalized variants always sort after
+// specialized variants. We will therefore always assign a higher index value
+// to a generalized variant than any other variant. Generalized variants are
+// sorted alphabetically among themselves.
+// Specialized variants may also sort after other specialized variants. Such
+// variants will be ordered after any of the variants they may follow.
+// We assume that if a variant x is followed by a variant y, then for any prefix
+// p of x, p-x is a prefix of y. This allows us to order tags based on the
+// maximum of the length of any of its prefixes.
+// TODO: it is possible to define a set of Prefix values on variants such that
+// a total order cannot be defined to the point that this algorithm breaks.
+// In other words, we cannot guarantee the same order of variants for the
+// future using the same algorithm or for non-compliant combinations of
+// variants. For this reason, consider using simple alphabetic sorting
+// of variants and ignore Prefix restrictions altogether.
+func (b *builder) writeVariant() {
+ generalized := stringSet{}
+ specialized := stringSet{}
+ specializedExtend := stringSet{}
+ // Collate the variants by type and check assumptions.
+ for _, v := range b.variant.slice() {
+ e := b.registry[v]
+ if len(e.prefix) == 0 {
+ generalized.add(v)
+ continue
+ }
+ c := strings.Split(e.prefix[0], "-")
+ hasScriptOrRegion := false
+ if len(c) > 1 {
+ _, hasScriptOrRegion = b.script.find(c[1])
+ if !hasScriptOrRegion {
+ _, hasScriptOrRegion = b.region.find(c[1])
+
+ }
+ }
+ if len(c) == 1 || len(c) == 2 && hasScriptOrRegion {
+ // Variant is preceded by a language.
+ specialized.add(v)
+ continue
+ }
+ // Variant is preceded by another variant.
+ specializedExtend.add(v)
+ prefix := c[0] + "-"
+ if hasScriptOrRegion {
+ prefix += c[1]
+ }
+ for _, p := range e.prefix {
+ // Verify that the prefix minus the last element is a prefix of the
+ // predecessor element.
+ i := strings.LastIndex(p, "-")
+ pred := b.registry[p[i+1:]]
+ if find(pred.prefix, p[:i]) < 0 {
+ log.Fatalf("prefix %q for variant %q not consistent with predecessor spec", p, v)
+ }
+ // The sorting used below does not work in the general case. It works
+ // if we assume that variants that may be followed by others only have
+ // prefixes of the same length. Verify this.
+ count := strings.Count(p[:i], "-")
+ for _, q := range pred.prefix {
+ if c := strings.Count(q, "-"); c != count {
+ log.Fatalf("variant %q preceding %q has a prefix %q of size %d; want %d", p[i+1:], v, q, c, count)
+ }
+ }
+ if !strings.HasPrefix(p, prefix) {
+ log.Fatalf("prefix %q of variant %q should start with %q", p, v, prefix)
+ }
+ }
+ }
+
+ // Sort extended variants.
+ a := specializedExtend.s
+ less := func(v, w string) bool {
+ // Sort by the maximum number of elements.
+ maxCount := func(s string) (max int) {
+ for _, p := range b.registry[s].prefix {
+ if c := strings.Count(p, "-"); c > max {
+ max = c
+ }
+ }
+ return
+ }
+ if cv, cw := maxCount(v), maxCount(w); cv != cw {
+ return cv < cw
+ }
+ // Sort by name as tie breaker.
+ return v < w
+ }
+ sort.Sort(funcSorter{less, sort.StringSlice(a)})
+ specializedExtend.frozen = true
+
+ // Create index from variant name to index.
+ variantIndex := make(map[string]uint8)
+ add := func(s []string) {
+ for _, v := range s {
+ variantIndex[v] = uint8(len(variantIndex))
+ }
+ }
+ add(specialized.slice())
+ add(specializedExtend.s)
+ numSpecialized := len(variantIndex)
+ add(generalized.slice())
+ if n := len(variantIndex); n > 255 {
+ log.Fatalf("maximum number of variants exceeded: was %d; want <= 255", n)
+ }
+ b.writeMap("variantIndex", variantIndex)
+ b.writeConst("variantNumSpecialized", numSpecialized)
+}
+
+func (b *builder) writeLanguageInfo() {
+}
+
+// writeLikelyData writes tables that are used both for finding parent relations and for
+// language matching. Each entry contains additional bits to indicate the status of the
+// data to know when it cannot be used for parent relations.
+func (b *builder) writeLikelyData() {
+ const (
+ isList = 1 << iota
+ scriptInFrom
+ regionInFrom
+ )
+ type ( // generated types
+ likelyScriptRegion struct {
+ region uint16
+ script uint8
+ flags uint8
+ }
+ likelyLangScript struct {
+ lang uint16
+ script uint8
+ flags uint8
+ }
+ likelyLangRegion struct {
+ lang uint16
+ region uint16
+ }
+ // likelyTag is used for getting likely tags for group regions, where
+ // the likely region might be a region contained in the group.
+ likelyTag struct {
+ lang uint16
+ region uint16
+ script uint8
+ }
+ )
+ var ( // generated variables
+ likelyRegionGroup = make([]likelyTag, len(b.groups))
+ likelyLang = make([]likelyScriptRegion, len(b.lang.s))
+ likelyRegion = make([]likelyLangScript, len(b.region.s))
+ likelyScript = make([]likelyLangRegion, len(b.script.s))
+ likelyLangList = []likelyScriptRegion{}
+ likelyRegionList = []likelyLangScript{}
+ )
+ type fromTo struct {
+ from, to []string
+ }
+ langToOther := map[int][]fromTo{}
+ regionToOther := map[int][]fromTo{}
+ for _, m := range b.supp.LikelySubtags.LikelySubtag {
+ from := strings.Split(m.From, "_")
+ to := strings.Split(m.To, "_")
+ if len(to) != 3 {
+ log.Fatalf("invalid number of subtags in %q: found %d, want 3", m.To, len(to))
+ }
+ if len(from) > 3 {
+ log.Fatalf("invalid number of subtags: found %d, want 1-3", len(from))
+ }
+ if from[0] != to[0] && from[0] != "und" {
+ log.Fatalf("unexpected language change in expansion: %s -> %s", from, to)
+ }
+ if len(from) == 3 {
+ if from[2] != to[2] {
+ log.Fatalf("unexpected region change in expansion: %s -> %s", from, to)
+ }
+ if from[0] != "und" {
+ log.Fatalf("unexpected fully specified from tag: %s -> %s", from, to)
+ }
+ }
+ if len(from) == 1 || from[0] != "und" {
+ id := 0
+ if from[0] != "und" {
+ id = b.lang.index(from[0])
+ }
+ langToOther[id] = append(langToOther[id], fromTo{from, to})
+ } else if len(from) == 2 && len(from[1]) == 4 {
+ sid := b.script.index(from[1])
+ likelyScript[sid].lang = uint16(b.langIndex(to[0]))
+ likelyScript[sid].region = uint16(b.region.index(to[2]))
+ } else {
+ r := b.region.index(from[len(from)-1])
+ if id, ok := b.groups[r]; ok {
+ if from[0] != "und" {
+ log.Fatalf("region changed unexpectedly: %s -> %s", from, to)
+ }
+ likelyRegionGroup[id].lang = uint16(b.langIndex(to[0]))
+ likelyRegionGroup[id].script = uint8(b.script.index(to[1]))
+ likelyRegionGroup[id].region = uint16(b.region.index(to[2]))
+ } else {
+ regionToOther[r] = append(regionToOther[r], fromTo{from, to})
+ }
+ }
+ }
+ b.writeType(likelyLangRegion{})
+ b.writeSlice("likelyScript", likelyScript)
+
+ for id := range b.lang.s {
+ list := langToOther[id]
+ if len(list) == 1 {
+ likelyLang[id].region = uint16(b.region.index(list[0].to[2]))
+ likelyLang[id].script = uint8(b.script.index(list[0].to[1]))
+ } else if len(list) > 1 {
+ likelyLang[id].flags = isList
+ likelyLang[id].region = uint16(len(likelyLangList))
+ likelyLang[id].script = uint8(len(list))
+ for _, x := range list {
+ flags := uint8(0)
+ if len(x.from) > 1 {
+ if x.from[1] == x.to[2] {
+ flags = regionInFrom
+ } else {
+ flags = scriptInFrom
+ }
+ }
+ likelyLangList = append(likelyLangList, likelyScriptRegion{
+ region: uint16(b.region.index(x.to[2])),
+ script: uint8(b.script.index(x.to[1])),
+ flags: flags,
+ })
+ }
+ }
+ }
+ // TODO: merge suppressScript data with this table.
+ b.writeType(likelyScriptRegion{})
+ b.writeSlice("likelyLang", likelyLang)
+ b.writeSlice("likelyLangList", likelyLangList)
+
+ for id := range b.region.s {
+ list := regionToOther[id]
+ if len(list) == 1 {
+ likelyRegion[id].lang = uint16(b.langIndex(list[0].to[0]))
+ likelyRegion[id].script = uint8(b.script.index(list[0].to[1]))
+ if len(list[0].from) > 2 {
+ likelyRegion[id].flags = scriptInFrom
+ }
+ } else if len(list) > 1 {
+ likelyRegion[id].flags = isList
+ likelyRegion[id].lang = uint16(len(likelyRegionList))
+ likelyRegion[id].script = uint8(len(list))
+ for i, x := range list {
+ if len(x.from) == 2 && i != 0 || i > 0 && len(x.from) != 3 {
+ log.Fatalf("unspecified script must be first in list: %v at %d", x.from, i)
+ }
+ x := likelyLangScript{
+ lang: uint16(b.langIndex(x.to[0])),
+ script: uint8(b.script.index(x.to[1])),
+ }
+ if len(list[0].from) > 2 {
+ x.flags = scriptInFrom
+ }
+ likelyRegionList = append(likelyRegionList, x)
+ }
+ }
+ }
+ b.writeType(likelyLangScript{})
+ b.writeSlice("likelyRegion", likelyRegion)
+ b.writeSlice("likelyRegionList", likelyRegionList)
+
+ b.writeType(likelyTag{})
+ b.writeSlice("likelyRegionGroup", likelyRegionGroup)
+}
+
+type mutualIntelligibility struct {
+ want, have uint16
+ distance uint8
+ oneway bool
+}
+
+type scriptIntelligibility struct {
+ wantLang, haveLang uint16
+ wantScript, haveScript uint8
+ distance uint8
+ // Always oneway
+}
+
+type regionIntelligibility struct {
+ lang uint16 // compact language id
+ script uint8 // 0 means any
+ group uint8 // 0 means any; if bit 7 is set it means inverse
+ distance uint8
+ // Always twoway.
+}
+
+// writeMatchData writes tables with languages and scripts for which there is
+// mutual intelligibility. The data is based on CLDR's languageMatching data.
+// Note that we use a different algorithm than the one defined by CLDR and that
+// we slightly modify the data. For example, we convert scores to confidence levels.
+// We also drop all region-related data as we use a different algorithm to
+// determine region equivalence.
+func (b *builder) writeMatchData() {
+ lm := b.supp.LanguageMatching.LanguageMatches
+ cldr.MakeSlice(&lm).SelectAnyOf("type", "written_new")
+
+ regionHierarchy := map[string][]string{}
+ for _, g := range b.supp.TerritoryContainment.Group {
+ regions := strings.Split(g.Contains, " ")
+ regionHierarchy[g.Type] = append(regionHierarchy[g.Type], regions...)
+ }
+ regionToGroups := make([]uint8, len(b.region.s))
+
+ idToIndex := map[string]uint8{}
+ for i, mv := range lm[0].MatchVariable {
+ if i > 6 {
+ log.Fatalf("Too many groups: %d", i)
+ }
+ idToIndex[mv.Id] = uint8(i + 1)
+ // TODO: also handle '-'
+ for _, r := range strings.Split(mv.Value, "+") {
+ todo := []string{r}
+ for k := 0; k < len(todo); k++ {
+ r := todo[k]
+ regionToGroups[b.region.index(r)] |= 1 << uint8(i)
+ todo = append(todo, regionHierarchy[r]...)
+ }
+ }
+ }
+ b.writeSlice("regionToGroups", regionToGroups)
+
+ // maps language id to in- and out-of-group region.
+ paradigmLocales := [][3]uint16{}
+ locales := strings.Split(lm[0].ParadigmLocales[0].Locales, " ")
+ for i := 0; i < len(locales); i += 2 {
+ x := [3]uint16{}
+ for j := 0; j < 2; j++ {
+ pc := strings.SplitN(locales[i+j], "-", 2)
+ x[0] = b.langIndex(pc[0])
+ if len(pc) == 2 {
+ x[1+j] = uint16(b.region.index(pc[1]))
+ }
+ }
+ paradigmLocales = append(paradigmLocales, x)
+ }
+ b.writeSlice("paradigmLocales", paradigmLocales)
+
+ b.writeType(mutualIntelligibility{})
+ b.writeType(scriptIntelligibility{})
+ b.writeType(regionIntelligibility{})
+
+ matchLang := []mutualIntelligibility{}
+ matchScript := []scriptIntelligibility{}
+ matchRegion := []regionIntelligibility{}
+ // Convert the languageMatch entries in lists keyed by desired language.
+ for _, m := range lm[0].LanguageMatch {
+ // Different versions of CLDR use different separators.
+ desired := strings.Replace(m.Desired, "-", "_", -1)
+ supported := strings.Replace(m.Supported, "-", "_", -1)
+ d := strings.Split(desired, "_")
+ s := strings.Split(supported, "_")
+ if len(d) != len(s) {
+ log.Fatalf("not supported: desired=%q; supported=%q", desired, supported)
+ continue
+ }
+ distance, _ := strconv.ParseInt(m.Distance, 10, 8)
+ switch len(d) {
+ case 2:
+ if desired == supported && desired == "*_*" {
+ continue
+ }
+ // language-script pair.
+ matchScript = append(matchScript, scriptIntelligibility{
+ wantLang: uint16(b.langIndex(d[0])),
+ haveLang: uint16(b.langIndex(s[0])),
+ wantScript: uint8(b.script.index(d[1])),
+ haveScript: uint8(b.script.index(s[1])),
+ distance: uint8(distance),
+ })
+ if m.Oneway != "true" {
+ matchScript = append(matchScript, scriptIntelligibility{
+ wantLang: uint16(b.langIndex(s[0])),
+ haveLang: uint16(b.langIndex(d[0])),
+ wantScript: uint8(b.script.index(s[1])),
+ haveScript: uint8(b.script.index(d[1])),
+ distance: uint8(distance),
+ })
+ }
+ case 1:
+ if desired == supported && desired == "*" {
+ continue
+ }
+ if distance == 1 {
+ // nb == no is already handled by macro mapping. Check there
+ // really is only this case.
+ if d[0] != "no" || s[0] != "nb" {
+ log.Fatalf("unhandled equivalence %s == %s", s[0], d[0])
+ }
+ continue
+ }
+ // TODO: consider dropping oneway field and just doubling the entry.
+ matchLang = append(matchLang, mutualIntelligibility{
+ want: uint16(b.langIndex(d[0])),
+ have: uint16(b.langIndex(s[0])),
+ distance: uint8(distance),
+ oneway: m.Oneway == "true",
+ })
+ case 3:
+ if desired == supported && desired == "*_*_*" {
+ continue
+ }
+ if desired != supported {
+ // This is now supported by CLDR, but only one case, which
+ // should already be covered by paradigm locales. For instance,
+ // test case "und, en, en-GU, en-IN, en-GB ; en-ZA ; en-GB" in
+ // testdata/CLDRLocaleMatcherTest.txt tests this.
+ if supported != "en_*_GB" {
+ log.Fatalf("not supported: desired=%q; supported=%q", desired, supported)
+ }
+ continue
+ }
+ ri := regionIntelligibility{
+ lang: b.langIndex(d[0]),
+ distance: uint8(distance),
+ }
+ if d[1] != "*" {
+ ri.script = uint8(b.script.index(d[1]))
+ }
+ switch {
+ case d[2] == "*":
+ ri.group = 0x80 // not contained in anything
+ case strings.HasPrefix(d[2], "$!"):
+ ri.group = 0x80
+ d[2] = "$" + d[2][len("$!"):]
+ fallthrough
+ case strings.HasPrefix(d[2], "$"):
+ ri.group |= idToIndex[d[2]]
+ }
+ matchRegion = append(matchRegion, ri)
+ default:
+ log.Fatalf("not supported: desired=%q; supported=%q", desired, supported)
+ }
+ }
+ sort.SliceStable(matchLang, func(i, j int) bool {
+ return matchLang[i].distance < matchLang[j].distance
+ })
+ b.writeSlice("matchLang", matchLang)
+
+ sort.SliceStable(matchScript, func(i, j int) bool {
+ return matchScript[i].distance < matchScript[j].distance
+ })
+ b.writeSlice("matchScript", matchScript)
+
+ sort.SliceStable(matchRegion, func(i, j int) bool {
+ return matchRegion[i].distance < matchRegion[j].distance
+ })
+ b.writeSlice("matchRegion", matchRegion)
+}
+
+func (b *builder) writeRegionInclusionData() {
+ var (
+ // mm holds for each group the set of groups with a distance of 1.
+ mm = make(map[int][]index)
+
+ // containment holds for each group the transitive closure of
+ // containment of other groups.
+ containment = make(map[index][]index)
+ )
+ for _, g := range b.supp.TerritoryContainment.Group {
+ // Skip UN and EURO zone as they are flattening the containment
+ // relationship.
+ if g.Type == "EZ" || g.Type == "UN" {
+ continue
+ }
+ group := b.region.index(g.Type)
+ groupIdx := b.groups[group]
+ for _, mem := range strings.Split(g.Contains, " ") {
+ r := b.region.index(mem)
+ mm[r] = append(mm[r], groupIdx)
+ if g, ok := b.groups[r]; ok {
+ mm[group] = append(mm[group], g)
+ containment[groupIdx] = append(containment[groupIdx], g)
+ }
+ }
+ }
+
+ regionContainment := make([]uint64, len(b.groups))
+ for _, g := range b.groups {
+ l := containment[g]
+
+ // Compute the transitive closure of containment.
+ for i := 0; i < len(l); i++ {
+ l = append(l, containment[l[i]]...)
+ }
+
+ // Compute the bitmask.
+ regionContainment[g] = 1 << g
+ for _, v := range l {
+ regionContainment[g] |= 1 << v
+ }
+ }
+ b.writeSlice("regionContainment", regionContainment)
+
+ regionInclusion := make([]uint8, len(b.region.s))
+ bvs := make(map[uint64]index)
+ // Make the first bitvector positions correspond with the groups.
+ for r, i := range b.groups {
+ bv := uint64(1 << i)
+ for _, g := range mm[r] {
+ bv |= 1 << g
+ }
+ bvs[bv] = i
+ regionInclusion[r] = uint8(bvs[bv])
+ }
+ for r := 1; r < len(b.region.s); r++ {
+ if _, ok := b.groups[r]; !ok {
+ bv := uint64(0)
+ for _, g := range mm[r] {
+ bv |= 1 << g
+ }
+ if bv == 0 {
+ // Pick the world for unspecified regions.
+ bv = 1 << b.groups[b.region.index("001")]
+ }
+ if _, ok := bvs[bv]; !ok {
+ bvs[bv] = index(len(bvs))
+ }
+ regionInclusion[r] = uint8(bvs[bv])
+ }
+ }
+ b.writeSlice("regionInclusion", regionInclusion)
+ regionInclusionBits := make([]uint64, len(bvs))
+ for k, v := range bvs {
+ regionInclusionBits[v] = uint64(k)
+ }
+ // Add bit vectors for increasingly large distances until a fixed point is reached.
+ regionInclusionNext := []uint8{}
+ for i := 0; i < len(regionInclusionBits); i++ {
+ bits := regionInclusionBits[i]
+ next := bits
+ for i := uint(0); i < uint(len(b.groups)); i++ {
+ if bits&(1<<i) != 0 {
+ next |= regionInclusionBits[i]
+ }
+ }
+ if _, ok := bvs[next]; !ok {
+ bvs[next] = index(len(bvs))
+ regionInclusionBits = append(regionInclusionBits, next)
+ }
+ regionInclusionNext = append(regionInclusionNext, uint8(bvs[next]))
+ }
+ b.writeSlice("regionInclusionBits", regionInclusionBits)
+ b.writeSlice("regionInclusionNext", regionInclusionNext)
+}
+
+type parentRel struct {
+ lang uint16
+ script uint8
+ maxScript uint8
+ toRegion uint16
+ fromRegion []uint16
+}
+
+func (b *builder) writeParents() {
+ b.writeType(parentRel{})
+
+ parents := []parentRel{}
+
+ // Construct parent overrides.
+ n := 0
+ for _, p := range b.data.Supplemental().ParentLocales.ParentLocale {
+ // Skipping non-standard scripts to root is implemented using addTags.
+ if p.Parent == "root" {
+ continue
+ }
+
+ sub := strings.Split(p.Parent, "_")
+ parent := parentRel{lang: b.langIndex(sub[0])}
+ if len(sub) == 2 {
+ // TODO: check that all undefined scripts are indeed Latn in these
+ // cases.
+ parent.maxScript = uint8(b.script.index("Latn"))
+ parent.toRegion = uint16(b.region.index(sub[1]))
+ } else {
+ parent.script = uint8(b.script.index(sub[1]))
+ parent.maxScript = parent.script
+ parent.toRegion = uint16(b.region.index(sub[2]))
+ }
+ for _, c := range strings.Split(p.Locales, " ") {
+ region := b.region.index(c[strings.LastIndex(c, "_")+1:])
+ parent.fromRegion = append(parent.fromRegion, uint16(region))
+ }
+ parents = append(parents, parent)
+ n += len(parent.fromRegion)
+ }
+ b.writeSliceAddSize("parents", n*2, parents)
+}
+
+func main() {
+ gen.Init()
+
+ gen.Repackage("gen_common.go", "common.go", "language")
+
+ w := gen.NewCodeWriter()
+ defer w.WriteGoFile("tables.go", "language")
+
+ fmt.Fprintln(w, `import "golang.org/x/text/internal/tag"`)
+
+ b := newBuilder(w)
+ gen.WriteCLDRVersion(w)
+
+ b.parseIndices()
+ b.writeType(fromTo{})
+ b.writeLanguage()
+ b.writeScript()
+ b.writeRegion()
+ b.writeVariant()
+ // TODO: b.writeLocale()
+ b.computeRegionGroups()
+ b.writeLikelyData()
+ b.writeMatchData()
+ b.writeRegionInclusionData()
+ b.writeParents()
+}
diff --git a/language/internal/gen_common.go b/language/internal/gen_common.go
new file mode 100644
index 0000000..83ce180
--- /dev/null
+++ b/language/internal/gen_common.go
@@ -0,0 +1,20 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+// This file contains code common to the maketables.go and the package code.
+
+// langAliasType is the type of an alias in langAliasMap.
+type langAliasType int8
+
+const (
+ langDeprecated langAliasType = iota
+ langMacro
+ langLegacy
+
+ langAliasTypeUnknown langAliasType = -1
+)
diff --git a/language/internal/gen_index.go b/language/internal/gen_index.go
new file mode 100644
index 0000000..5ca9bcc
--- /dev/null
+++ b/language/internal/gen_index.go
@@ -0,0 +1,162 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+// This file generates derivative tables based on the language package itself.
+
+import (
+ "bytes"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "log"
+ "reflect"
+ "sort"
+ "strings"
+
+ "golang.org/x/text/internal/gen"
+ "golang.org/x/text/language"
+ "golang.org/x/text/unicode/cldr"
+)
+
+var (
+ test = flag.Bool("test", false,
+ "test existing tables; can be used to compare web data with package data.")
+
+ draft = flag.String("draft",
+ "contributed",
+ `Minimal draft requirements (approved, contributed, provisional, unconfirmed).`)
+)
+
+func main() {
+ gen.Init()
+
+ // Read the CLDR zip file.
+ r := gen.OpenCLDRCoreZip()
+ defer r.Close()
+
+ d := &cldr.Decoder{}
+ data, err := d.DecodeZip(r)
+ if err != nil {
+ log.Fatalf("DecodeZip: %v", err)
+ }
+
+ w := gen.NewCodeWriter()
+ defer func() {
+ buf := &bytes.Buffer{}
+
+ if _, err = w.WriteGo(buf, "language", ""); err != nil {
+ log.Fatalf("Error formatting file index.go: %v", err)
+ }
+
+ // Since we're generating a table for our own package we need to rewrite
+ // doing the equivalent of go fmt -r 'language.b -> b'. Using
+ // bytes.Replace will do.
+ out := bytes.Replace(buf.Bytes(), []byte("language."), nil, -1)
+ if err := ioutil.WriteFile("index.go", out, 0600); err != nil {
+ log.Fatalf("Could not create file index.go: %v", err)
+ }
+ }()
+
+ m := map[language.Tag]bool{}
+ for _, lang := range data.Locales() {
+ // We include all locales unconditionally to be consistent with en_US.
+ // We want en_US, even though it has no data associated with it.
+
+ // TODO: put any of the languages for which no data exists at the end
+ // of the index. This allows all components based on ICU to use that
+ // as the cutoff point.
+ // if x := data.RawLDML(lang); false ||
+ // x.LocaleDisplayNames != nil ||
+ // x.Characters != nil ||
+ // x.Delimiters != nil ||
+ // x.Measurement != nil ||
+ // x.Dates != nil ||
+ // x.Numbers != nil ||
+ // x.Units != nil ||
+ // x.ListPatterns != nil ||
+ // x.Collations != nil ||
+ // x.Segmentations != nil ||
+ // x.Rbnf != nil ||
+ // x.Annotations != nil ||
+ // x.Metadata != nil {
+
+ // TODO: support POSIX natively, albeit non-standard.
+ tag := language.Make(strings.Replace(lang, "_POSIX", "-u-va-posix", 1))
+ m[tag] = true
+ // }
+ }
+ // Include locales for plural rules, which uses a different structure.
+ for _, plurals := range data.Supplemental().Plurals {
+ for _, rules := range plurals.PluralRules {
+ for _, lang := range strings.Split(rules.Locales, " ") {
+ m[language.Make(lang)] = true
+ }
+ }
+ }
+
+ var core, special []language.Tag
+
+ for t := range m {
+ if x := t.Extensions(); len(x) != 0 && fmt.Sprint(x) != "[u-va-posix]" {
+ log.Fatalf("Unexpected extension %v in %v", x, t)
+ }
+ if len(t.Variants()) == 0 && len(t.Extensions()) == 0 {
+ core = append(core, t)
+ } else {
+ special = append(special, t)
+ }
+ }
+
+ w.WriteComment(`
+ NumCompactTags is the number of common tags. The maximum tag is
+ NumCompactTags-1.`)
+ w.WriteConst("NumCompactTags", len(core)+len(special))
+
+ sort.Sort(byAlpha(special))
+ w.WriteVar("specialTags", special)
+
+ // TODO: order by frequency?
+ sort.Sort(byAlpha(core))
+
+ // Size computations are just an estimate.
+ w.Size += int(reflect.TypeOf(map[uint32]uint16{}).Size())
+ w.Size += len(core) * 6 // size of uint32 and uint16
+
+ fmt.Fprintln(w)
+ fmt.Fprintln(w, "var coreTags = map[uint32]uint16{")
+ fmt.Fprintln(w, "0x0: 0, // und")
+ i := len(special) + 1 // Und and special tags already written.
+ for _, t := range core {
+ if t == language.Und {
+ continue
+ }
+ fmt.Fprint(w.Hash, t, i)
+ b, s, r := t.Raw()
+ fmt.Fprintf(w, "0x%s%s%s: %d, // %s\n",
+ getIndex(b, 3), // 3 is enough as it is guaranteed to be a compact number
+ getIndex(s, 2),
+ getIndex(r, 3),
+ i, t)
+ i++
+ }
+ fmt.Fprintln(w, "}")
+}
+
+// getIndex prints the subtag type and extracts its index of size nibble.
+// If the index is less than n nibbles, the result is prefixed with 0s.
+func getIndex(x interface{}, n int) string {
+ s := fmt.Sprintf("%#v", x) // s is of form Type{typeID: 0x00}
+ s = s[strings.Index(s, "0x")+2 : len(s)-1]
+ return strings.Repeat("0", n-len(s)) + s
+}
+
+type byAlpha []language.Tag
+
+func (a byAlpha) Len() int { return len(a) }
+func (a byAlpha) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+func (a byAlpha) Less(i, j int) bool { return a[i].String() < a[j].String() }
diff --git a/language/internal/go1_1.go b/language/internal/go1_1.go
new file mode 100644
index 0000000..380f4c0
--- /dev/null
+++ b/language/internal/go1_1.go
@@ -0,0 +1,38 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !go1.2
+
+package language
+
+import "sort"
+
+func sortStable(s sort.Interface) {
+ ss := stableSort{
+ s: s,
+ pos: make([]int, s.Len()),
+ }
+ for i := range ss.pos {
+ ss.pos[i] = i
+ }
+ sort.Sort(&ss)
+}
+
+type stableSort struct {
+ s sort.Interface
+ pos []int
+}
+
+func (s *stableSort) Len() int {
+ return len(s.pos)
+}
+
+func (s *stableSort) Less(i, j int) bool {
+ return s.s.Less(i, j) || !s.s.Less(j, i) && s.pos[i] < s.pos[j]
+}
+
+func (s *stableSort) Swap(i, j int) {
+ s.s.Swap(i, j)
+ s.pos[i], s.pos[j] = s.pos[j], s.pos[i]
+}
diff --git a/language/internal/go1_2.go b/language/internal/go1_2.go
new file mode 100644
index 0000000..38268c5
--- /dev/null
+++ b/language/internal/go1_2.go
@@ -0,0 +1,11 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.2
+
+package language
+
+import "sort"
+
+var sortStable = sort.Stable
diff --git a/language/internal/httpexample_test.go b/language/internal/httpexample_test.go
new file mode 100644
index 0000000..40d0663
--- /dev/null
+++ b/language/internal/httpexample_test.go
@@ -0,0 +1,48 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language_test
+
+import (
+ "fmt"
+ "net/http"
+ "strings"
+
+ "golang.org/x/text/language"
+)
+
+// matcher is a language.Matcher configured for all supported languages.
+var matcher = language.NewMatcher([]language.Tag{
+ language.BritishEnglish,
+ language.Norwegian,
+ language.German,
+})
+
+// handler is a http.HandlerFunc.
+func handler(w http.ResponseWriter, r *http.Request) {
+ t, q, err := language.ParseAcceptLanguage(r.Header.Get("Accept-Language"))
+ // We ignore the error: the default language will be selected for t == nil.
+ tag, _, _ := matcher.Match(t...)
+ fmt.Printf("%5v (t: %6v; q: %3v; err: %v)\n", tag, t, q, err)
+}
+
+func ExampleParseAcceptLanguage() {
+ for _, al := range []string{
+ "nn;q=0.3, en-us;q=0.8, en,",
+ "gsw, en;q=0.7, en-US;q=0.8",
+ "gsw, nl, da",
+ "invalid",
+ } {
+ // Create dummy request with Accept-Language set and pass it to handler.
+ r, _ := http.NewRequest("GET", "example.com", strings.NewReader("Hello"))
+ r.Header.Set("Accept-Language", al)
+ handler(nil, r)
+ }
+
+ // Output:
+ // en-GB (t: [ en en-US nn]; q: [ 1 0.8 0.3]; err: <nil>)
+ // en-GB (t: [ gsw en-US en]; q: [ 1 0.8 0.7]; err: <nil>)
+ // de (t: [ gsw nl da]; q: [ 1 1 1]; err: <nil>)
+ // en-GB (t: []; q: []; err: language: tag is not well-formed)
+}
diff --git a/language/internal/index.go b/language/internal/index.go
new file mode 100644
index 0000000..5311e5c
--- /dev/null
+++ b/language/internal/index.go
@@ -0,0 +1,783 @@
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+package language
+
+// NumCompactTags is the number of common tags. The maximum tag is
+// NumCompactTags-1.
+const NumCompactTags = 768
+
+var specialTags = []Tag{ // 2 elements
+ 0: {lang: 0xd7, region: 0x6e, script: 0x0, pVariant: 0x5, pExt: 0xe, str: "ca-ES-valencia"},
+ 1: {lang: 0x139, region: 0x135, script: 0x0, pVariant: 0x5, pExt: 0x5, str: "en-US-u-va-posix"},
+} // Size: 72 bytes
+
+var coreTags = map[uint32]uint16{
+ 0x0: 0, // und
+ 0x01600000: 3, // af
+ 0x016000d2: 4, // af-NA
+ 0x01600161: 5, // af-ZA
+ 0x01c00000: 6, // agq
+ 0x01c00052: 7, // agq-CM
+ 0x02100000: 8, // ak
+ 0x02100080: 9, // ak-GH
+ 0x02700000: 10, // am
+ 0x0270006f: 11, // am-ET
+ 0x03a00000: 12, // ar
+ 0x03a00001: 13, // ar-001
+ 0x03a00023: 14, // ar-AE
+ 0x03a00039: 15, // ar-BH
+ 0x03a00062: 16, // ar-DJ
+ 0x03a00067: 17, // ar-DZ
+ 0x03a0006b: 18, // ar-EG
+ 0x03a0006c: 19, // ar-EH
+ 0x03a0006d: 20, // ar-ER
+ 0x03a00097: 21, // ar-IL
+ 0x03a0009b: 22, // ar-IQ
+ 0x03a000a1: 23, // ar-JO
+ 0x03a000a8: 24, // ar-KM
+ 0x03a000ac: 25, // ar-KW
+ 0x03a000b0: 26, // ar-LB
+ 0x03a000b9: 27, // ar-LY
+ 0x03a000ba: 28, // ar-MA
+ 0x03a000c9: 29, // ar-MR
+ 0x03a000e1: 30, // ar-OM
+ 0x03a000ed: 31, // ar-PS
+ 0x03a000f3: 32, // ar-QA
+ 0x03a00108: 33, // ar-SA
+ 0x03a0010b: 34, // ar-SD
+ 0x03a00115: 35, // ar-SO
+ 0x03a00117: 36, // ar-SS
+ 0x03a0011c: 37, // ar-SY
+ 0x03a00120: 38, // ar-TD
+ 0x03a00128: 39, // ar-TN
+ 0x03a0015e: 40, // ar-YE
+ 0x04000000: 41, // ars
+ 0x04300000: 42, // as
+ 0x04300099: 43, // as-IN
+ 0x04400000: 44, // asa
+ 0x0440012f: 45, // asa-TZ
+ 0x04800000: 46, // ast
+ 0x0480006e: 47, // ast-ES
+ 0x05800000: 48, // az
+ 0x0581f000: 49, // az-Cyrl
+ 0x0581f032: 50, // az-Cyrl-AZ
+ 0x05857000: 51, // az-Latn
+ 0x05857032: 52, // az-Latn-AZ
+ 0x05e00000: 53, // bas
+ 0x05e00052: 54, // bas-CM
+ 0x07100000: 55, // be
+ 0x07100047: 56, // be-BY
+ 0x07500000: 57, // bem
+ 0x07500162: 58, // bem-ZM
+ 0x07900000: 59, // bez
+ 0x0790012f: 60, // bez-TZ
+ 0x07e00000: 61, // bg
+ 0x07e00038: 62, // bg-BG
+ 0x08200000: 63, // bh
+ 0x0a000000: 64, // bm
+ 0x0a0000c3: 65, // bm-ML
+ 0x0a500000: 66, // bn
+ 0x0a500035: 67, // bn-BD
+ 0x0a500099: 68, // bn-IN
+ 0x0a900000: 69, // bo
+ 0x0a900053: 70, // bo-CN
+ 0x0a900099: 71, // bo-IN
+ 0x0b200000: 72, // br
+ 0x0b200078: 73, // br-FR
+ 0x0b500000: 74, // brx
+ 0x0b500099: 75, // brx-IN
+ 0x0b700000: 76, // bs
+ 0x0b71f000: 77, // bs-Cyrl
+ 0x0b71f033: 78, // bs-Cyrl-BA
+ 0x0b757000: 79, // bs-Latn
+ 0x0b757033: 80, // bs-Latn-BA
+ 0x0d700000: 81, // ca
+ 0x0d700022: 82, // ca-AD
+ 0x0d70006e: 83, // ca-ES
+ 0x0d700078: 84, // ca-FR
+ 0x0d70009e: 85, // ca-IT
+ 0x0db00000: 86, // ccp
+ 0x0db00035: 87, // ccp-BD
+ 0x0db00099: 88, // ccp-IN
+ 0x0dc00000: 89, // ce
+ 0x0dc00106: 90, // ce-RU
+ 0x0df00000: 91, // cgg
+ 0x0df00131: 92, // cgg-UG
+ 0x0e500000: 93, // chr
+ 0x0e500135: 94, // chr-US
+ 0x0e900000: 95, // ckb
+ 0x0e90009b: 96, // ckb-IQ
+ 0x0e90009c: 97, // ckb-IR
+ 0x0fa00000: 98, // cs
+ 0x0fa0005e: 99, // cs-CZ
+ 0x0fe00000: 100, // cu
+ 0x0fe00106: 101, // cu-RU
+ 0x10000000: 102, // cy
+ 0x1000007b: 103, // cy-GB
+ 0x10100000: 104, // da
+ 0x10100063: 105, // da-DK
+ 0x10100082: 106, // da-GL
+ 0x10800000: 107, // dav
+ 0x108000a4: 108, // dav-KE
+ 0x10d00000: 109, // de
+ 0x10d0002e: 110, // de-AT
+ 0x10d00036: 111, // de-BE
+ 0x10d0004e: 112, // de-CH
+ 0x10d00060: 113, // de-DE
+ 0x10d0009e: 114, // de-IT
+ 0x10d000b2: 115, // de-LI
+ 0x10d000b7: 116, // de-LU
+ 0x11700000: 117, // dje
+ 0x117000d4: 118, // dje-NE
+ 0x11f00000: 119, // dsb
+ 0x11f00060: 120, // dsb-DE
+ 0x12400000: 121, // dua
+ 0x12400052: 122, // dua-CM
+ 0x12800000: 123, // dv
+ 0x12b00000: 124, // dyo
+ 0x12b00114: 125, // dyo-SN
+ 0x12d00000: 126, // dz
+ 0x12d00043: 127, // dz-BT
+ 0x12f00000: 128, // ebu
+ 0x12f000a4: 129, // ebu-KE
+ 0x13000000: 130, // ee
+ 0x13000080: 131, // ee-GH
+ 0x13000122: 132, // ee-TG
+ 0x13600000: 133, // el
+ 0x1360005d: 134, // el-CY
+ 0x13600087: 135, // el-GR
+ 0x13900000: 136, // en
+ 0x13900001: 137, // en-001
+ 0x1390001a: 138, // en-150
+ 0x13900025: 139, // en-AG
+ 0x13900026: 140, // en-AI
+ 0x1390002d: 141, // en-AS
+ 0x1390002e: 142, // en-AT
+ 0x1390002f: 143, // en-AU
+ 0x13900034: 144, // en-BB
+ 0x13900036: 145, // en-BE
+ 0x1390003a: 146, // en-BI
+ 0x1390003d: 147, // en-BM
+ 0x13900042: 148, // en-BS
+ 0x13900046: 149, // en-BW
+ 0x13900048: 150, // en-BZ
+ 0x13900049: 151, // en-CA
+ 0x1390004a: 152, // en-CC
+ 0x1390004e: 153, // en-CH
+ 0x13900050: 154, // en-CK
+ 0x13900052: 155, // en-CM
+ 0x1390005c: 156, // en-CX
+ 0x1390005d: 157, // en-CY
+ 0x13900060: 158, // en-DE
+ 0x13900061: 159, // en-DG
+ 0x13900063: 160, // en-DK
+ 0x13900064: 161, // en-DM
+ 0x1390006d: 162, // en-ER
+ 0x13900072: 163, // en-FI
+ 0x13900073: 164, // en-FJ
+ 0x13900074: 165, // en-FK
+ 0x13900075: 166, // en-FM
+ 0x1390007b: 167, // en-GB
+ 0x1390007c: 168, // en-GD
+ 0x1390007f: 169, // en-GG
+ 0x13900080: 170, // en-GH
+ 0x13900081: 171, // en-GI
+ 0x13900083: 172, // en-GM
+ 0x1390008a: 173, // en-GU
+ 0x1390008c: 174, // en-GY
+ 0x1390008d: 175, // en-HK
+ 0x13900096: 176, // en-IE
+ 0x13900097: 177, // en-IL
+ 0x13900098: 178, // en-IM
+ 0x13900099: 179, // en-IN
+ 0x1390009a: 180, // en-IO
+ 0x1390009f: 181, // en-JE
+ 0x139000a0: 182, // en-JM
+ 0x139000a4: 183, // en-KE
+ 0x139000a7: 184, // en-KI
+ 0x139000a9: 185, // en-KN
+ 0x139000ad: 186, // en-KY
+ 0x139000b1: 187, // en-LC
+ 0x139000b4: 188, // en-LR
+ 0x139000b5: 189, // en-LS
+ 0x139000bf: 190, // en-MG
+ 0x139000c0: 191, // en-MH
+ 0x139000c6: 192, // en-MO
+ 0x139000c7: 193, // en-MP
+ 0x139000ca: 194, // en-MS
+ 0x139000cb: 195, // en-MT
+ 0x139000cc: 196, // en-MU
+ 0x139000ce: 197, // en-MW
+ 0x139000d0: 198, // en-MY
+ 0x139000d2: 199, // en-NA
+ 0x139000d5: 200, // en-NF
+ 0x139000d6: 201, // en-NG
+ 0x139000d9: 202, // en-NL
+ 0x139000dd: 203, // en-NR
+ 0x139000df: 204, // en-NU
+ 0x139000e0: 205, // en-NZ
+ 0x139000e6: 206, // en-PG
+ 0x139000e7: 207, // en-PH
+ 0x139000e8: 208, // en-PK
+ 0x139000eb: 209, // en-PN
+ 0x139000ec: 210, // en-PR
+ 0x139000f0: 211, // en-PW
+ 0x13900107: 212, // en-RW
+ 0x13900109: 213, // en-SB
+ 0x1390010a: 214, // en-SC
+ 0x1390010b: 215, // en-SD
+ 0x1390010c: 216, // en-SE
+ 0x1390010d: 217, // en-SG
+ 0x1390010e: 218, // en-SH
+ 0x1390010f: 219, // en-SI
+ 0x13900112: 220, // en-SL
+ 0x13900117: 221, // en-SS
+ 0x1390011b: 222, // en-SX
+ 0x1390011d: 223, // en-SZ
+ 0x1390011f: 224, // en-TC
+ 0x13900125: 225, // en-TK
+ 0x13900129: 226, // en-TO
+ 0x1390012c: 227, // en-TT
+ 0x1390012d: 228, // en-TV
+ 0x1390012f: 229, // en-TZ
+ 0x13900131: 230, // en-UG
+ 0x13900133: 231, // en-UM
+ 0x13900135: 232, // en-US
+ 0x13900139: 233, // en-VC
+ 0x1390013c: 234, // en-VG
+ 0x1390013d: 235, // en-VI
+ 0x1390013f: 236, // en-VU
+ 0x13900142: 237, // en-WS
+ 0x13900161: 238, // en-ZA
+ 0x13900162: 239, // en-ZM
+ 0x13900164: 240, // en-ZW
+ 0x13c00000: 241, // eo
+ 0x13c00001: 242, // eo-001
+ 0x13e00000: 243, // es
+ 0x13e0001f: 244, // es-419
+ 0x13e0002c: 245, // es-AR
+ 0x13e0003f: 246, // es-BO
+ 0x13e00041: 247, // es-BR
+ 0x13e00048: 248, // es-BZ
+ 0x13e00051: 249, // es-CL
+ 0x13e00054: 250, // es-CO
+ 0x13e00056: 251, // es-CR
+ 0x13e00059: 252, // es-CU
+ 0x13e00065: 253, // es-DO
+ 0x13e00068: 254, // es-EA
+ 0x13e00069: 255, // es-EC
+ 0x13e0006e: 256, // es-ES
+ 0x13e00086: 257, // es-GQ
+ 0x13e00089: 258, // es-GT
+ 0x13e0008f: 259, // es-HN
+ 0x13e00094: 260, // es-IC
+ 0x13e000cf: 261, // es-MX
+ 0x13e000d8: 262, // es-NI
+ 0x13e000e2: 263, // es-PA
+ 0x13e000e4: 264, // es-PE
+ 0x13e000e7: 265, // es-PH
+ 0x13e000ec: 266, // es-PR
+ 0x13e000f1: 267, // es-PY
+ 0x13e0011a: 268, // es-SV
+ 0x13e00135: 269, // es-US
+ 0x13e00136: 270, // es-UY
+ 0x13e0013b: 271, // es-VE
+ 0x14000000: 272, // et
+ 0x1400006a: 273, // et-EE
+ 0x14500000: 274, // eu
+ 0x1450006e: 275, // eu-ES
+ 0x14600000: 276, // ewo
+ 0x14600052: 277, // ewo-CM
+ 0x14800000: 278, // fa
+ 0x14800024: 279, // fa-AF
+ 0x1480009c: 280, // fa-IR
+ 0x14e00000: 281, // ff
+ 0x14e00052: 282, // ff-CM
+ 0x14e00084: 283, // ff-GN
+ 0x14e000c9: 284, // ff-MR
+ 0x14e00114: 285, // ff-SN
+ 0x15100000: 286, // fi
+ 0x15100072: 287, // fi-FI
+ 0x15300000: 288, // fil
+ 0x153000e7: 289, // fil-PH
+ 0x15800000: 290, // fo
+ 0x15800063: 291, // fo-DK
+ 0x15800076: 292, // fo-FO
+ 0x15e00000: 293, // fr
+ 0x15e00036: 294, // fr-BE
+ 0x15e00037: 295, // fr-BF
+ 0x15e0003a: 296, // fr-BI
+ 0x15e0003b: 297, // fr-BJ
+ 0x15e0003c: 298, // fr-BL
+ 0x15e00049: 299, // fr-CA
+ 0x15e0004b: 300, // fr-CD
+ 0x15e0004c: 301, // fr-CF
+ 0x15e0004d: 302, // fr-CG
+ 0x15e0004e: 303, // fr-CH
+ 0x15e0004f: 304, // fr-CI
+ 0x15e00052: 305, // fr-CM
+ 0x15e00062: 306, // fr-DJ
+ 0x15e00067: 307, // fr-DZ
+ 0x15e00078: 308, // fr-FR
+ 0x15e0007a: 309, // fr-GA
+ 0x15e0007e: 310, // fr-GF
+ 0x15e00084: 311, // fr-GN
+ 0x15e00085: 312, // fr-GP
+ 0x15e00086: 313, // fr-GQ
+ 0x15e00091: 314, // fr-HT
+ 0x15e000a8: 315, // fr-KM
+ 0x15e000b7: 316, // fr-LU
+ 0x15e000ba: 317, // fr-MA
+ 0x15e000bb: 318, // fr-MC
+ 0x15e000be: 319, // fr-MF
+ 0x15e000bf: 320, // fr-MG
+ 0x15e000c3: 321, // fr-ML
+ 0x15e000c8: 322, // fr-MQ
+ 0x15e000c9: 323, // fr-MR
+ 0x15e000cc: 324, // fr-MU
+ 0x15e000d3: 325, // fr-NC
+ 0x15e000d4: 326, // fr-NE
+ 0x15e000e5: 327, // fr-PF
+ 0x15e000ea: 328, // fr-PM
+ 0x15e00102: 329, // fr-RE
+ 0x15e00107: 330, // fr-RW
+ 0x15e0010a: 331, // fr-SC
+ 0x15e00114: 332, // fr-SN
+ 0x15e0011c: 333, // fr-SY
+ 0x15e00120: 334, // fr-TD
+ 0x15e00122: 335, // fr-TG
+ 0x15e00128: 336, // fr-TN
+ 0x15e0013f: 337, // fr-VU
+ 0x15e00140: 338, // fr-WF
+ 0x15e0015f: 339, // fr-YT
+ 0x16900000: 340, // fur
+ 0x1690009e: 341, // fur-IT
+ 0x16d00000: 342, // fy
+ 0x16d000d9: 343, // fy-NL
+ 0x16e00000: 344, // ga
+ 0x16e00096: 345, // ga-IE
+ 0x17e00000: 346, // gd
+ 0x17e0007b: 347, // gd-GB
+ 0x19000000: 348, // gl
+ 0x1900006e: 349, // gl-ES
+ 0x1a300000: 350, // gsw
+ 0x1a30004e: 351, // gsw-CH
+ 0x1a300078: 352, // gsw-FR
+ 0x1a3000b2: 353, // gsw-LI
+ 0x1a400000: 354, // gu
+ 0x1a400099: 355, // gu-IN
+ 0x1a900000: 356, // guw
+ 0x1ab00000: 357, // guz
+ 0x1ab000a4: 358, // guz-KE
+ 0x1ac00000: 359, // gv
+ 0x1ac00098: 360, // gv-IM
+ 0x1b400000: 361, // ha
+ 0x1b400080: 362, // ha-GH
+ 0x1b4000d4: 363, // ha-NE
+ 0x1b4000d6: 364, // ha-NG
+ 0x1b800000: 365, // haw
+ 0x1b800135: 366, // haw-US
+ 0x1bc00000: 367, // he
+ 0x1bc00097: 368, // he-IL
+ 0x1be00000: 369, // hi
+ 0x1be00099: 370, // hi-IN
+ 0x1d100000: 371, // hr
+ 0x1d100033: 372, // hr-BA
+ 0x1d100090: 373, // hr-HR
+ 0x1d200000: 374, // hsb
+ 0x1d200060: 375, // hsb-DE
+ 0x1d500000: 376, // hu
+ 0x1d500092: 377, // hu-HU
+ 0x1d700000: 378, // hy
+ 0x1d700028: 379, // hy-AM
+ 0x1e100000: 380, // id
+ 0x1e100095: 381, // id-ID
+ 0x1e700000: 382, // ig
+ 0x1e7000d6: 383, // ig-NG
+ 0x1ea00000: 384, // ii
+ 0x1ea00053: 385, // ii-CN
+ 0x1f500000: 386, // io
+ 0x1f800000: 387, // is
+ 0x1f80009d: 388, // is-IS
+ 0x1f900000: 389, // it
+ 0x1f90004e: 390, // it-CH
+ 0x1f90009e: 391, // it-IT
+ 0x1f900113: 392, // it-SM
+ 0x1f900138: 393, // it-VA
+ 0x1fa00000: 394, // iu
+ 0x20000000: 395, // ja
+ 0x200000a2: 396, // ja-JP
+ 0x20300000: 397, // jbo
+ 0x20700000: 398, // jgo
+ 0x20700052: 399, // jgo-CM
+ 0x20a00000: 400, // jmc
+ 0x20a0012f: 401, // jmc-TZ
+ 0x20e00000: 402, // jv
+ 0x21000000: 403, // ka
+ 0x2100007d: 404, // ka-GE
+ 0x21200000: 405, // kab
+ 0x21200067: 406, // kab-DZ
+ 0x21600000: 407, // kaj
+ 0x21700000: 408, // kam
+ 0x217000a4: 409, // kam-KE
+ 0x21f00000: 410, // kcg
+ 0x22300000: 411, // kde
+ 0x2230012f: 412, // kde-TZ
+ 0x22700000: 413, // kea
+ 0x2270005a: 414, // kea-CV
+ 0x23400000: 415, // khq
+ 0x234000c3: 416, // khq-ML
+ 0x23900000: 417, // ki
+ 0x239000a4: 418, // ki-KE
+ 0x24200000: 419, // kk
+ 0x242000ae: 420, // kk-KZ
+ 0x24400000: 421, // kkj
+ 0x24400052: 422, // kkj-CM
+ 0x24500000: 423, // kl
+ 0x24500082: 424, // kl-GL
+ 0x24600000: 425, // kln
+ 0x246000a4: 426, // kln-KE
+ 0x24a00000: 427, // km
+ 0x24a000a6: 428, // km-KH
+ 0x25100000: 429, // kn
+ 0x25100099: 430, // kn-IN
+ 0x25400000: 431, // ko
+ 0x254000aa: 432, // ko-KP
+ 0x254000ab: 433, // ko-KR
+ 0x25600000: 434, // kok
+ 0x25600099: 435, // kok-IN
+ 0x26a00000: 436, // ks
+ 0x26a00099: 437, // ks-IN
+ 0x26b00000: 438, // ksb
+ 0x26b0012f: 439, // ksb-TZ
+ 0x26d00000: 440, // ksf
+ 0x26d00052: 441, // ksf-CM
+ 0x26e00000: 442, // ksh
+ 0x26e00060: 443, // ksh-DE
+ 0x27400000: 444, // ku
+ 0x28100000: 445, // kw
+ 0x2810007b: 446, // kw-GB
+ 0x28a00000: 447, // ky
+ 0x28a000a5: 448, // ky-KG
+ 0x29100000: 449, // lag
+ 0x2910012f: 450, // lag-TZ
+ 0x29500000: 451, // lb
+ 0x295000b7: 452, // lb-LU
+ 0x2a300000: 453, // lg
+ 0x2a300131: 454, // lg-UG
+ 0x2af00000: 455, // lkt
+ 0x2af00135: 456, // lkt-US
+ 0x2b500000: 457, // ln
+ 0x2b50002a: 458, // ln-AO
+ 0x2b50004b: 459, // ln-CD
+ 0x2b50004c: 460, // ln-CF
+ 0x2b50004d: 461, // ln-CG
+ 0x2b800000: 462, // lo
+ 0x2b8000af: 463, // lo-LA
+ 0x2bf00000: 464, // lrc
+ 0x2bf0009b: 465, // lrc-IQ
+ 0x2bf0009c: 466, // lrc-IR
+ 0x2c000000: 467, // lt
+ 0x2c0000b6: 468, // lt-LT
+ 0x2c200000: 469, // lu
+ 0x2c20004b: 470, // lu-CD
+ 0x2c400000: 471, // luo
+ 0x2c4000a4: 472, // luo-KE
+ 0x2c500000: 473, // luy
+ 0x2c5000a4: 474, // luy-KE
+ 0x2c700000: 475, // lv
+ 0x2c7000b8: 476, // lv-LV
+ 0x2d100000: 477, // mas
+ 0x2d1000a4: 478, // mas-KE
+ 0x2d10012f: 479, // mas-TZ
+ 0x2e900000: 480, // mer
+ 0x2e9000a4: 481, // mer-KE
+ 0x2ed00000: 482, // mfe
+ 0x2ed000cc: 483, // mfe-MU
+ 0x2f100000: 484, // mg
+ 0x2f1000bf: 485, // mg-MG
+ 0x2f200000: 486, // mgh
+ 0x2f2000d1: 487, // mgh-MZ
+ 0x2f400000: 488, // mgo
+ 0x2f400052: 489, // mgo-CM
+ 0x2ff00000: 490, // mk
+ 0x2ff000c2: 491, // mk-MK
+ 0x30400000: 492, // ml
+ 0x30400099: 493, // ml-IN
+ 0x30b00000: 494, // mn
+ 0x30b000c5: 495, // mn-MN
+ 0x31b00000: 496, // mr
+ 0x31b00099: 497, // mr-IN
+ 0x31f00000: 498, // ms
+ 0x31f0003e: 499, // ms-BN
+ 0x31f000d0: 500, // ms-MY
+ 0x31f0010d: 501, // ms-SG
+ 0x32000000: 502, // mt
+ 0x320000cb: 503, // mt-MT
+ 0x32500000: 504, // mua
+ 0x32500052: 505, // mua-CM
+ 0x33100000: 506, // my
+ 0x331000c4: 507, // my-MM
+ 0x33a00000: 508, // mzn
+ 0x33a0009c: 509, // mzn-IR
+ 0x34100000: 510, // nah
+ 0x34500000: 511, // naq
+ 0x345000d2: 512, // naq-NA
+ 0x34700000: 513, // nb
+ 0x347000da: 514, // nb-NO
+ 0x34700110: 515, // nb-SJ
+ 0x34e00000: 516, // nd
+ 0x34e00164: 517, // nd-ZW
+ 0x35000000: 518, // nds
+ 0x35000060: 519, // nds-DE
+ 0x350000d9: 520, // nds-NL
+ 0x35100000: 521, // ne
+ 0x35100099: 522, // ne-IN
+ 0x351000db: 523, // ne-NP
+ 0x36700000: 524, // nl
+ 0x36700030: 525, // nl-AW
+ 0x36700036: 526, // nl-BE
+ 0x36700040: 527, // nl-BQ
+ 0x3670005b: 528, // nl-CW
+ 0x367000d9: 529, // nl-NL
+ 0x36700116: 530, // nl-SR
+ 0x3670011b: 531, // nl-SX
+ 0x36800000: 532, // nmg
+ 0x36800052: 533, // nmg-CM
+ 0x36a00000: 534, // nn
+ 0x36a000da: 535, // nn-NO
+ 0x36c00000: 536, // nnh
+ 0x36c00052: 537, // nnh-CM
+ 0x36f00000: 538, // no
+ 0x37500000: 539, // nqo
+ 0x37600000: 540, // nr
+ 0x37a00000: 541, // nso
+ 0x38000000: 542, // nus
+ 0x38000117: 543, // nus-SS
+ 0x38700000: 544, // ny
+ 0x38900000: 545, // nyn
+ 0x38900131: 546, // nyn-UG
+ 0x39000000: 547, // om
+ 0x3900006f: 548, // om-ET
+ 0x390000a4: 549, // om-KE
+ 0x39500000: 550, // or
+ 0x39500099: 551, // or-IN
+ 0x39800000: 552, // os
+ 0x3980007d: 553, // os-GE
+ 0x39800106: 554, // os-RU
+ 0x39d00000: 555, // pa
+ 0x39d05000: 556, // pa-Arab
+ 0x39d050e8: 557, // pa-Arab-PK
+ 0x39d33000: 558, // pa-Guru
+ 0x39d33099: 559, // pa-Guru-IN
+ 0x3a100000: 560, // pap
+ 0x3b300000: 561, // pl
+ 0x3b3000e9: 562, // pl-PL
+ 0x3bd00000: 563, // prg
+ 0x3bd00001: 564, // prg-001
+ 0x3be00000: 565, // ps
+ 0x3be00024: 566, // ps-AF
+ 0x3c000000: 567, // pt
+ 0x3c00002a: 568, // pt-AO
+ 0x3c000041: 569, // pt-BR
+ 0x3c00004e: 570, // pt-CH
+ 0x3c00005a: 571, // pt-CV
+ 0x3c000086: 572, // pt-GQ
+ 0x3c00008b: 573, // pt-GW
+ 0x3c0000b7: 574, // pt-LU
+ 0x3c0000c6: 575, // pt-MO
+ 0x3c0000d1: 576, // pt-MZ
+ 0x3c0000ee: 577, // pt-PT
+ 0x3c000118: 578, // pt-ST
+ 0x3c000126: 579, // pt-TL
+ 0x3c400000: 580, // qu
+ 0x3c40003f: 581, // qu-BO
+ 0x3c400069: 582, // qu-EC
+ 0x3c4000e4: 583, // qu-PE
+ 0x3d400000: 584, // rm
+ 0x3d40004e: 585, // rm-CH
+ 0x3d900000: 586, // rn
+ 0x3d90003a: 587, // rn-BI
+ 0x3dc00000: 588, // ro
+ 0x3dc000bc: 589, // ro-MD
+ 0x3dc00104: 590, // ro-RO
+ 0x3de00000: 591, // rof
+ 0x3de0012f: 592, // rof-TZ
+ 0x3e200000: 593, // ru
+ 0x3e200047: 594, // ru-BY
+ 0x3e2000a5: 595, // ru-KG
+ 0x3e2000ae: 596, // ru-KZ
+ 0x3e2000bc: 597, // ru-MD
+ 0x3e200106: 598, // ru-RU
+ 0x3e200130: 599, // ru-UA
+ 0x3e500000: 600, // rw
+ 0x3e500107: 601, // rw-RW
+ 0x3e600000: 602, // rwk
+ 0x3e60012f: 603, // rwk-TZ
+ 0x3eb00000: 604, // sah
+ 0x3eb00106: 605, // sah-RU
+ 0x3ec00000: 606, // saq
+ 0x3ec000a4: 607, // saq-KE
+ 0x3f300000: 608, // sbp
+ 0x3f30012f: 609, // sbp-TZ
+ 0x3fa00000: 610, // sd
+ 0x3fa000e8: 611, // sd-PK
+ 0x3fc00000: 612, // sdh
+ 0x3fd00000: 613, // se
+ 0x3fd00072: 614, // se-FI
+ 0x3fd000da: 615, // se-NO
+ 0x3fd0010c: 616, // se-SE
+ 0x3ff00000: 617, // seh
+ 0x3ff000d1: 618, // seh-MZ
+ 0x40100000: 619, // ses
+ 0x401000c3: 620, // ses-ML
+ 0x40200000: 621, // sg
+ 0x4020004c: 622, // sg-CF
+ 0x40800000: 623, // shi
+ 0x40857000: 624, // shi-Latn
+ 0x408570ba: 625, // shi-Latn-MA
+ 0x408dc000: 626, // shi-Tfng
+ 0x408dc0ba: 627, // shi-Tfng-MA
+ 0x40c00000: 628, // si
+ 0x40c000b3: 629, // si-LK
+ 0x41200000: 630, // sk
+ 0x41200111: 631, // sk-SK
+ 0x41600000: 632, // sl
+ 0x4160010f: 633, // sl-SI
+ 0x41c00000: 634, // sma
+ 0x41d00000: 635, // smi
+ 0x41e00000: 636, // smj
+ 0x41f00000: 637, // smn
+ 0x41f00072: 638, // smn-FI
+ 0x42200000: 639, // sms
+ 0x42300000: 640, // sn
+ 0x42300164: 641, // sn-ZW
+ 0x42900000: 642, // so
+ 0x42900062: 643, // so-DJ
+ 0x4290006f: 644, // so-ET
+ 0x429000a4: 645, // so-KE
+ 0x42900115: 646, // so-SO
+ 0x43100000: 647, // sq
+ 0x43100027: 648, // sq-AL
+ 0x431000c2: 649, // sq-MK
+ 0x4310014d: 650, // sq-XK
+ 0x43200000: 651, // sr
+ 0x4321f000: 652, // sr-Cyrl
+ 0x4321f033: 653, // sr-Cyrl-BA
+ 0x4321f0bd: 654, // sr-Cyrl-ME
+ 0x4321f105: 655, // sr-Cyrl-RS
+ 0x4321f14d: 656, // sr-Cyrl-XK
+ 0x43257000: 657, // sr-Latn
+ 0x43257033: 658, // sr-Latn-BA
+ 0x432570bd: 659, // sr-Latn-ME
+ 0x43257105: 660, // sr-Latn-RS
+ 0x4325714d: 661, // sr-Latn-XK
+ 0x43700000: 662, // ss
+ 0x43a00000: 663, // ssy
+ 0x43b00000: 664, // st
+ 0x44400000: 665, // sv
+ 0x44400031: 666, // sv-AX
+ 0x44400072: 667, // sv-FI
+ 0x4440010c: 668, // sv-SE
+ 0x44500000: 669, // sw
+ 0x4450004b: 670, // sw-CD
+ 0x445000a4: 671, // sw-KE
+ 0x4450012f: 672, // sw-TZ
+ 0x44500131: 673, // sw-UG
+ 0x44e00000: 674, // syr
+ 0x45000000: 675, // ta
+ 0x45000099: 676, // ta-IN
+ 0x450000b3: 677, // ta-LK
+ 0x450000d0: 678, // ta-MY
+ 0x4500010d: 679, // ta-SG
+ 0x46100000: 680, // te
+ 0x46100099: 681, // te-IN
+ 0x46400000: 682, // teo
+ 0x464000a4: 683, // teo-KE
+ 0x46400131: 684, // teo-UG
+ 0x46700000: 685, // tg
+ 0x46700124: 686, // tg-TJ
+ 0x46b00000: 687, // th
+ 0x46b00123: 688, // th-TH
+ 0x46f00000: 689, // ti
+ 0x46f0006d: 690, // ti-ER
+ 0x46f0006f: 691, // ti-ET
+ 0x47100000: 692, // tig
+ 0x47600000: 693, // tk
+ 0x47600127: 694, // tk-TM
+ 0x48000000: 695, // tn
+ 0x48200000: 696, // to
+ 0x48200129: 697, // to-TO
+ 0x48a00000: 698, // tr
+ 0x48a0005d: 699, // tr-CY
+ 0x48a0012b: 700, // tr-TR
+ 0x48e00000: 701, // ts
+ 0x49400000: 702, // tt
+ 0x49400106: 703, // tt-RU
+ 0x4a400000: 704, // twq
+ 0x4a4000d4: 705, // twq-NE
+ 0x4a900000: 706, // tzm
+ 0x4a9000ba: 707, // tzm-MA
+ 0x4ac00000: 708, // ug
+ 0x4ac00053: 709, // ug-CN
+ 0x4ae00000: 710, // uk
+ 0x4ae00130: 711, // uk-UA
+ 0x4b400000: 712, // ur
+ 0x4b400099: 713, // ur-IN
+ 0x4b4000e8: 714, // ur-PK
+ 0x4bc00000: 715, // uz
+ 0x4bc05000: 716, // uz-Arab
+ 0x4bc05024: 717, // uz-Arab-AF
+ 0x4bc1f000: 718, // uz-Cyrl
+ 0x4bc1f137: 719, // uz-Cyrl-UZ
+ 0x4bc57000: 720, // uz-Latn
+ 0x4bc57137: 721, // uz-Latn-UZ
+ 0x4be00000: 722, // vai
+ 0x4be57000: 723, // vai-Latn
+ 0x4be570b4: 724, // vai-Latn-LR
+ 0x4bee3000: 725, // vai-Vaii
+ 0x4bee30b4: 726, // vai-Vaii-LR
+ 0x4c000000: 727, // ve
+ 0x4c300000: 728, // vi
+ 0x4c30013e: 729, // vi-VN
+ 0x4c900000: 730, // vo
+ 0x4c900001: 731, // vo-001
+ 0x4cc00000: 732, // vun
+ 0x4cc0012f: 733, // vun-TZ
+ 0x4ce00000: 734, // wa
+ 0x4cf00000: 735, // wae
+ 0x4cf0004e: 736, // wae-CH
+ 0x4e500000: 737, // wo
+ 0x4e500114: 738, // wo-SN
+ 0x4f200000: 739, // xh
+ 0x4fb00000: 740, // xog
+ 0x4fb00131: 741, // xog-UG
+ 0x50900000: 742, // yav
+ 0x50900052: 743, // yav-CM
+ 0x51200000: 744, // yi
+ 0x51200001: 745, // yi-001
+ 0x51800000: 746, // yo
+ 0x5180003b: 747, // yo-BJ
+ 0x518000d6: 748, // yo-NG
+ 0x51f00000: 749, // yue
+ 0x51f38000: 750, // yue-Hans
+ 0x51f38053: 751, // yue-Hans-CN
+ 0x51f39000: 752, // yue-Hant
+ 0x51f3908d: 753, // yue-Hant-HK
+ 0x52800000: 754, // zgh
+ 0x528000ba: 755, // zgh-MA
+ 0x52900000: 756, // zh
+ 0x52938000: 757, // zh-Hans
+ 0x52938053: 758, // zh-Hans-CN
+ 0x5293808d: 759, // zh-Hans-HK
+ 0x529380c6: 760, // zh-Hans-MO
+ 0x5293810d: 761, // zh-Hans-SG
+ 0x52939000: 762, // zh-Hant
+ 0x5293908d: 763, // zh-Hant-HK
+ 0x529390c6: 764, // zh-Hant-MO
+ 0x5293912e: 765, // zh-Hant-TW
+ 0x52f00000: 766, // zu
+ 0x52f00161: 767, // zu-ZA
+}
+
+// Total table size 4676 bytes (4KiB); checksum: 17BE3673
diff --git a/language/internal/language.go b/language/internal/language.go
new file mode 100644
index 0000000..f81f976
--- /dev/null
+++ b/language/internal/language.go
@@ -0,0 +1,919 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:generate go run gen.go gen_common.go -output tables.go
+//go:generate go run gen_index.go
+
+package language
+
+// TODO: Remove above NOTE after:
+// - verifying that tables are dropped correctly (most notably matcher tables).
+
+import (
+ "errors"
+ "fmt"
+ "strings"
+)
+
+const (
+ // maxCoreSize is the maximum size of a BCP 47 tag without variants and
+ // extensions. Equals max lang (3) + script (4) + max reg (3) + 2 dashes.
+ maxCoreSize = 12
+
+ // max99thPercentileSize is a somewhat arbitrary buffer size that presumably
+ // is large enough to hold at least 99% of the BCP 47 tags.
+ max99thPercentileSize = 32
+
+ // maxSimpleUExtensionSize is the maximum size of a -u extension with one
+ // key-type pair. Equals len("-u-") + key (2) + dash + max value (8).
+ maxSimpleUExtensionSize = 14
+)
+
+// Tag represents a BCP 47 language tag. It is used to specify an instance of a
+// specific language or locale. All language tag values are guaranteed to be
+// well-formed.
+type Tag struct {
+ lang langID
+ region regionID
+ // TODO: we will soon run out of positions for script. Idea: instead of
+ // storing lang, region, and script codes, store only the compact index and
+ // have a lookup table from this code to its expansion. This greatly speeds
+ // up table lookup, speed up common variant cases.
+ // This will also immediately free up 3 extra bytes. Also, the pVariant
+ // field can now be moved to the lookup table, as the compact index uniquely
+ // determines the offset of a possible variant.
+ script scriptID
+ pVariant byte // offset in str, includes preceding '-'
+ pExt uint16 // offset of first extension, includes preceding '-'
+
+ // str is the string representation of the Tag. It will only be used if the
+ // tag has variants or extensions.
+ str string
+}
+
+// Make is a convenience wrapper for Parse that omits the error.
+// In case of an error, a sensible default is returned.
+func Make(s string) Tag {
+ return Default.Make(s)
+}
+
+// Make is a convenience wrapper for c.Parse that omits the error.
+// In case of an error, a sensible default is returned.
+func (c CanonType) Make(s string) Tag {
+ t, _ := c.Parse(s)
+ return t
+}
+
+// Raw returns the raw base language, script and region, without making an
+// attempt to infer their values.
+func (t Tag) Raw() (b Base, s Script, r Region) {
+ return Base{t.lang}, Script{t.script}, Region{t.region}
+}
+
+// equalTags compares language, script and region subtags only.
+func (t Tag) equalTags(a Tag) bool {
+ return t.lang == a.lang && t.script == a.script && t.region == a.region
+}
+
+// IsRoot returns true if t is equal to language "und".
+func (t Tag) IsRoot() bool {
+ if int(t.pVariant) < len(t.str) {
+ return false
+ }
+ return t.equalTags(und)
+}
+
+// private reports whether the Tag consists solely of a private use tag.
+func (t Tag) private() bool {
+ return t.str != "" && t.pVariant == 0
+}
+
+// CanonType can be used to enable or disable various types of canonicalization.
+type CanonType int
+
+const (
+ // Replace deprecated base languages with their preferred replacements.
+ DeprecatedBase CanonType = 1 << iota
+ // Replace deprecated scripts with their preferred replacements.
+ DeprecatedScript
+ // Replace deprecated regions with their preferred replacements.
+ DeprecatedRegion
+ // Remove redundant scripts.
+ SuppressScript
+ // Normalize legacy encodings. This includes legacy languages defined in
+ // CLDR as well as bibliographic codes defined in ISO-639.
+ Legacy
+ // Map the dominant language of a macro language group to the macro language
+ // subtag. For example cmn -> zh.
+ Macro
+ // The CLDR flag should be used if full compatibility with CLDR is required.
+ // There are a few cases where language.Tag may differ from CLDR. To follow all
+ // of CLDR's suggestions, use All|CLDR.
+ CLDR
+
+ // Raw can be used to Compose or Parse without Canonicalization.
+ Raw CanonType = 0
+
+ // Replace all deprecated tags with their preferred replacements.
+ Deprecated = DeprecatedBase | DeprecatedScript | DeprecatedRegion
+
+ // All canonicalizations recommended by BCP 47.
+ BCP47 = Deprecated | SuppressScript
+
+ // All canonicalizations.
+ All = BCP47 | Legacy | Macro
+
+ // Default is the canonicalization used by Parse, Make and Compose. To
+ // preserve as much information as possible, canonicalizations that remove
+ // potentially valuable information are not included. The Matcher is
+ // designed to recognize similar tags that would be the same if
+ // they were canonicalized using All.
+ Default = Deprecated | Legacy
+
+ canonLang = DeprecatedBase | Legacy | Macro
+
+ // TODO: LikelyScript, LikelyRegion: suppress similar to ICU.
+)
+
+// canonicalize returns the canonicalized equivalent of the tag and
+// whether there was any change.
+func (t Tag) canonicalize(c CanonType) (Tag, bool) {
+ if c == Raw {
+ return t, false
+ }
+ changed := false
+ if c&SuppressScript != 0 {
+ if t.lang < langNoIndexOffset && uint8(t.script) == suppressScript[t.lang] {
+ t.script = 0
+ changed = true
+ }
+ }
+ if c&canonLang != 0 {
+ for {
+ if l, aliasType := normLang(t.lang); l != t.lang {
+ switch aliasType {
+ case langLegacy:
+ if c&Legacy != 0 {
+ if t.lang == _sh && t.script == 0 {
+ t.script = _Latn
+ }
+ t.lang = l
+ changed = true
+ }
+ case langMacro:
+ if c&Macro != 0 {
+ // We deviate here from CLDR. The mapping "nb" -> "no"
+ // qualifies as a typical Macro language mapping. However,
+ // for legacy reasons, CLDR maps "no", the macro language
+ // code for Norwegian, to the dominant variant "nb". This
+ // change is currently under consideration for CLDR as well.
+ // See http://unicode.org/cldr/trac/ticket/2698 and also
+ // http://unicode.org/cldr/trac/ticket/1790 for some of the
+ // practical implications. TODO: this check could be removed
+ // if CLDR adopts this change.
+ if c&CLDR == 0 || t.lang != _nb {
+ changed = true
+ t.lang = l
+ }
+ }
+ case langDeprecated:
+ if c&DeprecatedBase != 0 {
+ if t.lang == _mo && t.region == 0 {
+ t.region = _MD
+ }
+ t.lang = l
+ changed = true
+ // Other canonicalization types may still apply.
+ continue
+ }
+ }
+ } else if c&Legacy != 0 && t.lang == _no && c&CLDR != 0 {
+ t.lang = _nb
+ changed = true
+ }
+ break
+ }
+ }
+ if c&DeprecatedScript != 0 {
+ if t.script == _Qaai {
+ changed = true
+ t.script = _Zinh
+ }
+ }
+ if c&DeprecatedRegion != 0 {
+ if r := normRegion(t.region); r != 0 {
+ changed = true
+ t.region = r
+ }
+ }
+ return t, changed
+}
+
+// Canonicalize returns the canonicalized equivalent of the tag.
+func (c CanonType) Canonicalize(t Tag) (Tag, error) {
+ t, changed := t.canonicalize(c)
+ if changed {
+ t.remakeString()
+ }
+ return t, nil
+}
+
+// Confidence indicates the level of certainty for a given return value.
+// For example, Serbian may be written in Cyrillic or Latin script.
+// The confidence level indicates whether a value was explicitly specified,
+// whether it is typically the only possible value, or whether there is
+// an ambiguity.
+type Confidence int
+
+const (
+ No Confidence = iota // full confidence that there was no match
+ Low // most likely value picked out of a set of alternatives
+ High // value is generally assumed to be the correct match
+ Exact // exact match or explicitly specified value
+)
+
+var confName = []string{"No", "Low", "High", "Exact"}
+
+func (c Confidence) String() string {
+ return confName[c]
+}
+
+// remakeString is used to update t.str in case lang, script or region changed.
+// It is assumed that pExt and pVariant still point to the start of the
+// respective parts.
+func (t *Tag) remakeString() {
+ if t.str == "" {
+ return
+ }
+ extra := t.str[t.pVariant:]
+ if t.pVariant > 0 {
+ extra = extra[1:]
+ }
+ if t.equalTags(und) && strings.HasPrefix(extra, "x-") {
+ t.str = extra
+ t.pVariant = 0
+ t.pExt = 0
+ return
+ }
+ var buf [max99thPercentileSize]byte // avoid extra memory allocation in most cases.
+ b := buf[:t.genCoreBytes(buf[:])]
+ if extra != "" {
+ diff := len(b) - int(t.pVariant)
+ b = append(b, '-')
+ b = append(b, extra...)
+ t.pVariant = uint8(int(t.pVariant) + diff)
+ t.pExt = uint16(int(t.pExt) + diff)
+ } else {
+ t.pVariant = uint8(len(b))
+ t.pExt = uint16(len(b))
+ }
+ t.str = string(b)
+}
+
+// genCoreBytes writes a string for the base languages, script and region tags
+// to the given buffer and returns the number of bytes written. It will never
+// write more than maxCoreSize bytes.
+func (t *Tag) genCoreBytes(buf []byte) int {
+ n := t.lang.stringToBuf(buf[:])
+ if t.script != 0 {
+ n += copy(buf[n:], "-")
+ n += copy(buf[n:], t.script.String())
+ }
+ if t.region != 0 {
+ n += copy(buf[n:], "-")
+ n += copy(buf[n:], t.region.String())
+ }
+ return n
+}
+
+// String returns the canonical string representation of the language tag.
+func (t Tag) String() string {
+ if t.str != "" {
+ return t.str
+ }
+ if t.script == 0 && t.region == 0 {
+ return t.lang.String()
+ }
+ buf := [maxCoreSize]byte{}
+ return string(buf[:t.genCoreBytes(buf[:])])
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (t Tag) MarshalText() (text []byte, err error) {
+ if t.str != "" {
+ text = append(text, t.str...)
+ } else if t.script == 0 && t.region == 0 {
+ text = append(text, t.lang.String()...)
+ } else {
+ buf := [maxCoreSize]byte{}
+ text = buf[:t.genCoreBytes(buf[:])]
+ }
+ return text, nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (t *Tag) UnmarshalText(text []byte) error {
+ tag, err := Raw.Parse(string(text))
+ *t = tag
+ return err
+}
+
+// Base returns the base language of the language tag. If the base language is
+// unspecified, an attempt will be made to infer it from the context.
+// It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
+func (t Tag) Base() (Base, Confidence) {
+ if t.lang != 0 {
+ return Base{t.lang}, Exact
+ }
+ c := High
+ if t.script == 0 && !(Region{t.region}).IsCountry() {
+ c = Low
+ }
+ if tag, err := addTags(t); err == nil && tag.lang != 0 {
+ return Base{tag.lang}, c
+ }
+ return Base{0}, No
+}
+
+// Script infers the script for the language tag. If it was not explicitly given, it will infer
+// a most likely candidate.
+// If more than one script is commonly used for a language, the most likely one
+// is returned with a low confidence indication. For example, it returns (Cyrl, Low)
+// for Serbian.
+// If a script cannot be inferred (Zzzz, No) is returned. We do not use Zyyy (undetermined)
+// as one would suspect from the IANA registry for BCP 47. In a Unicode context Zyyy marks
+// common characters (like 1, 2, 3, '.', etc.) and is therefore more like multiple scripts.
+// See http://www.unicode.org/reports/tr24/#Values for more details. Zzzz is also used for
+// unknown value in CLDR. (Zzzz, Exact) is returned if Zzzz was explicitly specified.
+// Note that an inferred script is never guaranteed to be the correct one. Latin is
+// almost exclusively used for Afrikaans, but Arabic has been used for some texts
+// in the past. Also, the script that is commonly used may change over time.
+// It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
+func (t Tag) Script() (Script, Confidence) {
+ if t.script != 0 {
+ return Script{t.script}, Exact
+ }
+ sc, c := scriptID(_Zzzz), No
+ if t.lang < langNoIndexOffset {
+ if scr := scriptID(suppressScript[t.lang]); scr != 0 {
+ // Note: it is not always the case that a language with a suppress
+ // script value is only written in one script (e.g. kk, ms, pa).
+ if t.region == 0 {
+ return Script{scriptID(scr)}, High
+ }
+ sc, c = scr, High
+ }
+ }
+ if tag, err := addTags(t); err == nil {
+ if tag.script != sc {
+ sc, c = tag.script, Low
+ }
+ } else {
+ t, _ = (Deprecated | Macro).Canonicalize(t)
+ if tag, err := addTags(t); err == nil && tag.script != sc {
+ sc, c = tag.script, Low
+ }
+ }
+ return Script{sc}, c
+}
+
+// Region returns the region for the language tag. If it was not explicitly given, it will
+// infer a most likely candidate from the context.
+// It uses a variant of CLDR's Add Likely Subtags algorithm. This is subject to change.
+func (t Tag) Region() (Region, Confidence) {
+ if t.region != 0 {
+ return Region{t.region}, Exact
+ }
+ if t, err := addTags(t); err == nil {
+ return Region{t.region}, Low // TODO: differentiate between high and low.
+ }
+ t, _ = (Deprecated | Macro).Canonicalize(t)
+ if tag, err := addTags(t); err == nil {
+ return Region{tag.region}, Low
+ }
+ return Region{_ZZ}, No // TODO: return world instead of undetermined?
+}
+
+// Variant returns the variants specified explicitly for this language tag.
+// or nil if no variant was specified.
+func (t Tag) Variants() []Variant {
+ v := []Variant{}
+ if int(t.pVariant) < int(t.pExt) {
+ for x, str := "", t.str[t.pVariant:t.pExt]; str != ""; {
+ x, str = nextToken(str)
+ v = append(v, Variant{x})
+ }
+ }
+ return v
+}
+
+// Parent returns the CLDR parent of t. In CLDR, missing fields in data for a
+// specific language are substituted with fields from the parent language.
+// The parent for a language may change for newer versions of CLDR.
+func (t Tag) Parent() Tag {
+ if t.str != "" {
+ // Strip the variants and extensions.
+ t, _ = Raw.Compose(t.Raw())
+ if t.region == 0 && t.script != 0 && t.lang != 0 {
+ base, _ := addTags(Tag{lang: t.lang})
+ if base.script == t.script {
+ return Tag{lang: t.lang}
+ }
+ }
+ return t
+ }
+ if t.lang != 0 {
+ if t.region != 0 {
+ maxScript := t.script
+ if maxScript == 0 {
+ max, _ := addTags(t)
+ maxScript = max.script
+ }
+
+ for i := range parents {
+ if langID(parents[i].lang) == t.lang && scriptID(parents[i].maxScript) == maxScript {
+ for _, r := range parents[i].fromRegion {
+ if regionID(r) == t.region {
+ return Tag{
+ lang: t.lang,
+ script: scriptID(parents[i].script),
+ region: regionID(parents[i].toRegion),
+ }
+ }
+ }
+ }
+ }
+
+ // Strip the script if it is the default one.
+ base, _ := addTags(Tag{lang: t.lang})
+ if base.script != maxScript {
+ return Tag{lang: t.lang, script: maxScript}
+ }
+ return Tag{lang: t.lang}
+ } else if t.script != 0 {
+ // The parent for an base-script pair with a non-default script is
+ // "und" instead of the base language.
+ base, _ := addTags(Tag{lang: t.lang})
+ if base.script != t.script {
+ return und
+ }
+ return Tag{lang: t.lang}
+ }
+ }
+ return und
+}
+
+// returns token t and the rest of the string.
+func nextToken(s string) (t, tail string) {
+ p := strings.Index(s[1:], "-")
+ if p == -1 {
+ return s[1:], ""
+ }
+ p++
+ return s[1:p], s[p:]
+}
+
+// Extension is a single BCP 47 extension.
+type Extension struct {
+ s string
+}
+
+// String returns the string representation of the extension, including the
+// type tag.
+func (e Extension) String() string {
+ return e.s
+}
+
+// ParseExtension parses s as an extension and returns it on success.
+func ParseExtension(s string) (e Extension, err error) {
+ scan := makeScannerString(s)
+ var end int
+ if n := len(scan.token); n != 1 {
+ return Extension{}, errSyntax
+ }
+ scan.toLower(0, len(scan.b))
+ end = parseExtension(&scan)
+ if end != len(s) {
+ return Extension{}, errSyntax
+ }
+ return Extension{string(scan.b)}, nil
+}
+
+// Type returns the one-byte extension type of e. It returns 0 for the zero
+// exception.
+func (e Extension) Type() byte {
+ if e.s == "" {
+ return 0
+ }
+ return e.s[0]
+}
+
+// Tokens returns the list of tokens of e.
+func (e Extension) Tokens() []string {
+ return strings.Split(e.s, "-")
+}
+
+// Extension returns the extension of type x for tag t. It will return
+// false for ok if t does not have the requested extension. The returned
+// extension will be invalid in this case.
+func (t Tag) Extension(x byte) (ext Extension, ok bool) {
+ for i := int(t.pExt); i < len(t.str)-1; {
+ var ext string
+ i, ext = getExtension(t.str, i)
+ if ext[0] == x {
+ return Extension{ext}, true
+ }
+ }
+ return Extension{}, false
+}
+
+// Extensions returns all extensions of t.
+func (t Tag) Extensions() []Extension {
+ e := []Extension{}
+ for i := int(t.pExt); i < len(t.str)-1; {
+ var ext string
+ i, ext = getExtension(t.str, i)
+ e = append(e, Extension{ext})
+ }
+ return e
+}
+
+// TypeForKey returns the type associated with the given key, where key and type
+// are of the allowed values defined for the Unicode locale extension ('u') in
+// http://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
+// TypeForKey will traverse the inheritance chain to get the correct value.
+func (t Tag) TypeForKey(key string) string {
+ if start, end, _ := t.findTypeForKey(key); end != start {
+ return t.str[start:end]
+ }
+ return ""
+}
+
+var (
+ errPrivateUse = errors.New("cannot set a key on a private use tag")
+ errInvalidArguments = errors.New("invalid key or type")
+)
+
+// SetTypeForKey returns a new Tag with the key set to type, where key and type
+// are of the allowed values defined for the Unicode locale extension ('u') in
+// http://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
+// An empty value removes an existing pair with the same key.
+func (t Tag) SetTypeForKey(key, value string) (Tag, error) {
+ if t.private() {
+ return t, errPrivateUse
+ }
+ if len(key) != 2 {
+ return t, errInvalidArguments
+ }
+
+ // Remove the setting if value is "".
+ if value == "" {
+ start, end, _ := t.findTypeForKey(key)
+ if start != end {
+ // Remove key tag and leading '-'.
+ start -= 4
+
+ // Remove a possible empty extension.
+ if (end == len(t.str) || t.str[end+2] == '-') && t.str[start-2] == '-' {
+ start -= 2
+ }
+ if start == int(t.pVariant) && end == len(t.str) {
+ t.str = ""
+ t.pVariant, t.pExt = 0, 0
+ } else {
+ t.str = fmt.Sprintf("%s%s", t.str[:start], t.str[end:])
+ }
+ }
+ return t, nil
+ }
+
+ if len(value) < 3 || len(value) > 8 {
+ return t, errInvalidArguments
+ }
+
+ var (
+ buf [maxCoreSize + maxSimpleUExtensionSize]byte
+ uStart int // start of the -u extension.
+ )
+
+ // Generate the tag string if needed.
+ if t.str == "" {
+ uStart = t.genCoreBytes(buf[:])
+ buf[uStart] = '-'
+ uStart++
+ }
+
+ // Create new key-type pair and parse it to verify.
+ b := buf[uStart:]
+ copy(b, "u-")
+ copy(b[2:], key)
+ b[4] = '-'
+ b = b[:5+copy(b[5:], value)]
+ scan := makeScanner(b)
+ if parseExtensions(&scan); scan.err != nil {
+ return t, scan.err
+ }
+
+ // Assemble the replacement string.
+ if t.str == "" {
+ t.pVariant, t.pExt = byte(uStart-1), uint16(uStart-1)
+ t.str = string(buf[:uStart+len(b)])
+ } else {
+ s := t.str
+ start, end, hasExt := t.findTypeForKey(key)
+ if start == end {
+ if hasExt {
+ b = b[2:]
+ }
+ t.str = fmt.Sprintf("%s-%s%s", s[:start], b, s[end:])
+ } else {
+ t.str = fmt.Sprintf("%s%s%s", s[:start], value, s[end:])
+ }
+ }
+ return t, nil
+}
+
+// findKeyAndType returns the start and end position for the type corresponding
+// to key or the point at which to insert the key-value pair if the type
+// wasn't found. The hasExt return value reports whether an -u extension was present.
+// Note: the extensions are typically very small and are likely to contain
+// only one key-type pair.
+func (t Tag) findTypeForKey(key string) (start, end int, hasExt bool) {
+ p := int(t.pExt)
+ if len(key) != 2 || p == len(t.str) || p == 0 {
+ return p, p, false
+ }
+ s := t.str
+
+ // Find the correct extension.
+ for p++; s[p] != 'u'; p++ {
+ if s[p] > 'u' {
+ p--
+ return p, p, false
+ }
+ if p = nextExtension(s, p); p == len(s) {
+ return len(s), len(s), false
+ }
+ }
+ // Proceed to the hyphen following the extension name.
+ p++
+
+ // curKey is the key currently being processed.
+ curKey := ""
+
+ // Iterate over keys until we get the end of a section.
+ for {
+ // p points to the hyphen preceding the current token.
+ if p3 := p + 3; s[p3] == '-' {
+ // Found a key.
+ // Check whether we just processed the key that was requested.
+ if curKey == key {
+ return start, p, true
+ }
+ // Set to the next key and continue scanning type tokens.
+ curKey = s[p+1 : p3]
+ if curKey > key {
+ return p, p, true
+ }
+ // Start of the type token sequence.
+ start = p + 4
+ // A type is at least 3 characters long.
+ p += 7 // 4 + 3
+ } else {
+ // Attribute or type, which is at least 3 characters long.
+ p += 4
+ }
+ // p points past the third character of a type or attribute.
+ max := p + 5 // maximum length of token plus hyphen.
+ if len(s) < max {
+ max = len(s)
+ }
+ for ; p < max && s[p] != '-'; p++ {
+ }
+ // Bail if we have exhausted all tokens or if the next token starts
+ // a new extension.
+ if p == len(s) || s[p+2] == '-' {
+ if curKey == key {
+ return start, p, true
+ }
+ return p, p, true
+ }
+ }
+}
+
+// CompactIndex returns an index, where 0 <= index < NumCompactTags, for tags
+// for which data exists in the text repository. The index will change over time
+// and should not be stored in persistent storage. If t does not match a compact
+// index, exact will be false and the compact index will be returned for the
+// first match after repeatedly taking the Parent of t.
+func CompactIndex(t Tag) (index int, exact bool) {
+ // TODO: perhaps give more frequent tags a lower index.
+ // TODO: we could make the indexes stable. This will excluded some
+ // possibilities for optimization, so don't do this quite yet.
+ exact = true
+
+ b, s, r := t.Raw()
+ switch {
+ case len(t.str) > 0:
+ if strings.HasPrefix(t.str, "x-") {
+ // We have no entries for user-defined tags.
+ return 0, false
+ }
+ if uint16(t.pVariant) != t.pExt {
+ if int(t.pExt) < len(t.str) {
+ exact = false
+ t, _ = Raw.Compose(b, s, r, t.Variants())
+ }
+ } else if _, ok := t.Extension('u'); ok {
+ // TODO: va may mean something else. Consider not considering it.
+ // Strip all but the 'va' entry.
+ old := t
+ variant := t.TypeForKey("va")
+ t, _ = Raw.Compose(b, s, r)
+ if variant != "" {
+ t, _ = t.SetTypeForKey("va", variant)
+ }
+ exact = old == t
+ }
+ if len(t.str) > 0 {
+ // We have some variants.
+ for i, s := range specialTags {
+ if s == t {
+ return i + 1, exact
+ }
+ }
+ exact = false
+ }
+ }
+ for ; t != Und; t = t.Parent() {
+ // No variants specified: just compare core components.
+ // The key has the form lllssrrr, where l, s, and r are nibbles for
+ // respectively the langID, scriptID, and regionID.
+ key := uint32(b.langID) << (8 + 12)
+ key |= uint32(s.scriptID) << 12
+ key |= uint32(r.regionID)
+ if x, ok := coreTags[key]; ok {
+ return int(x), exact
+ }
+ exact = false
+ }
+ return int(0), exact
+}
+
+// Base is an ISO 639 language code, used for encoding the base language
+// of a language tag.
+type Base struct {
+ langID
+}
+
+// ParseBase parses a 2- or 3-letter ISO 639 code.
+// It returns a ValueError if s is a well-formed but unknown language identifier
+// or another error if another error occurred.
+func ParseBase(s string) (Base, error) {
+ if n := len(s); n < 2 || 3 < n {
+ return Base{}, errSyntax
+ }
+ var buf [3]byte
+ l, err := getLangID(buf[:copy(buf[:], s)])
+ return Base{l}, err
+}
+
+// Script is a 4-letter ISO 15924 code for representing scripts.
+// It is idiomatically represented in title case.
+type Script struct {
+ scriptID
+}
+
+// ParseScript parses a 4-letter ISO 15924 code.
+// It returns a ValueError if s is a well-formed but unknown script identifier
+// or another error if another error occurred.
+func ParseScript(s string) (Script, error) {
+ if len(s) != 4 {
+ return Script{}, errSyntax
+ }
+ var buf [4]byte
+ sc, err := getScriptID(script, buf[:copy(buf[:], s)])
+ return Script{sc}, err
+}
+
+// Region is an ISO 3166-1 or UN M.49 code for representing countries and regions.
+type Region struct {
+ regionID
+}
+
+// EncodeM49 returns the Region for the given UN M.49 code.
+// It returns an error if r is not a valid code.
+func EncodeM49(r int) (Region, error) {
+ rid, err := getRegionM49(r)
+ return Region{rid}, err
+}
+
+// ParseRegion parses a 2- or 3-letter ISO 3166-1 or a UN M.49 code.
+// It returns a ValueError if s is a well-formed but unknown region identifier
+// or another error if another error occurred.
+func ParseRegion(s string) (Region, error) {
+ if n := len(s); n < 2 || 3 < n {
+ return Region{}, errSyntax
+ }
+ var buf [3]byte
+ r, err := getRegionID(buf[:copy(buf[:], s)])
+ return Region{r}, err
+}
+
+// IsCountry returns whether this region is a country or autonomous area. This
+// includes non-standard definitions from CLDR.
+func (r Region) IsCountry() bool {
+ if r.regionID == 0 || r.IsGroup() || r.IsPrivateUse() && r.regionID != _XK {
+ return false
+ }
+ return true
+}
+
+// IsGroup returns whether this region defines a collection of regions. This
+// includes non-standard definitions from CLDR.
+func (r Region) IsGroup() bool {
+ if r.regionID == 0 {
+ return false
+ }
+ return int(regionInclusion[r.regionID]) < len(regionContainment)
+}
+
+// Contains returns whether Region c is contained by Region r. It returns true
+// if c == r.
+func (r Region) Contains(c Region) bool {
+ return r.regionID.contains(c.regionID)
+}
+
+func (r regionID) contains(c regionID) bool {
+ if r == c {
+ return true
+ }
+ g := regionInclusion[r]
+ if g >= nRegionGroups {
+ return false
+ }
+ m := regionContainment[g]
+
+ d := regionInclusion[c]
+ b := regionInclusionBits[d]
+
+ // A contained country may belong to multiple disjoint groups. Matching any
+ // of these indicates containment. If the contained region is a group, it
+ // must strictly be a subset.
+ if d >= nRegionGroups {
+ return b&m != 0
+ }
+ return b&^m == 0
+}
+
+var errNoTLD = errors.New("language: region is not a valid ccTLD")
+
+// TLD returns the country code top-level domain (ccTLD). UK is returned for GB.
+// In all other cases it returns either the region itself or an error.
+//
+// This method may return an error for a region for which there exists a
+// canonical form with a ccTLD. To get that ccTLD canonicalize r first. The
+// region will already be canonicalized it was obtained from a Tag that was
+// obtained using any of the default methods.
+func (r Region) TLD() (Region, error) {
+ // See http://en.wikipedia.org/wiki/Country_code_top-level_domain for the
+ // difference between ISO 3166-1 and IANA ccTLD.
+ if r.regionID == _GB {
+ r = Region{_UK}
+ }
+ if (r.typ() & ccTLD) == 0 {
+ return Region{}, errNoTLD
+ }
+ return r, nil
+}
+
+// Canonicalize returns the region or a possible replacement if the region is
+// deprecated. It will not return a replacement for deprecated regions that
+// are split into multiple regions.
+func (r Region) Canonicalize() Region {
+ if cr := normRegion(r.regionID); cr != 0 {
+ return Region{cr}
+ }
+ return r
+}
+
+// Variant represents a registered variant of a language as defined by BCP 47.
+type Variant struct {
+ variant string
+}
+
+// ParseVariant parses and returns a Variant. An error is returned if s is not
+// a valid variant.
+func ParseVariant(s string) (Variant, error) {
+ s = strings.ToLower(s)
+ if _, ok := variantIndex[s]; ok {
+ return Variant{s}, nil
+ }
+ return Variant{}, mkErrInvalid([]byte(s))
+}
+
+// String returns the string representation of the variant.
+func (v Variant) String() string {
+ return v.variant
+}
diff --git a/language/internal/language_test.go b/language/internal/language_test.go
new file mode 100644
index 0000000..14bc8a7
--- /dev/null
+++ b/language/internal/language_test.go
@@ -0,0 +1,911 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language
+
+import (
+ "reflect"
+ "testing"
+
+ "golang.org/x/text/internal/testtext"
+)
+
+func TestTagSize(t *testing.T) {
+ id := Tag{}
+ typ := reflect.TypeOf(id)
+ if typ.Size() > 24 {
+ t.Errorf("size of Tag was %d; want 24", typ.Size())
+ }
+}
+
+func TestIsRoot(t *testing.T) {
+ loc := Tag{}
+ if !loc.IsRoot() {
+ t.Errorf("unspecified should be root.")
+ }
+ for i, tt := range parseTests() {
+ loc, _ := Parse(tt.in)
+ undef := tt.lang == "und" && tt.script == "" && tt.region == "" && tt.ext == ""
+ if loc.IsRoot() != undef {
+ t.Errorf("%d: was %v; want %v", i, loc.IsRoot(), undef)
+ }
+ }
+}
+
+func TestEquality(t *testing.T) {
+ for i, tt := range parseTests()[48:49] {
+ s := tt.in
+ tag := Make(s)
+ t1 := Make(tag.String())
+ if tag != t1 {
+ t.Errorf("%d:%s: equality test 1 failed\n got: %#v\nwant: %#v)", i, s, t1, tag)
+ }
+ t2, _ := Compose(tag)
+ if tag != t2 {
+ t.Errorf("%d:%s: equality test 2 failed\n got: %#v\nwant: %#v", i, s, t2, tag)
+ }
+ }
+}
+
+func TestMakeString(t *testing.T) {
+ tests := []struct{ in, out string }{
+ {"und", "und"},
+ {"und", "und-CW"},
+ {"nl", "nl-NL"},
+ {"de-1901", "nl-1901"},
+ {"de-1901", "de-Arab-1901"},
+ {"x-a-b", "de-Arab-x-a-b"},
+ {"x-a-b", "x-a-b"},
+ }
+ for i, tt := range tests {
+ id, _ := Parse(tt.in)
+ mod, _ := Parse(tt.out)
+ id.setTagsFrom(mod)
+ for j := 0; j < 2; j++ {
+ id.remakeString()
+ if str := id.String(); str != tt.out {
+ t.Errorf("%d:%d: found %s; want %s", i, j, id.String(), tt.out)
+ }
+ }
+ // The bytes to string conversion as used in remakeString
+ // occasionally measures as more than one alloc, breaking this test.
+ // To alleviate this we set the number of runs to more than 1.
+ if n := testtext.AllocsPerRun(8, id.remakeString); n > 1 {
+ t.Errorf("%d: # allocs got %.1f; want <= 1", i, n)
+ }
+ }
+}
+
+func TestCompactIndex(t *testing.T) {
+ tests := []struct {
+ tag string
+ index int
+ ok bool
+ }{
+ // TODO: these values will change with each CLDR update. This issue
+ // will be solved if we decide to fix the indexes.
+ {"und", 0, true},
+ {"ca-ES-valencia", 1, true},
+ {"ca-ES-valencia-u-va-posix", 1, false},
+ {"ca-ES-valencia-u-co-phonebk", 1, false},
+ {"ca-ES-valencia-u-co-phonebk-va-posix", 1, false},
+ {"x-klingon", 0, false},
+ {"en-US", 232, true},
+ {"en-US-u-va-posix", 2, true},
+ {"en", 136, true},
+ {"en-u-co-phonebk", 136, false},
+ {"en-001", 137, true},
+ {"sh", 0, false}, // We don't normalize.
+ }
+ for _, tt := range tests {
+ x, ok := CompactIndex(Raw.MustParse(tt.tag))
+ if x != tt.index || ok != tt.ok {
+ t.Errorf("%s: got %d, %v; want %d %v", tt.tag, x, ok, tt.index, tt.ok)
+ }
+ }
+}
+
+func TestMarshal(t *testing.T) {
+ testCases := []string{
+ // TODO: these values will change with each CLDR update. This issue
+ // will be solved if we decide to fix the indexes.
+ "und",
+ "ca-ES-valencia",
+ "ca-ES-valencia-u-va-posix",
+ "ca-ES-valencia-u-co-phonebk",
+ "ca-ES-valencia-u-co-phonebk-va-posix",
+ "x-klingon",
+ "en-US",
+ "en-US-u-va-posix",
+ "en",
+ "en-u-co-phonebk",
+ "en-001",
+ "sh",
+ }
+ for _, tc := range testCases {
+ var tag Tag
+ err := tag.UnmarshalText([]byte(tc))
+ if err != nil {
+ t.Errorf("UnmarshalText(%q): unexpected error: %v", tc, err)
+ }
+ b, err := tag.MarshalText()
+ if err != nil {
+ t.Errorf("MarshalText(%q): unexpected error: %v", tc, err)
+ }
+ if got := string(b); got != tc {
+ t.Errorf("%s: got %q; want %q", tc, got, tc)
+ }
+ }
+}
+
+func TestBase(t *testing.T) {
+ tests := []struct {
+ loc, lang string
+ conf Confidence
+ }{
+ {"und", "en", Low},
+ {"x-abc", "und", No},
+ {"en", "en", Exact},
+ {"und-Cyrl", "ru", High},
+ // If a region is not included, the official language should be English.
+ {"und-US", "en", High},
+ // TODO: not-explicitly listed scripts should probably be und, No
+ // Modify addTags to return info on how the match was derived.
+ // {"und-Aghb", "und", No},
+ }
+ for i, tt := range tests {
+ loc, _ := Parse(tt.loc)
+ lang, conf := loc.Base()
+ if lang.String() != tt.lang {
+ t.Errorf("%d: language was %s; want %s", i, lang, tt.lang)
+ }
+ if conf != tt.conf {
+ t.Errorf("%d: confidence was %d; want %d", i, conf, tt.conf)
+ }
+ }
+}
+
+func TestParseBase(t *testing.T) {
+ tests := []struct {
+ in string
+ out string
+ ok bool
+ }{
+ {"en", "en", true},
+ {"EN", "en", true},
+ {"nld", "nl", true},
+ {"dut", "dut", true}, // bibliographic
+ {"aaj", "und", false}, // unknown
+ {"qaa", "qaa", true},
+ {"a", "und", false},
+ {"", "und", false},
+ {"aaaa", "und", false},
+ }
+ for i, tt := range tests {
+ x, err := ParseBase(tt.in)
+ if x.String() != tt.out || err == nil != tt.ok {
+ t.Errorf("%d:%s: was %s, %v; want %s, %v", i, tt.in, x, err == nil, tt.out, tt.ok)
+ }
+ if y, _, _ := Raw.Make(tt.out).Raw(); x != y {
+ t.Errorf("%d:%s: tag was %s; want %s", i, tt.in, x, y)
+ }
+ }
+}
+
+func TestScript(t *testing.T) {
+ tests := []struct {
+ loc, scr string
+ conf Confidence
+ }{
+ {"und", "Latn", Low},
+ {"en-Latn", "Latn", Exact},
+ {"en", "Latn", High},
+ {"sr", "Cyrl", Low},
+ {"kk", "Cyrl", High},
+ {"kk-CN", "Arab", Low},
+ {"cmn", "Hans", Low},
+ {"ru", "Cyrl", High},
+ {"ru-RU", "Cyrl", High},
+ {"yue", "Hant", Low},
+ {"x-abc", "Zzzz", Low},
+ {"und-zyyy", "Zyyy", Exact},
+ }
+ for i, tt := range tests {
+ loc, _ := Parse(tt.loc)
+ sc, conf := loc.Script()
+ if sc.String() != tt.scr {
+ t.Errorf("%d:%s: script was %s; want %s", i, tt.loc, sc, tt.scr)
+ }
+ if conf != tt.conf {
+ t.Errorf("%d:%s: confidence was %d; want %d", i, tt.loc, conf, tt.conf)
+ }
+ }
+}
+
+func TestParseScript(t *testing.T) {
+ tests := []struct {
+ in string
+ out string
+ ok bool
+ }{
+ {"Latn", "Latn", true},
+ {"zzzz", "Zzzz", true},
+ {"zyyy", "Zyyy", true},
+ {"Latm", "Zzzz", false},
+ {"Zzz", "Zzzz", false},
+ {"", "Zzzz", false},
+ {"Zzzxx", "Zzzz", false},
+ }
+ for i, tt := range tests {
+ x, err := ParseScript(tt.in)
+ if x.String() != tt.out || err == nil != tt.ok {
+ t.Errorf("%d:%s: was %s, %v; want %s, %v", i, tt.in, x, err == nil, tt.out, tt.ok)
+ }
+ if err == nil {
+ if _, y, _ := Raw.Make("und-" + tt.out).Raw(); x != y {
+ t.Errorf("%d:%s: tag was %s; want %s", i, tt.in, x, y)
+ }
+ }
+ }
+}
+
+func TestRegion(t *testing.T) {
+ tests := []struct {
+ loc, reg string
+ conf Confidence
+ }{
+ {"und", "US", Low},
+ {"en", "US", Low},
+ {"zh-Hant", "TW", Low},
+ {"en-US", "US", Exact},
+ {"cmn", "CN", Low},
+ {"ru", "RU", Low},
+ {"yue", "HK", Low},
+ {"x-abc", "ZZ", Low},
+ }
+ for i, tt := range tests {
+ loc, _ := Raw.Parse(tt.loc)
+ reg, conf := loc.Region()
+ if reg.String() != tt.reg {
+ t.Errorf("%d:%s: region was %s; want %s", i, tt.loc, reg, tt.reg)
+ }
+ if conf != tt.conf {
+ t.Errorf("%d:%s: confidence was %d; want %d", i, tt.loc, conf, tt.conf)
+ }
+ }
+}
+
+func TestEncodeM49(t *testing.T) {
+ tests := []struct {
+ m49 int
+ code string
+ ok bool
+ }{
+ {1, "001", true},
+ {840, "US", true},
+ {899, "ZZ", false},
+ }
+ for i, tt := range tests {
+ if r, err := EncodeM49(tt.m49); r.String() != tt.code || err == nil != tt.ok {
+ t.Errorf("%d:%d: was %s, %v; want %s, %v", i, tt.m49, r, err == nil, tt.code, tt.ok)
+ }
+ }
+ for i := 1; i <= 1000; i++ {
+ if r, err := EncodeM49(i); err == nil && r.M49() == 0 {
+ t.Errorf("%d has no error, but maps to undefined region", i)
+ }
+ }
+}
+
+func TestParseRegion(t *testing.T) {
+ tests := []struct {
+ in string
+ out string
+ ok bool
+ }{
+ {"001", "001", true},
+ {"840", "US", true},
+ {"899", "ZZ", false},
+ {"USA", "US", true},
+ {"US", "US", true},
+ {"BC", "ZZ", false},
+ {"C", "ZZ", false},
+ {"CCCC", "ZZ", false},
+ {"01", "ZZ", false},
+ }
+ for i, tt := range tests {
+ r, err := ParseRegion(tt.in)
+ if r.String() != tt.out || err == nil != tt.ok {
+ t.Errorf("%d:%s: was %s, %v; want %s, %v", i, tt.in, r, err == nil, tt.out, tt.ok)
+ }
+ if err == nil {
+ if _, _, y := Raw.Make("und-" + tt.out).Raw(); r != y {
+ t.Errorf("%d:%s: tag was %s; want %s", i, tt.in, r, y)
+ }
+ }
+ }
+}
+
+func TestIsCountry(t *testing.T) {
+ tests := []struct {
+ reg string
+ country bool
+ }{
+ {"US", true},
+ {"001", false},
+ {"958", false},
+ {"419", false},
+ {"203", true},
+ {"020", true},
+ {"900", false},
+ {"999", false},
+ {"QO", false},
+ {"EU", false},
+ {"AA", false},
+ {"XK", true},
+ }
+ for i, tt := range tests {
+ reg, _ := getRegionID([]byte(tt.reg))
+ r := Region{reg}
+ if r.IsCountry() != tt.country {
+ t.Errorf("%d: IsCountry(%s) was %v; want %v", i, tt.reg, r.IsCountry(), tt.country)
+ }
+ }
+}
+
+func TestIsGroup(t *testing.T) {
+ tests := []struct {
+ reg string
+ group bool
+ }{
+ {"US", false},
+ {"001", true},
+ {"958", false},
+ {"419", true},
+ {"203", false},
+ {"020", false},
+ {"900", false},
+ {"999", false},
+ {"QO", true},
+ {"EU", true},
+ {"AA", false},
+ {"XK", false},
+ }
+ for i, tt := range tests {
+ reg, _ := getRegionID([]byte(tt.reg))
+ r := Region{reg}
+ if r.IsGroup() != tt.group {
+ t.Errorf("%d: IsGroup(%s) was %v; want %v", i, tt.reg, r.IsGroup(), tt.group)
+ }
+ }
+}
+
+func TestContains(t *testing.T) {
+ tests := []struct {
+ enclosing, contained string
+ contains bool
+ }{
+ // A region contains itself.
+ {"US", "US", true},
+ {"001", "001", true},
+
+ // Direct containment.
+ {"001", "002", true},
+ {"039", "XK", true},
+ {"150", "XK", true},
+ {"EU", "AT", true},
+ {"QO", "AQ", true},
+
+ // Indirect containemnt.
+ {"001", "US", true},
+ {"001", "419", true},
+ {"001", "013", true},
+
+ // No containment.
+ {"US", "001", false},
+ {"155", "EU", false},
+ }
+ for i, tt := range tests {
+ enc, _ := getRegionID([]byte(tt.enclosing))
+ con, _ := getRegionID([]byte(tt.contained))
+ r := Region{enc}
+ if got := r.Contains(Region{con}); got != tt.contains {
+ t.Errorf("%d: %s.Contains(%s) was %v; want %v", i, tt.enclosing, tt.contained, got, tt.contains)
+ }
+ }
+}
+
+func TestRegionCanonicalize(t *testing.T) {
+ for i, tt := range []struct{ in, out string }{
+ {"UK", "GB"},
+ {"TP", "TL"},
+ {"QU", "EU"},
+ {"SU", "SU"},
+ {"VD", "VN"},
+ {"DD", "DE"},
+ } {
+ r := MustParseRegion(tt.in)
+ want := MustParseRegion(tt.out)
+ if got := r.Canonicalize(); got != want {
+ t.Errorf("%d: got %v; want %v", i, got, want)
+ }
+ }
+}
+
+func TestRegionTLD(t *testing.T) {
+ for _, tt := range []struct {
+ in, out string
+ ok bool
+ }{
+ {"EH", "EH", true},
+ {"FR", "FR", true},
+ {"TL", "TL", true},
+
+ // In ccTLD before in ISO.
+ {"GG", "GG", true},
+
+ // Non-standard assignment of ccTLD to ISO code.
+ {"GB", "UK", true},
+
+ // Exceptionally reserved in ISO and valid ccTLD.
+ {"UK", "UK", true},
+ {"AC", "AC", true},
+ {"EU", "EU", true},
+ {"SU", "SU", true},
+
+ // Exceptionally reserved in ISO and invalid ccTLD.
+ {"CP", "ZZ", false},
+ {"DG", "ZZ", false},
+ {"EA", "ZZ", false},
+ {"FX", "ZZ", false},
+ {"IC", "ZZ", false},
+ {"TA", "ZZ", false},
+
+ // Transitionally reserved in ISO (e.g. deprecated) but valid ccTLD as
+ // it is still being phased out.
+ {"AN", "AN", true},
+ {"TP", "TP", true},
+
+ // Transitionally reserved in ISO (e.g. deprecated) and invalid ccTLD.
+ // Defined in package language as it has a mapping in CLDR.
+ {"BU", "ZZ", false},
+ {"CS", "ZZ", false},
+ {"NT", "ZZ", false},
+ {"YU", "ZZ", false},
+ {"ZR", "ZZ", false},
+ // Not defined in package: SF.
+
+ // Indeterminately reserved in ISO.
+ // Defined in package language as it has a legacy mapping in CLDR.
+ {"DY", "ZZ", false},
+ {"RH", "ZZ", false},
+ {"VD", "ZZ", false},
+ // Not defined in package: EW, FL, JA, LF, PI, RA, RB, RC, RI, RL, RM,
+ // RN, RP, WG, WL, WV, and YV.
+
+ // Not assigned in ISO, but legacy definitions in CLDR.
+ {"DD", "ZZ", false},
+ {"YD", "ZZ", false},
+
+ // Normal mappings but somewhat special status in ccTLD.
+ {"BL", "BL", true},
+ {"MF", "MF", true},
+ {"BV", "BV", true},
+ {"SJ", "SJ", true},
+
+ // Have values when normalized, but not as is.
+ {"QU", "ZZ", false},
+
+ // ISO Private Use.
+ {"AA", "ZZ", false},
+ {"QM", "ZZ", false},
+ {"QO", "ZZ", false},
+ {"XA", "ZZ", false},
+ {"XK", "ZZ", false}, // Sometimes used for Kosovo, but invalid ccTLD.
+ } {
+ if tt.in == "" {
+ continue
+ }
+
+ r := MustParseRegion(tt.in)
+ var want Region
+ if tt.out != "ZZ" {
+ want = MustParseRegion(tt.out)
+ }
+ tld, err := r.TLD()
+ if got := err == nil; got != tt.ok {
+ t.Errorf("error(%v): got %v; want %v", r, got, tt.ok)
+ }
+ if tld != want {
+ t.Errorf("TLD(%v): got %v; want %v", r, tld, want)
+ }
+ }
+}
+
+func TestCanonicalize(t *testing.T) {
+ // TODO: do a full test using CLDR data in a separate regression test.
+ tests := []struct {
+ in, out string
+ option CanonType
+ }{
+ {"en-Latn", "en", SuppressScript},
+ {"sr-Cyrl", "sr-Cyrl", SuppressScript},
+ {"sh", "sr-Latn", Legacy},
+ {"sh-HR", "sr-Latn-HR", Legacy},
+ {"sh-Cyrl-HR", "sr-Cyrl-HR", Legacy},
+ {"tl", "fil", Legacy},
+ {"no", "no", Legacy},
+ {"no", "nb", Legacy | CLDR},
+ {"cmn", "cmn", Legacy},
+ {"cmn", "zh", Macro},
+ {"cmn-u-co-stroke", "zh-u-co-stroke", Macro},
+ {"yue", "yue", Macro},
+ {"nb", "no", Macro},
+ {"nb", "nb", Macro | CLDR},
+ {"no", "no", Macro},
+ {"no", "no", Macro | CLDR},
+ {"iw", "he", DeprecatedBase},
+ {"iw", "he", Deprecated | CLDR},
+ {"mo", "ro-MD", Deprecated}, // Adopted by CLDR as of version 25.
+ {"alb", "sq", Legacy}, // bibliographic
+ {"dut", "nl", Legacy}, // bibliographic
+ // As of CLDR 25, mo is no longer considered a legacy mapping.
+ {"mo", "mo", Legacy | CLDR},
+ {"und-AN", "und-AN", Deprecated},
+ {"und-YD", "und-YE", DeprecatedRegion},
+ {"und-YD", "und-YD", DeprecatedBase},
+ {"und-Qaai", "und-Zinh", DeprecatedScript},
+ {"und-Qaai", "und-Qaai", DeprecatedBase},
+ {"drh", "mn", All}, // drh -> khk -> mn
+ }
+ for i, tt := range tests {
+ in, _ := Raw.Parse(tt.in)
+ in, _ = tt.option.Canonicalize(in)
+ if in.String() != tt.out {
+ t.Errorf("%d:%s: was %s; want %s", i, tt.in, in.String(), tt.out)
+ }
+ if int(in.pVariant) > int(in.pExt) || int(in.pExt) > len(in.str) {
+ t.Errorf("%d:%s:offsets %d <= %d <= %d must be true", i, tt.in, in.pVariant, in.pExt, len(in.str))
+ }
+ }
+ // Test idempotence.
+ for _, base := range Supported.BaseLanguages() {
+ tag, _ := Raw.Compose(base)
+ got, _ := All.Canonicalize(tag)
+ want, _ := All.Canonicalize(got)
+ if got != want {
+ t.Errorf("idem(%s): got %s; want %s", tag, got, want)
+ }
+ }
+}
+
+func TestTypeForKey(t *testing.T) {
+ tests := []struct{ key, in, out string }{
+ {"co", "en", ""},
+ {"co", "en-u-abc", ""},
+ {"co", "en-u-co-phonebk", "phonebk"},
+ {"co", "en-u-co-phonebk-cu-aud", "phonebk"},
+ {"co", "x-foo-u-co-phonebk", ""},
+ {"nu", "en-u-co-phonebk-nu-arabic", "arabic"},
+ {"kc", "cmn-u-co-stroke", ""},
+ }
+ for _, tt := range tests {
+ if v := Make(tt.in).TypeForKey(tt.key); v != tt.out {
+ t.Errorf("%q[%q]: was %q; want %q", tt.in, tt.key, v, tt.out)
+ }
+ }
+}
+
+func TestSetTypeForKey(t *testing.T) {
+ tests := []struct {
+ key, value, in, out string
+ err bool
+ }{
+ // replace existing value
+ {"co", "pinyin", "en-u-co-phonebk", "en-u-co-pinyin", false},
+ {"co", "pinyin", "en-u-co-phonebk-cu-xau", "en-u-co-pinyin-cu-xau", false},
+ {"co", "pinyin", "en-u-co-phonebk-v-xx", "en-u-co-pinyin-v-xx", false},
+ {"co", "pinyin", "en-u-co-phonebk-x-x", "en-u-co-pinyin-x-x", false},
+ {"nu", "arabic", "en-u-co-phonebk-nu-vaai", "en-u-co-phonebk-nu-arabic", false},
+ // add to existing -u extension
+ {"co", "pinyin", "en-u-ca-gregory", "en-u-ca-gregory-co-pinyin", false},
+ {"co", "pinyin", "en-u-ca-gregory-nu-vaai", "en-u-ca-gregory-co-pinyin-nu-vaai", false},
+ {"co", "pinyin", "en-u-ca-gregory-v-va", "en-u-ca-gregory-co-pinyin-v-va", false},
+ {"co", "pinyin", "en-u-ca-gregory-x-a", "en-u-ca-gregory-co-pinyin-x-a", false},
+ {"ca", "gregory", "en-u-co-pinyin", "en-u-ca-gregory-co-pinyin", false},
+ // remove pair
+ {"co", "", "en-u-co-phonebk", "en", false},
+ {"co", "", "en-u-ca-gregory-co-phonebk", "en-u-ca-gregory", false},
+ {"co", "", "en-u-co-phonebk-nu-arabic", "en-u-nu-arabic", false},
+ {"co", "", "en", "en", false},
+ // add -u extension
+ {"co", "pinyin", "en", "en-u-co-pinyin", false},
+ {"co", "pinyin", "und", "und-u-co-pinyin", false},
+ {"co", "pinyin", "en-a-aaa", "en-a-aaa-u-co-pinyin", false},
+ {"co", "pinyin", "en-x-aaa", "en-u-co-pinyin-x-aaa", false},
+ {"co", "pinyin", "en-v-aa", "en-u-co-pinyin-v-aa", false},
+ {"co", "pinyin", "en-a-aaa-x-x", "en-a-aaa-u-co-pinyin-x-x", false},
+ {"co", "pinyin", "en-a-aaa-v-va", "en-a-aaa-u-co-pinyin-v-va", false},
+ // error on invalid values
+ {"co", "pinyinxxx", "en", "en", true},
+ {"co", "piny.n", "en", "en", true},
+ {"co", "pinyinxxx", "en-a-aaa", "en-a-aaa", true},
+ {"co", "pinyinxxx", "en-u-aaa", "en-u-aaa", true},
+ {"co", "pinyinxxx", "en-u-aaa-co-pinyin", "en-u-aaa-co-pinyin", true},
+ {"co", "pinyi.", "en-u-aaa-co-pinyin", "en-u-aaa-co-pinyin", true},
+ {"col", "pinyin", "en", "en", true},
+ {"co", "cu", "en", "en", true},
+ // error when setting on a private use tag
+ {"co", "phonebook", "x-foo", "x-foo", true},
+ }
+ for i, tt := range tests {
+ tag := Make(tt.in)
+ if v, err := tag.SetTypeForKey(tt.key, tt.value); v.String() != tt.out {
+ t.Errorf("%d:%q[%q]=%q: was %q; want %q", i, tt.in, tt.key, tt.value, v, tt.out)
+ } else if (err != nil) != tt.err {
+ t.Errorf("%d:%q[%q]=%q: error was %v; want %v", i, tt.in, tt.key, tt.value, err != nil, tt.err)
+ } else if val := v.TypeForKey(tt.key); err == nil && val != tt.value {
+ t.Errorf("%d:%q[%q]==%q: was %v; want %v", i, tt.out, tt.key, tt.value, val, tt.value)
+ }
+ if len(tag.String()) <= 3 {
+ // Simulate a tag for which the string has not been set.
+ tag.str, tag.pExt, tag.pVariant = "", 0, 0
+ if tag, err := tag.SetTypeForKey(tt.key, tt.value); err == nil {
+ if val := tag.TypeForKey(tt.key); err == nil && val != tt.value {
+ t.Errorf("%d:%q[%q]==%q: was %v; want %v", i, tt.out, tt.key, tt.value, val, tt.value)
+ }
+ }
+ }
+ }
+}
+
+func TestFindKeyAndType(t *testing.T) {
+ // out is either the matched type in case of a match or the original
+ // string up till the insertion point.
+ tests := []struct {
+ key string
+ hasExt bool
+ in, out string
+ }{
+ // Don't search past a private use extension.
+ {"co", false, "en-x-foo-u-co-pinyin", "en"},
+ {"co", false, "x-foo-u-co-pinyin", ""},
+ {"co", false, "en-s-fff-x-foo", "en-s-fff"},
+ // Insertion points in absence of -u extension.
+ {"cu", false, "en", ""}, // t.str is ""
+ {"cu", false, "en-v-va", "en"},
+ {"cu", false, "en-a-va", "en-a-va"},
+ {"cu", false, "en-a-va-v-va", "en-a-va"},
+ {"cu", false, "en-x-a", "en"},
+ // Tags with the -u extension.
+ {"co", true, "en-u-co-standard", "standard"},
+ {"co", true, "yue-u-co-pinyin", "pinyin"},
+ {"co", true, "en-u-co-abc", "abc"},
+ {"co", true, "en-u-co-abc-def", "abc-def"},
+ {"co", true, "en-u-co-abc-def-x-foo", "abc-def"},
+ {"co", true, "en-u-co-standard-nu-arab", "standard"},
+ {"co", true, "yue-u-co-pinyin-nu-arab", "pinyin"},
+ // Insertion points.
+ {"cu", true, "en-u-co-standard", "en-u-co-standard"},
+ {"cu", true, "yue-u-co-pinyin-x-foo", "yue-u-co-pinyin"},
+ {"cu", true, "en-u-co-abc", "en-u-co-abc"},
+ {"cu", true, "en-u-nu-arabic", "en-u"},
+ {"cu", true, "en-u-co-abc-def-nu-arabic", "en-u-co-abc-def"},
+ }
+ for i, tt := range tests {
+ start, end, hasExt := Make(tt.in).findTypeForKey(tt.key)
+ if start != end {
+ res := tt.in[start:end]
+ if res != tt.out {
+ t.Errorf("%d:%s: was %q; want %q", i, tt.in, res, tt.out)
+ }
+ } else {
+ if hasExt != tt.hasExt {
+ t.Errorf("%d:%s: hasExt was %v; want %v", i, tt.in, hasExt, tt.hasExt)
+ continue
+ }
+ if tt.in[:start] != tt.out {
+ t.Errorf("%d:%s: insertion point was %q; want %q", i, tt.in, tt.in[:start], tt.out)
+ }
+ }
+ }
+}
+
+func TestParent(t *testing.T) {
+ tests := []struct{ in, out string }{
+ // Strip variants and extensions first
+ {"de-u-co-phonebk", "de"},
+ {"de-1994", "de"},
+ {"de-Latn-1994", "de"}, // remove superfluous script.
+
+ // Ensure the canonical Tag for an entry is in the chain for base-script
+ // pairs.
+ {"zh-Hans", "zh"},
+
+ // Skip the script if it is the maximized version. CLDR files for the
+ // skipped tag are always empty.
+ {"zh-Hans-TW", "zh"},
+ {"zh-Hans-CN", "zh"},
+
+ // Insert the script if the maximized script is not the same as the
+ // maximized script of the base language.
+ {"zh-TW", "zh-Hant"},
+ {"zh-HK", "zh-Hant"},
+ {"zh-Hant-TW", "zh-Hant"},
+ {"zh-Hant-HK", "zh-Hant"},
+
+ // Non-default script skips to und.
+ // CLDR
+ {"az-Cyrl", "und"},
+ {"bs-Cyrl", "und"},
+ {"en-Dsrt", "und"},
+ {"ha-Arab", "und"},
+ {"mn-Mong", "und"},
+ {"pa-Arab", "und"},
+ {"shi-Latn", "und"},
+ {"sr-Latn", "und"},
+ {"uz-Arab", "und"},
+ {"uz-Cyrl", "und"},
+ {"vai-Latn", "und"},
+ {"zh-Hant", "und"},
+ // extra
+ {"nl-Cyrl", "und"},
+
+ // World english inherits from en-001.
+ {"en-150", "en-001"},
+ {"en-AU", "en-001"},
+ {"en-BE", "en-001"},
+ {"en-GG", "en-001"},
+ {"en-GI", "en-001"},
+ {"en-HK", "en-001"},
+ {"en-IE", "en-001"},
+ {"en-IM", "en-001"},
+ {"en-IN", "en-001"},
+ {"en-JE", "en-001"},
+ {"en-MT", "en-001"},
+ {"en-NZ", "en-001"},
+ {"en-PK", "en-001"},
+ {"en-SG", "en-001"},
+
+ // Spanish in Latin-American countries have es-419 as parent.
+ {"es-AR", "es-419"},
+ {"es-BO", "es-419"},
+ {"es-CL", "es-419"},
+ {"es-CO", "es-419"},
+ {"es-CR", "es-419"},
+ {"es-CU", "es-419"},
+ {"es-DO", "es-419"},
+ {"es-EC", "es-419"},
+ {"es-GT", "es-419"},
+ {"es-HN", "es-419"},
+ {"es-MX", "es-419"},
+ {"es-NI", "es-419"},
+ {"es-PA", "es-419"},
+ {"es-PE", "es-419"},
+ {"es-PR", "es-419"},
+ {"es-PY", "es-419"},
+ {"es-SV", "es-419"},
+ {"es-US", "es-419"},
+ {"es-UY", "es-419"},
+ {"es-VE", "es-419"},
+ // exceptions (according to CLDR)
+ {"es-CW", "es"},
+
+ // Inherit from pt-PT, instead of pt for these countries.
+ {"pt-AO", "pt-PT"},
+ {"pt-CV", "pt-PT"},
+ {"pt-GW", "pt-PT"},
+ {"pt-MO", "pt-PT"},
+ {"pt-MZ", "pt-PT"},
+ {"pt-ST", "pt-PT"},
+ {"pt-TL", "pt-PT"},
+ }
+ for _, tt := range tests {
+ tag := Raw.MustParse(tt.in)
+ if p := Raw.MustParse(tt.out); p != tag.Parent() {
+ t.Errorf("%s: was %v; want %v", tt.in, tag.Parent(), p)
+ }
+ }
+}
+
+var (
+ // Tags without error that don't need to be changed.
+ benchBasic = []string{
+ "en",
+ "en-Latn",
+ "en-GB",
+ "za",
+ "zh-Hant",
+ "zh",
+ "zh-HK",
+ "ar-MK",
+ "en-CA",
+ "fr-CA",
+ "fr-CH",
+ "fr",
+ "lv",
+ "he-IT",
+ "tlh",
+ "ja",
+ "ja-Jpan",
+ "ja-Jpan-JP",
+ "de-1996",
+ "de-CH",
+ "sr",
+ "sr-Latn",
+ }
+ // Tags with extensions, not changes required.
+ benchExt = []string{
+ "x-a-b-c-d",
+ "x-aa-bbbb-cccccccc-d",
+ "en-x_cc-b-bbb-a-aaa",
+ "en-c_cc-b-bbb-a-aaa-x-x",
+ "en-u-co-phonebk",
+ "en-Cyrl-u-co-phonebk",
+ "en-US-u-co-phonebk-cu-xau",
+ "en-nedix-u-co-phonebk",
+ "en-t-t0-abcd",
+ "en-t-nl-latn",
+ "en-t-t0-abcd-x-a",
+ }
+ // Change, but not memory allocation required.
+ benchSimpleChange = []string{
+ "EN",
+ "i-klingon",
+ "en-latn",
+ "zh-cmn-Hans-CN",
+ "iw-NL",
+ }
+ // Change and memory allocation required.
+ benchChangeAlloc = []string{
+ "en-c_cc-b-bbb-a-aaa",
+ "en-u-cu-xua-co-phonebk",
+ "en-u-cu-xua-co-phonebk-a-cd",
+ "en-u-def-abc-cu-xua-co-phonebk",
+ "en-t-en-Cyrl-NL-1994",
+ "en-t-en-Cyrl-NL-1994-t0-abc-def",
+ }
+ // Tags that result in errors.
+ benchErr = []string{
+ // IllFormed
+ "x_A.-B-C_D",
+ "en-u-cu-co-phonebk",
+ "en-u-cu-xau-co",
+ "en-t-nl-abcd",
+ // Invalid
+ "xx",
+ "nl-Uuuu",
+ "nl-QB",
+ }
+ benchChange = append(benchSimpleChange, benchChangeAlloc...)
+ benchAll = append(append(append(benchBasic, benchExt...), benchChange...), benchErr...)
+)
+
+func doParse(b *testing.B, tag []string) {
+ for i := 0; i < b.N; i++ {
+ // Use the modulo instead of looping over all tags so that we get a somewhat
+ // meaningful ns/op.
+ Parse(tag[i%len(tag)])
+ }
+}
+
+func BenchmarkParse(b *testing.B) {
+ doParse(b, benchAll)
+}
+
+func BenchmarkParseBasic(b *testing.B) {
+ doParse(b, benchBasic)
+}
+
+func BenchmarkParseError(b *testing.B) {
+ doParse(b, benchErr)
+}
+
+func BenchmarkParseSimpleChange(b *testing.B) {
+ doParse(b, benchSimpleChange)
+}
+
+func BenchmarkParseChangeAlloc(b *testing.B) {
+ doParse(b, benchChangeAlloc)
+}
diff --git a/language/internal/lookup.go b/language/internal/lookup.go
new file mode 100644
index 0000000..1d80ac3
--- /dev/null
+++ b/language/internal/lookup.go
@@ -0,0 +1,396 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language
+
+import (
+ "bytes"
+ "fmt"
+ "sort"
+ "strconv"
+
+ "golang.org/x/text/internal/tag"
+)
+
+// findIndex tries to find the given tag in idx and returns a standardized error
+// if it could not be found.
+func findIndex(idx tag.Index, key []byte, form string) (index int, err error) {
+ if !tag.FixCase(form, key) {
+ return 0, errSyntax
+ }
+ i := idx.Index(key)
+ if i == -1 {
+ return 0, mkErrInvalid(key)
+ }
+ return i, nil
+}
+
+func searchUint(imap []uint16, key uint16) int {
+ return sort.Search(len(imap), func(i int) bool {
+ return imap[i] >= key
+ })
+}
+
+type langID uint16
+
+// getLangID returns the langID of s if s is a canonical subtag
+// or langUnknown if s is not a canonical subtag.
+func getLangID(s []byte) (langID, error) {
+ if len(s) == 2 {
+ return getLangISO2(s)
+ }
+ return getLangISO3(s)
+}
+
+// mapLang returns the mapped langID of id according to mapping m.
+func normLang(id langID) (langID, langAliasType) {
+ k := sort.Search(len(langAliasMap), func(i int) bool {
+ return langAliasMap[i].from >= uint16(id)
+ })
+ if k < len(langAliasMap) && langAliasMap[k].from == uint16(id) {
+ return langID(langAliasMap[k].to), langAliasTypes[k]
+ }
+ return id, langAliasTypeUnknown
+}
+
+// getLangISO2 returns the langID for the given 2-letter ISO language code
+// or unknownLang if this does not exist.
+func getLangISO2(s []byte) (langID, error) {
+ if !tag.FixCase("zz", s) {
+ return 0, errSyntax
+ }
+ if i := lang.Index(s); i != -1 && lang.Elem(i)[3] != 0 {
+ return langID(i), nil
+ }
+ return 0, mkErrInvalid(s)
+}
+
+const base = 'z' - 'a' + 1
+
+func strToInt(s []byte) uint {
+ v := uint(0)
+ for i := 0; i < len(s); i++ {
+ v *= base
+ v += uint(s[i] - 'a')
+ }
+ return v
+}
+
+// converts the given integer to the original ASCII string passed to strToInt.
+// len(s) must match the number of characters obtained.
+func intToStr(v uint, s []byte) {
+ for i := len(s) - 1; i >= 0; i-- {
+ s[i] = byte(v%base) + 'a'
+ v /= base
+ }
+}
+
+// getLangISO3 returns the langID for the given 3-letter ISO language code
+// or unknownLang if this does not exist.
+func getLangISO3(s []byte) (langID, error) {
+ if tag.FixCase("und", s) {
+ // first try to match canonical 3-letter entries
+ for i := lang.Index(s[:2]); i != -1; i = lang.Next(s[:2], i) {
+ if e := lang.Elem(i); e[3] == 0 && e[2] == s[2] {
+ // We treat "und" as special and always translate it to "unspecified".
+ // Note that ZZ and Zzzz are private use and are not treated as
+ // unspecified by default.
+ id := langID(i)
+ if id == nonCanonicalUnd {
+ return 0, nil
+ }
+ return id, nil
+ }
+ }
+ if i := altLangISO3.Index(s); i != -1 {
+ return langID(altLangIndex[altLangISO3.Elem(i)[3]]), nil
+ }
+ n := strToInt(s)
+ if langNoIndex[n/8]&(1<<(n%8)) != 0 {
+ return langID(n) + langNoIndexOffset, nil
+ }
+ // Check for non-canonical uses of ISO3.
+ for i := lang.Index(s[:1]); i != -1; i = lang.Next(s[:1], i) {
+ if e := lang.Elem(i); e[2] == s[1] && e[3] == s[2] {
+ return langID(i), nil
+ }
+ }
+ return 0, mkErrInvalid(s)
+ }
+ return 0, errSyntax
+}
+
+// stringToBuf writes the string to b and returns the number of bytes
+// written. cap(b) must be >= 3.
+func (id langID) stringToBuf(b []byte) int {
+ if id >= langNoIndexOffset {
+ intToStr(uint(id)-langNoIndexOffset, b[:3])
+ return 3
+ } else if id == 0 {
+ return copy(b, "und")
+ }
+ l := lang[id<<2:]
+ if l[3] == 0 {
+ return copy(b, l[:3])
+ }
+ return copy(b, l[:2])
+}
+
+// String returns the BCP 47 representation of the langID.
+// Use b as variable name, instead of id, to ensure the variable
+// used is consistent with that of Base in which this type is embedded.
+func (b langID) String() string {
+ if b == 0 {
+ return "und"
+ } else if b >= langNoIndexOffset {
+ b -= langNoIndexOffset
+ buf := [3]byte{}
+ intToStr(uint(b), buf[:])
+ return string(buf[:])
+ }
+ l := lang.Elem(int(b))
+ if l[3] == 0 {
+ return l[:3]
+ }
+ return l[:2]
+}
+
+// ISO3 returns the ISO 639-3 language code.
+func (b langID) ISO3() string {
+ if b == 0 || b >= langNoIndexOffset {
+ return b.String()
+ }
+ l := lang.Elem(int(b))
+ if l[3] == 0 {
+ return l[:3]
+ } else if l[2] == 0 {
+ return altLangISO3.Elem(int(l[3]))[:3]
+ }
+ // This allocation will only happen for 3-letter ISO codes
+ // that are non-canonical BCP 47 language identifiers.
+ return l[0:1] + l[2:4]
+}
+
+// IsPrivateUse reports whether this language code is reserved for private use.
+func (b langID) IsPrivateUse() bool {
+ return langPrivateStart <= b && b <= langPrivateEnd
+}
+
+type regionID uint16
+
+// getRegionID returns the region id for s if s is a valid 2-letter region code
+// or unknownRegion.
+func getRegionID(s []byte) (regionID, error) {
+ if len(s) == 3 {
+ if isAlpha(s[0]) {
+ return getRegionISO3(s)
+ }
+ if i, err := strconv.ParseUint(string(s), 10, 10); err == nil {
+ return getRegionM49(int(i))
+ }
+ }
+ return getRegionISO2(s)
+}
+
+// getRegionISO2 returns the regionID for the given 2-letter ISO country code
+// or unknownRegion if this does not exist.
+func getRegionISO2(s []byte) (regionID, error) {
+ i, err := findIndex(regionISO, s, "ZZ")
+ if err != nil {
+ return 0, err
+ }
+ return regionID(i) + isoRegionOffset, nil
+}
+
+// getRegionISO3 returns the regionID for the given 3-letter ISO country code
+// or unknownRegion if this does not exist.
+func getRegionISO3(s []byte) (regionID, error) {
+ if tag.FixCase("ZZZ", s) {
+ for i := regionISO.Index(s[:1]); i != -1; i = regionISO.Next(s[:1], i) {
+ if e := regionISO.Elem(i); e[2] == s[1] && e[3] == s[2] {
+ return regionID(i) + isoRegionOffset, nil
+ }
+ }
+ for i := 0; i < len(altRegionISO3); i += 3 {
+ if tag.Compare(altRegionISO3[i:i+3], s) == 0 {
+ return regionID(altRegionIDs[i/3]), nil
+ }
+ }
+ return 0, mkErrInvalid(s)
+ }
+ return 0, errSyntax
+}
+
+func getRegionM49(n int) (regionID, error) {
+ if 0 < n && n <= 999 {
+ const (
+ searchBits = 7
+ regionBits = 9
+ regionMask = 1<<regionBits - 1
+ )
+ idx := n >> searchBits
+ buf := fromM49[m49Index[idx]:m49Index[idx+1]]
+ val := uint16(n) << regionBits // we rely on bits shifting out
+ i := sort.Search(len(buf), func(i int) bool {
+ return buf[i] >= val
+ })
+ if r := fromM49[int(m49Index[idx])+i]; r&^regionMask == val {
+ return regionID(r & regionMask), nil
+ }
+ }
+ var e ValueError
+ fmt.Fprint(bytes.NewBuffer([]byte(e.v[:])), n)
+ return 0, e
+}
+
+// normRegion returns a region if r is deprecated or 0 otherwise.
+// TODO: consider supporting BYS (-> BLR), CSK (-> 200 or CZ), PHI (-> PHL) and AFI (-> DJ).
+// TODO: consider mapping split up regions to new most populous one (like CLDR).
+func normRegion(r regionID) regionID {
+ m := regionOldMap
+ k := sort.Search(len(m), func(i int) bool {
+ return m[i].from >= uint16(r)
+ })
+ if k < len(m) && m[k].from == uint16(r) {
+ return regionID(m[k].to)
+ }
+ return 0
+}
+
+const (
+ iso3166UserAssigned = 1 << iota
+ ccTLD
+ bcp47Region
+)
+
+func (r regionID) typ() byte {
+ return regionTypes[r]
+}
+
+// String returns the BCP 47 representation for the region.
+// It returns "ZZ" for an unspecified region.
+func (r regionID) String() string {
+ if r < isoRegionOffset {
+ if r == 0 {
+ return "ZZ"
+ }
+ return fmt.Sprintf("%03d", r.M49())
+ }
+ r -= isoRegionOffset
+ return regionISO.Elem(int(r))[:2]
+}
+
+// ISO3 returns the 3-letter ISO code of r.
+// Note that not all regions have a 3-letter ISO code.
+// In such cases this method returns "ZZZ".
+func (r regionID) ISO3() string {
+ if r < isoRegionOffset {
+ return "ZZZ"
+ }
+ r -= isoRegionOffset
+ reg := regionISO.Elem(int(r))
+ switch reg[2] {
+ case 0:
+ return altRegionISO3[reg[3]:][:3]
+ case ' ':
+ return "ZZZ"
+ }
+ return reg[0:1] + reg[2:4]
+}
+
+// M49 returns the UN M.49 encoding of r, or 0 if this encoding
+// is not defined for r.
+func (r regionID) M49() int {
+ return int(m49[r])
+}
+
+// IsPrivateUse reports whether r has the ISO 3166 User-assigned status. This
+// may include private-use tags that are assigned by CLDR and used in this
+// implementation. So IsPrivateUse and IsCountry can be simultaneously true.
+func (r regionID) IsPrivateUse() bool {
+ return r.typ()&iso3166UserAssigned != 0
+}
+
+type scriptID uint8
+
+// getScriptID returns the script id for string s. It assumes that s
+// is of the format [A-Z][a-z]{3}.
+func getScriptID(idx tag.Index, s []byte) (scriptID, error) {
+ i, err := findIndex(idx, s, "Zzzz")
+ return scriptID(i), err
+}
+
+// String returns the script code in title case.
+// It returns "Zzzz" for an unspecified script.
+func (s scriptID) String() string {
+ if s == 0 {
+ return "Zzzz"
+ }
+ return script.Elem(int(s))
+}
+
+// IsPrivateUse reports whether this script code is reserved for private use.
+func (s scriptID) IsPrivateUse() bool {
+ return _Qaaa <= s && s <= _Qabx
+}
+
+const (
+ maxAltTaglen = len("en-US-POSIX")
+ maxLen = maxAltTaglen
+)
+
+var (
+ // grandfatheredMap holds a mapping from legacy and grandfathered tags to
+ // their base language or index to more elaborate tag.
+ grandfatheredMap = map[[maxLen]byte]int16{
+ [maxLen]byte{'a', 'r', 't', '-', 'l', 'o', 'j', 'b', 'a', 'n'}: _jbo, // art-lojban
+ [maxLen]byte{'i', '-', 'a', 'm', 'i'}: _ami, // i-ami
+ [maxLen]byte{'i', '-', 'b', 'n', 'n'}: _bnn, // i-bnn
+ [maxLen]byte{'i', '-', 'h', 'a', 'k'}: _hak, // i-hak
+ [maxLen]byte{'i', '-', 'k', 'l', 'i', 'n', 'g', 'o', 'n'}: _tlh, // i-klingon
+ [maxLen]byte{'i', '-', 'l', 'u', 'x'}: _lb, // i-lux
+ [maxLen]byte{'i', '-', 'n', 'a', 'v', 'a', 'j', 'o'}: _nv, // i-navajo
+ [maxLen]byte{'i', '-', 'p', 'w', 'n'}: _pwn, // i-pwn
+ [maxLen]byte{'i', '-', 't', 'a', 'o'}: _tao, // i-tao
+ [maxLen]byte{'i', '-', 't', 'a', 'y'}: _tay, // i-tay
+ [maxLen]byte{'i', '-', 't', 's', 'u'}: _tsu, // i-tsu
+ [maxLen]byte{'n', 'o', '-', 'b', 'o', 'k'}: _nb, // no-bok
+ [maxLen]byte{'n', 'o', '-', 'n', 'y', 'n'}: _nn, // no-nyn
+ [maxLen]byte{'s', 'g', 'n', '-', 'b', 'e', '-', 'f', 'r'}: _sfb, // sgn-BE-FR
+ [maxLen]byte{'s', 'g', 'n', '-', 'b', 'e', '-', 'n', 'l'}: _vgt, // sgn-BE-NL
+ [maxLen]byte{'s', 'g', 'n', '-', 'c', 'h', '-', 'd', 'e'}: _sgg, // sgn-CH-DE
+ [maxLen]byte{'z', 'h', '-', 'g', 'u', 'o', 'y', 'u'}: _cmn, // zh-guoyu
+ [maxLen]byte{'z', 'h', '-', 'h', 'a', 'k', 'k', 'a'}: _hak, // zh-hakka
+ [maxLen]byte{'z', 'h', '-', 'm', 'i', 'n', '-', 'n', 'a', 'n'}: _nan, // zh-min-nan
+ [maxLen]byte{'z', 'h', '-', 'x', 'i', 'a', 'n', 'g'}: _hsn, // zh-xiang
+
+ // Grandfathered tags with no modern replacement will be converted as
+ // follows:
+ [maxLen]byte{'c', 'e', 'l', '-', 'g', 'a', 'u', 'l', 'i', 's', 'h'}: -1, // cel-gaulish
+ [maxLen]byte{'e', 'n', '-', 'g', 'b', '-', 'o', 'e', 'd'}: -2, // en-GB-oed
+ [maxLen]byte{'i', '-', 'd', 'e', 'f', 'a', 'u', 'l', 't'}: -3, // i-default
+ [maxLen]byte{'i', '-', 'e', 'n', 'o', 'c', 'h', 'i', 'a', 'n'}: -4, // i-enochian
+ [maxLen]byte{'i', '-', 'm', 'i', 'n', 'g', 'o'}: -5, // i-mingo
+ [maxLen]byte{'z', 'h', '-', 'm', 'i', 'n'}: -6, // zh-min
+
+ // CLDR-specific tag.
+ [maxLen]byte{'r', 'o', 'o', 't'}: 0, // root
+ [maxLen]byte{'e', 'n', '-', 'u', 's', '-', 'p', 'o', 's', 'i', 'x'}: -7, // en_US_POSIX"
+ }
+
+ altTagIndex = [...]uint8{0, 17, 31, 45, 61, 74, 86, 102}
+
+ altTags = "xtg-x-cel-gaulishen-GB-oxendicten-x-i-defaultund-x-i-enochiansee-x-i-mingonan-x-zh-minen-US-u-va-posix"
+)
+
+func grandfathered(s [maxAltTaglen]byte) (t Tag, ok bool) {
+ if v, ok := grandfatheredMap[s]; ok {
+ if v < 0 {
+ return Make(altTags[altTagIndex[-v-1]:altTagIndex[-v]]), true
+ }
+ t.lang = langID(v)
+ return t, true
+ }
+ return t, false
+}
diff --git a/language/internal/lookup_test.go b/language/internal/lookup_test.go
new file mode 100644
index 0000000..9833830
--- /dev/null
+++ b/language/internal/lookup_test.go
@@ -0,0 +1,457 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language
+
+import (
+ "testing"
+
+ "golang.org/x/text/internal/tag"
+)
+
+func b(s string) []byte {
+ return []byte(s)
+}
+
+func TestLangID(t *testing.T) {
+ tests := []struct {
+ id, bcp47, iso3, norm string
+ err error
+ }{
+ {id: "", bcp47: "und", iso3: "und", err: errSyntax},
+ {id: " ", bcp47: "und", iso3: "und", err: errSyntax},
+ {id: " ", bcp47: "und", iso3: "und", err: errSyntax},
+ {id: " ", bcp47: "und", iso3: "und", err: errSyntax},
+ {id: "xxx", bcp47: "und", iso3: "und", err: mkErrInvalid([]byte("xxx"))},
+ {id: "und", bcp47: "und", iso3: "und"},
+ {id: "aju", bcp47: "aju", iso3: "aju", norm: "jrb"},
+ {id: "jrb", bcp47: "jrb", iso3: "jrb"},
+ {id: "es", bcp47: "es", iso3: "spa"},
+ {id: "spa", bcp47: "es", iso3: "spa"},
+ {id: "ji", bcp47: "ji", iso3: "yid-", norm: "yi"},
+ {id: "jw", bcp47: "jw", iso3: "jav-", norm: "jv"},
+ {id: "ar", bcp47: "ar", iso3: "ara"},
+ {id: "kw", bcp47: "kw", iso3: "cor"},
+ {id: "arb", bcp47: "arb", iso3: "arb", norm: "ar"},
+ {id: "ar", bcp47: "ar", iso3: "ara"},
+ {id: "kur", bcp47: "ku", iso3: "kur"},
+ {id: "nl", bcp47: "nl", iso3: "nld"},
+ {id: "NL", bcp47: "nl", iso3: "nld"},
+ {id: "gsw", bcp47: "gsw", iso3: "gsw"},
+ {id: "gSW", bcp47: "gsw", iso3: "gsw"},
+ {id: "und", bcp47: "und", iso3: "und"},
+ {id: "sh", bcp47: "sh", iso3: "hbs", norm: "sr"},
+ {id: "hbs", bcp47: "sh", iso3: "hbs", norm: "sr"},
+ {id: "no", bcp47: "no", iso3: "nor", norm: "no"},
+ {id: "nor", bcp47: "no", iso3: "nor", norm: "no"},
+ {id: "cmn", bcp47: "cmn", iso3: "cmn", norm: "zh"},
+ }
+ for i, tt := range tests {
+ want, err := getLangID(b(tt.id))
+ if err != tt.err {
+ t.Errorf("%d:err(%s): found %q; want %q", i, tt.id, err, tt.err)
+ }
+ if err != nil {
+ continue
+ }
+ if id, _ := getLangISO2(b(tt.bcp47)); len(tt.bcp47) == 2 && want != id {
+ t.Errorf("%d:getISO2(%s): found %v; want %v", i, tt.bcp47, id, want)
+ }
+ if len(tt.iso3) == 3 {
+ if id, _ := getLangISO3(b(tt.iso3)); want != id {
+ t.Errorf("%d:getISO3(%s): found %q; want %q", i, tt.iso3, id, want)
+ }
+ if id, _ := getLangID(b(tt.iso3)); want != id {
+ t.Errorf("%d:getID3(%s): found %v; want %v", i, tt.iso3, id, want)
+ }
+ }
+ norm := want
+ if tt.norm != "" {
+ norm, _ = getLangID(b(tt.norm))
+ }
+ id, _ := normLang(want)
+ if id != norm {
+ t.Errorf("%d:norm(%s): found %v; want %v", i, tt.id, id, norm)
+ }
+ if id := want.String(); tt.bcp47 != id {
+ t.Errorf("%d:String(): found %s; want %s", i, id, tt.bcp47)
+ }
+ if id := want.ISO3(); tt.iso3[:3] != id {
+ t.Errorf("%d:iso3(): found %s; want %s", i, id, tt.iso3[:3])
+ }
+ }
+}
+
+func TestGrandfathered(t *testing.T) {
+ for _, tt := range []struct{ in, out string }{
+ {"art-lojban", "jbo"},
+ {"i-ami", "ami"},
+ {"i-bnn", "bnn"},
+ {"i-hak", "hak"},
+ {"i-klingon", "tlh"},
+ {"i-lux", "lb"},
+ {"i-navajo", "nv"},
+ {"i-pwn", "pwn"},
+ {"i-tao", "tao"},
+ {"i-tay", "tay"},
+ {"i-tsu", "tsu"},
+ {"no-bok", "nb"},
+ {"no-nyn", "nn"},
+ {"sgn-BE-FR", "sfb"},
+ {"sgn-BE-NL", "vgt"},
+ {"sgn-CH-DE", "sgg"},
+ {"sgn-ch-de", "sgg"},
+ {"zh-guoyu", "cmn"},
+ {"zh-hakka", "hak"},
+ {"zh-min-nan", "nan"},
+ {"zh-xiang", "hsn"},
+
+ // Grandfathered tags with no modern replacement will be converted as follows:
+ {"cel-gaulish", "xtg-x-cel-gaulish"},
+ {"en-GB-oed", "en-GB-oxendict"},
+ {"en-gb-oed", "en-GB-oxendict"},
+ {"i-default", "en-x-i-default"},
+ {"i-enochian", "und-x-i-enochian"},
+ {"i-mingo", "see-x-i-mingo"},
+ {"zh-min", "nan-x-zh-min"},
+
+ {"root", "und"},
+ {"en_US_POSIX", "en-US-u-va-posix"},
+ {"en_us_posix", "en-US-u-va-posix"},
+ {"en-us-posix", "en-US-u-va-posix"},
+ } {
+ got := Raw.Make(tt.in)
+ want := Raw.MustParse(tt.out)
+ if got != want {
+ t.Errorf("%s: got %q; want %q", tt.in, got, want)
+ }
+ }
+}
+
+func TestRegionID(t *testing.T) {
+ tests := []struct {
+ in, out string
+ }{
+ {"_ ", ""},
+ {"_000", ""},
+ {"419", "419"},
+ {"AA", "AA"},
+ {"ATF", "TF"},
+ {"HV", "HV"},
+ {"CT", "CT"},
+ {"DY", "DY"},
+ {"IC", "IC"},
+ {"FQ", "FQ"},
+ {"JT", "JT"},
+ {"ZZ", "ZZ"},
+ {"EU", "EU"},
+ {"QO", "QO"},
+ {"FX", "FX"},
+ }
+ for i, tt := range tests {
+ if tt.in[0] == '_' {
+ id := tt.in[1:]
+ if _, err := getRegionID(b(id)); err == nil {
+ t.Errorf("%d:err(%s): found nil; want error", i, id)
+ }
+ continue
+ }
+ want, _ := getRegionID(b(tt.in))
+ if s := want.String(); s != tt.out {
+ t.Errorf("%d:%s: found %q; want %q", i, tt.in, s, tt.out)
+ }
+ if len(tt.in) == 2 {
+ want, _ := getRegionISO2(b(tt.in))
+ if s := want.String(); s != tt.out {
+ t.Errorf("%d:getISO2(%s): found %q; want %q", i, tt.in, s, tt.out)
+ }
+ }
+ }
+}
+
+func TestRegionType(t *testing.T) {
+ for _, tt := range []struct {
+ r string
+ t byte
+ }{
+ {"NL", bcp47Region | ccTLD},
+ {"EU", bcp47Region | ccTLD}, // exceptionally reserved
+ {"AN", bcp47Region | ccTLD}, // transitionally reserved
+
+ {"DD", bcp47Region}, // deleted in ISO, deprecated in BCP 47
+ {"NT", bcp47Region}, // transitionally reserved, deprecated in BCP 47
+
+ {"XA", iso3166UserAssigned | bcp47Region},
+ {"ZZ", iso3166UserAssigned | bcp47Region},
+ {"AA", iso3166UserAssigned | bcp47Region},
+ {"QO", iso3166UserAssigned | bcp47Region},
+ {"QM", iso3166UserAssigned | bcp47Region},
+ {"XK", iso3166UserAssigned | bcp47Region},
+
+ {"CT", 0}, // deleted in ISO, not in BCP 47, canonicalized in CLDR
+ } {
+ r := MustParseRegion(tt.r)
+ if tp := r.typ(); tp != tt.t {
+ t.Errorf("Type(%s): got %x; want %x", tt.r, tp, tt.t)
+ }
+ }
+}
+
+func TestRegionISO3(t *testing.T) {
+ tests := []struct {
+ from, iso3, to string
+ }{
+ {" ", "ZZZ", "ZZ"},
+ {"000", "ZZZ", "ZZ"},
+ {"AA", "AAA", ""},
+ {"CT", "CTE", ""},
+ {"DY", "DHY", ""},
+ {"EU", "QUU", ""},
+ {"HV", "HVO", ""},
+ {"IC", "ZZZ", "ZZ"},
+ {"JT", "JTN", ""},
+ {"PZ", "PCZ", ""},
+ {"QU", "QUU", "EU"},
+ {"QO", "QOO", ""},
+ {"YD", "YMD", ""},
+ {"FQ", "ATF", "TF"},
+ {"TF", "ATF", ""},
+ {"FX", "FXX", ""},
+ {"ZZ", "ZZZ", ""},
+ {"419", "ZZZ", "ZZ"},
+ }
+ for _, tt := range tests {
+ r, _ := getRegionID(b(tt.from))
+ if s := r.ISO3(); s != tt.iso3 {
+ t.Errorf("iso3(%q): found %q; want %q", tt.from, s, tt.iso3)
+ }
+ if tt.iso3 == "" {
+ continue
+ }
+ want := tt.to
+ if tt.to == "" {
+ want = tt.from
+ }
+ r, _ = getRegionID(b(want))
+ if id, _ := getRegionISO3(b(tt.iso3)); id != r {
+ t.Errorf("%s: found %q; want %q", tt.iso3, id, want)
+ }
+ }
+}
+
+func TestRegionM49(t *testing.T) {
+ fromTests := []struct {
+ m49 int
+ id string
+ }{
+ {0, ""},
+ {-1, ""},
+ {1000, ""},
+ {10000, ""},
+
+ {001, "001"},
+ {104, "MM"},
+ {180, "CD"},
+ {230, "ET"},
+ {231, "ET"},
+ {249, "FX"},
+ {250, "FR"},
+ {276, "DE"},
+ {278, "DD"},
+ {280, "DE"},
+ {419, "419"},
+ {626, "TL"},
+ {736, "SD"},
+ {840, "US"},
+ {854, "BF"},
+ {891, "CS"},
+ {899, ""},
+ {958, "AA"},
+ {966, "QT"},
+ {967, "EU"},
+ {999, "ZZ"},
+ }
+ for _, tt := range fromTests {
+ id, err := getRegionM49(tt.m49)
+ if want, have := err != nil, tt.id == ""; want != have {
+ t.Errorf("error(%d): have %v; want %v", tt.m49, have, want)
+ continue
+ }
+ r, _ := getRegionID(b(tt.id))
+ if r != id {
+ t.Errorf("region(%d): have %s; want %s", tt.m49, id, r)
+ }
+ }
+
+ toTests := []struct {
+ m49 int
+ id string
+ }{
+ {0, "000"},
+ {0, "IC"}, // Some codes don't have an ID
+
+ {001, "001"},
+ {104, "MM"},
+ {104, "BU"},
+ {180, "CD"},
+ {180, "ZR"},
+ {231, "ET"},
+ {250, "FR"},
+ {249, "FX"},
+ {276, "DE"},
+ {278, "DD"},
+ {419, "419"},
+ {626, "TL"},
+ {626, "TP"},
+ {729, "SD"},
+ {826, "GB"},
+ {840, "US"},
+ {854, "BF"},
+ {891, "YU"},
+ {891, "CS"},
+ {958, "AA"},
+ {966, "QT"},
+ {967, "EU"},
+ {967, "QU"},
+ {999, "ZZ"},
+ // For codes that don't have an M49 code use the replacement value,
+ // if available.
+ {854, "HV"}, // maps to Burkino Faso
+ }
+ for _, tt := range toTests {
+ r, _ := getRegionID(b(tt.id))
+ if r.M49() != tt.m49 {
+ t.Errorf("m49(%q): have %d; want %d", tt.id, r.M49(), tt.m49)
+ }
+ }
+}
+
+func TestRegionDeprecation(t *testing.T) {
+ tests := []struct{ in, out string }{
+ {"BU", "MM"},
+ {"BUR", "MM"},
+ {"CT", "KI"},
+ {"DD", "DE"},
+ {"DDR", "DE"},
+ {"DY", "BJ"},
+ {"FX", "FR"},
+ {"HV", "BF"},
+ {"JT", "UM"},
+ {"MI", "UM"},
+ {"NH", "VU"},
+ {"NQ", "AQ"},
+ {"PU", "UM"},
+ {"PZ", "PA"},
+ {"QU", "EU"},
+ {"RH", "ZW"},
+ {"TP", "TL"},
+ {"UK", "GB"},
+ {"VD", "VN"},
+ {"WK", "UM"},
+ {"YD", "YE"},
+ {"NL", "NL"},
+ }
+ for _, tt := range tests {
+ rIn, _ := getRegionID([]byte(tt.in))
+ rOut, _ := getRegionISO2([]byte(tt.out))
+ r := normRegion(rIn)
+ if rOut == rIn && r != 0 {
+ t.Errorf("%s: was %q; want %q", tt.in, r, tt.in)
+ }
+ if rOut != rIn && r != rOut {
+ t.Errorf("%s: was %q; want %q", tt.in, r, tt.out)
+ }
+
+ }
+}
+
+func TestGetScriptID(t *testing.T) {
+ idx := tag.Index("0000BbbbDdddEeeeZzzz\xff\xff\xff\xff")
+ tests := []struct {
+ in string
+ out scriptID
+ }{
+ {" ", 0},
+ {" ", 0},
+ {" ", 0},
+ {"", 0},
+ {"Aaaa", 0},
+ {"Bbbb", 1},
+ {"Dddd", 2},
+ {"dddd", 2},
+ {"dDDD", 2},
+ {"Eeee", 3},
+ {"Zzzz", 4},
+ }
+ for i, tt := range tests {
+ if id, err := getScriptID(idx, b(tt.in)); id != tt.out {
+ t.Errorf("%d:%s: found %d; want %d", i, tt.in, id, tt.out)
+ } else if id == 0 && err == nil {
+ t.Errorf("%d:%s: no error; expected one", i, tt.in)
+ }
+ }
+}
+
+func TestIsPrivateUse(t *testing.T) {
+ type test struct {
+ s string
+ private bool
+ }
+ tests := []test{
+ {"en", false},
+ {"und", false},
+ {"pzn", false},
+ {"qaa", true},
+ {"qtz", true},
+ {"qua", false},
+ }
+ for i, tt := range tests {
+ x, _ := getLangID([]byte(tt.s))
+ if b := x.IsPrivateUse(); b != tt.private {
+ t.Errorf("%d: langID.IsPrivateUse(%s) was %v; want %v", i, tt.s, b, tt.private)
+ }
+ }
+ tests = []test{
+ {"001", false},
+ {"419", false},
+ {"899", false},
+ {"900", false},
+ {"957", false},
+ {"958", true},
+ {"AA", true},
+ {"AC", false},
+ {"EU", false}, // CLDR grouping, exceptionally reserved in ISO.
+ {"QU", true}, // Canonicalizes to EU, User-assigned in ISO.
+ {"QO", true}, // CLDR grouping, User-assigned in ISO.
+ {"QA", false},
+ {"QM", true},
+ {"QZ", true},
+ {"XA", true},
+ {"XK", true}, // Assigned to Kosovo in CLDR, User-assigned in ISO.
+ {"XZ", true},
+ {"ZW", false},
+ {"ZZ", true},
+ }
+ for i, tt := range tests {
+ x, _ := getRegionID([]byte(tt.s))
+ if b := x.IsPrivateUse(); b != tt.private {
+ t.Errorf("%d: regionID.IsPrivateUse(%s) was %v; want %v", i, tt.s, b, tt.private)
+ }
+ }
+ tests = []test{
+ {"Latn", false},
+ {"Laaa", false}, // invalid
+ {"Qaaa", true},
+ {"Qabx", true},
+ {"Qaby", false},
+ {"Zyyy", false},
+ {"Zzzz", false},
+ }
+ for i, tt := range tests {
+ x, _ := getScriptID(script, []byte(tt.s))
+ if b := x.IsPrivateUse(); b != tt.private {
+ t.Errorf("%d: scriptID.IsPrivateUse(%s) was %v; want %v", i, tt.s, b, tt.private)
+ }
+ }
+}
diff --git a/language/internal/match.go b/language/internal/match.go
new file mode 100644
index 0000000..15b74d1
--- /dev/null
+++ b/language/internal/match.go
@@ -0,0 +1,933 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language
+
+import "errors"
+
+// A MatchOption configures a Matcher.
+type MatchOption func(*matcher)
+
+// PreferSameScript will, in the absence of a match, result in the first
+// preferred tag with the same script as a supported tag to match this supported
+// tag. The default is currently true, but this may change in the future.
+func PreferSameScript(preferSame bool) MatchOption {
+ return func(m *matcher) { m.preferSameScript = preferSame }
+}
+
+// TODO(v1.0.0): consider making Matcher a concrete type, instead of interface.
+// There doesn't seem to be too much need for multiple types.
+// Making it a concrete type allows MatchStrings to be a method, which will
+// improve its discoverability.
+
+// MatchStrings parses and matches the given strings until one of them matches
+// the language in the Matcher. A string may be an Accept-Language header as
+// handled by ParseAcceptLanguage. The default language is returned if no
+// other language matched.
+func MatchStrings(m Matcher, lang ...string) (tag Tag, index int) {
+ for _, accept := range lang {
+ desired, _, err := ParseAcceptLanguage(accept)
+ if err != nil {
+ continue
+ }
+ if tag, index, conf := m.Match(desired...); conf != No {
+ return tag, index
+ }
+ }
+ tag, index, _ = m.Match()
+ return
+}
+
+// Matcher is the interface that wraps the Match method.
+//
+// Match returns the best match for any of the given tags, along with
+// a unique index associated with the returned tag and a confidence
+// score.
+type Matcher interface {
+ Match(t ...Tag) (tag Tag, index int, c Confidence)
+}
+
+// Comprehends reports the confidence score for a speaker of a given language
+// to being able to comprehend the written form of an alternative language.
+func Comprehends(speaker, alternative Tag) Confidence {
+ _, _, c := NewMatcher([]Tag{alternative}).Match(speaker)
+ return c
+}
+
+// NewMatcher returns a Matcher that matches an ordered list of preferred tags
+// against a list of supported tags based on written intelligibility, closeness
+// of dialect, equivalence of subtags and various other rules. It is initialized
+// with the list of supported tags. The first element is used as the default
+// value in case no match is found.
+//
+// Its Match method matches the first of the given Tags to reach a certain
+// confidence threshold. The tags passed to Match should therefore be specified
+// in order of preference. Extensions are ignored for matching.
+//
+// The index returned by the Match method corresponds to the index of the
+// matched tag in t, but is augmented with the Unicode extension ('u')of the
+// corresponding preferred tag. This allows user locale options to be passed
+// transparently.
+func NewMatcher(t []Tag, options ...MatchOption) Matcher {
+ return newMatcher(t, options)
+}
+
+func (m *matcher) Match(want ...Tag) (t Tag, index int, c Confidence) {
+ match, w, c := m.getBest(want...)
+ if match != nil {
+ t, index = match.tag, match.index
+ } else {
+ // TODO: this should be an option
+ t = m.default_.tag
+ if m.preferSameScript {
+ outer:
+ for _, w := range want {
+ script, _ := w.Script()
+ if script.scriptID == 0 {
+ // Don't do anything if there is no script, such as with
+ // private subtags.
+ continue
+ }
+ for i, h := range m.supported {
+ if script.scriptID == h.maxScript {
+ t, index = h.tag, i
+ break outer
+ }
+ }
+ }
+ }
+ // TODO: select first language tag based on script.
+ }
+ if w.region != 0 && t.region != 0 && t.region.contains(w.region) {
+ t, _ = Raw.Compose(t, Region{w.region})
+ }
+ // Copy options from the user-provided tag into the result tag. This is hard
+ // to do after the fact, so we do it here.
+ // TODO: add in alternative variants to -u-va-.
+ // TODO: add preferred region to -u-rg-.
+ if e := w.Extensions(); len(e) > 0 {
+ t, _ = Raw.Compose(t, e)
+ }
+ return t, index, c
+}
+
+type scriptRegionFlags uint8
+
+const (
+ isList = 1 << iota
+ scriptInFrom
+ regionInFrom
+)
+
+func (t *Tag) setUndefinedLang(id langID) {
+ if t.lang == 0 {
+ t.lang = id
+ }
+}
+
+func (t *Tag) setUndefinedScript(id scriptID) {
+ if t.script == 0 {
+ t.script = id
+ }
+}
+
+func (t *Tag) setUndefinedRegion(id regionID) {
+ if t.region == 0 || t.region.contains(id) {
+ t.region = id
+ }
+}
+
+// ErrMissingLikelyTagsData indicates no information was available
+// to compute likely values of missing tags.
+var ErrMissingLikelyTagsData = errors.New("missing likely tags data")
+
+// addLikelySubtags sets subtags to their most likely value, given the locale.
+// In most cases this means setting fields for unknown values, but in some
+// cases it may alter a value. It returns an ErrMissingLikelyTagsData error
+// if the given locale cannot be expanded.
+func (t Tag) addLikelySubtags() (Tag, error) {
+ id, err := addTags(t)
+ if err != nil {
+ return t, err
+ } else if id.equalTags(t) {
+ return t, nil
+ }
+ id.remakeString()
+ return id, nil
+}
+
+// specializeRegion attempts to specialize a group region.
+func specializeRegion(t *Tag) bool {
+ if i := regionInclusion[t.region]; i < nRegionGroups {
+ x := likelyRegionGroup[i]
+ if langID(x.lang) == t.lang && scriptID(x.script) == t.script {
+ t.region = regionID(x.region)
+ }
+ return true
+ }
+ return false
+}
+
+func addTags(t Tag) (Tag, error) {
+ // We leave private use identifiers alone.
+ if t.private() {
+ return t, nil
+ }
+ if t.script != 0 && t.region != 0 {
+ if t.lang != 0 {
+ // already fully specified
+ specializeRegion(&t)
+ return t, nil
+ }
+ // Search matches for und-script-region. Note that for these cases
+ // region will never be a group so there is no need to check for this.
+ list := likelyRegion[t.region : t.region+1]
+ if x := list[0]; x.flags&isList != 0 {
+ list = likelyRegionList[x.lang : x.lang+uint16(x.script)]
+ }
+ for _, x := range list {
+ // Deviating from the spec. See match_test.go for details.
+ if scriptID(x.script) == t.script {
+ t.setUndefinedLang(langID(x.lang))
+ return t, nil
+ }
+ }
+ }
+ if t.lang != 0 {
+ // Search matches for lang-script and lang-region, where lang != und.
+ if t.lang < langNoIndexOffset {
+ x := likelyLang[t.lang]
+ if x.flags&isList != 0 {
+ list := likelyLangList[x.region : x.region+uint16(x.script)]
+ if t.script != 0 {
+ for _, x := range list {
+ if scriptID(x.script) == t.script && x.flags&scriptInFrom != 0 {
+ t.setUndefinedRegion(regionID(x.region))
+ return t, nil
+ }
+ }
+ } else if t.region != 0 {
+ count := 0
+ goodScript := true
+ tt := t
+ for _, x := range list {
+ // We visit all entries for which the script was not
+ // defined, including the ones where the region was not
+ // defined. This allows for proper disambiguation within
+ // regions.
+ if x.flags&scriptInFrom == 0 && t.region.contains(regionID(x.region)) {
+ tt.region = regionID(x.region)
+ tt.setUndefinedScript(scriptID(x.script))
+ goodScript = goodScript && tt.script == scriptID(x.script)
+ count++
+ }
+ }
+ if count == 1 {
+ return tt, nil
+ }
+ // Even if we fail to find a unique Region, we might have
+ // an unambiguous script.
+ if goodScript {
+ t.script = tt.script
+ }
+ }
+ }
+ }
+ } else {
+ // Search matches for und-script.
+ if t.script != 0 {
+ x := likelyScript[t.script]
+ if x.region != 0 {
+ t.setUndefinedRegion(regionID(x.region))
+ t.setUndefinedLang(langID(x.lang))
+ return t, nil
+ }
+ }
+ // Search matches for und-region. If und-script-region exists, it would
+ // have been found earlier.
+ if t.region != 0 {
+ if i := regionInclusion[t.region]; i < nRegionGroups {
+ x := likelyRegionGroup[i]
+ if x.region != 0 {
+ t.setUndefinedLang(langID(x.lang))
+ t.setUndefinedScript(scriptID(x.script))
+ t.region = regionID(x.region)
+ }
+ } else {
+ x := likelyRegion[t.region]
+ if x.flags&isList != 0 {
+ x = likelyRegionList[x.lang]
+ }
+ if x.script != 0 && x.flags != scriptInFrom {
+ t.setUndefinedLang(langID(x.lang))
+ t.setUndefinedScript(scriptID(x.script))
+ return t, nil
+ }
+ }
+ }
+ }
+
+ // Search matches for lang.
+ if t.lang < langNoIndexOffset {
+ x := likelyLang[t.lang]
+ if x.flags&isList != 0 {
+ x = likelyLangList[x.region]
+ }
+ if x.region != 0 {
+ t.setUndefinedScript(scriptID(x.script))
+ t.setUndefinedRegion(regionID(x.region))
+ }
+ specializeRegion(&t)
+ if t.lang == 0 {
+ t.lang = _en // default language
+ }
+ return t, nil
+ }
+ return t, ErrMissingLikelyTagsData
+}
+
+func (t *Tag) setTagsFrom(id Tag) {
+ t.lang = id.lang
+ t.script = id.script
+ t.region = id.region
+}
+
+// minimize removes the region or script subtags from t such that
+// t.addLikelySubtags() == t.minimize().addLikelySubtags().
+func (t Tag) minimize() (Tag, error) {
+ t, err := minimizeTags(t)
+ if err != nil {
+ return t, err
+ }
+ t.remakeString()
+ return t, nil
+}
+
+// minimizeTags mimics the behavior of the ICU 51 C implementation.
+func minimizeTags(t Tag) (Tag, error) {
+ if t.equalTags(und) {
+ return t, nil
+ }
+ max, err := addTags(t)
+ if err != nil {
+ return t, err
+ }
+ for _, id := range [...]Tag{
+ {lang: t.lang},
+ {lang: t.lang, region: t.region},
+ {lang: t.lang, script: t.script},
+ } {
+ if x, err := addTags(id); err == nil && max.equalTags(x) {
+ t.setTagsFrom(id)
+ break
+ }
+ }
+ return t, nil
+}
+
+// Tag Matching
+// CLDR defines an algorithm for finding the best match between two sets of language
+// tags. The basic algorithm defines how to score a possible match and then find
+// the match with the best score
+// (see http://www.unicode.org/reports/tr35/#LanguageMatching).
+// Using scoring has several disadvantages. The scoring obfuscates the importance of
+// the various factors considered, making the algorithm harder to understand. Using
+// scoring also requires the full score to be computed for each pair of tags.
+//
+// We will use a different algorithm which aims to have the following properties:
+// - clarity on the precedence of the various selection factors, and
+// - improved performance by allowing early termination of a comparison.
+//
+// Matching algorithm (overview)
+// Input:
+// - supported: a set of supported tags
+// - default: the default tag to return in case there is no match
+// - desired: list of desired tags, ordered by preference, starting with
+// the most-preferred.
+//
+// Algorithm:
+// 1) Set the best match to the lowest confidence level
+// 2) For each tag in "desired":
+// a) For each tag in "supported":
+// 1) compute the match between the two tags.
+// 2) if the match is better than the previous best match, replace it
+// with the new match. (see next section)
+// b) if the current best match is Exact and pin is true the result will be
+// frozen to the language found thusfar, although better matches may
+// still be found for the same language.
+// 3) If the best match so far is below a certain threshold, return "default".
+//
+// Ranking:
+// We use two phases to determine whether one pair of tags are a better match
+// than another pair of tags. First, we determine a rough confidence level. If the
+// levels are different, the one with the highest confidence wins.
+// Second, if the rough confidence levels are identical, we use a set of tie-breaker
+// rules.
+//
+// The confidence level of matching a pair of tags is determined by finding the
+// lowest confidence level of any matches of the corresponding subtags (the
+// result is deemed as good as its weakest link).
+// We define the following levels:
+// Exact - An exact match of a subtag, before adding likely subtags.
+// MaxExact - An exact match of a subtag, after adding likely subtags.
+// [See Note 2].
+// High - High level of mutual intelligibility between different subtag
+// variants.
+// Low - Low level of mutual intelligibility between different subtag
+// variants.
+// No - No mutual intelligibility.
+//
+// The following levels can occur for each type of subtag:
+// Base: Exact, MaxExact, High, Low, No
+// Script: Exact, MaxExact [see Note 3], Low, No
+// Region: Exact, MaxExact, High
+// Variant: Exact, High
+// Private: Exact, No
+//
+// Any result with a confidence level of Low or higher is deemed a possible match.
+// Once a desired tag matches any of the supported tags with a level of MaxExact
+// or higher, the next desired tag is not considered (see Step 2.b).
+// Note that CLDR provides languageMatching data that defines close equivalence
+// classes for base languages, scripts and regions.
+//
+// Tie-breaking
+// If we get the same confidence level for two matches, we apply a sequence of
+// tie-breaking rules. The first that succeeds defines the result. The rules are
+// applied in the following order.
+// 1) Original language was defined and was identical.
+// 2) Original region was defined and was identical.
+// 3) Distance between two maximized regions was the smallest.
+// 4) Original script was defined and was identical.
+// 5) Distance from want tag to have tag using the parent relation [see Note 5.]
+// If there is still no winner after these rules are applied, the first match
+// found wins.
+//
+// Notes:
+// [2] In practice, as matching of Exact is done in a separate phase from
+// matching the other levels, we reuse the Exact level to mean MaxExact in
+// the second phase. As a consequence, we only need the levels defined by
+// the Confidence type. The MaxExact confidence level is mapped to High in
+// the public API.
+// [3] We do not differentiate between maximized script values that were derived
+// from suppressScript versus most likely tag data. We determined that in
+// ranking the two, one ranks just after the other. Moreover, the two cannot
+// occur concurrently. As a consequence, they are identical for practical
+// purposes.
+// [4] In case of deprecated, macro-equivalents and legacy mappings, we assign
+// the MaxExact level to allow iw vs he to still be a closer match than
+// en-AU vs en-US, for example.
+// [5] In CLDR a locale inherits fields that are unspecified for this locale
+// from its parent. Therefore, if a locale is a parent of another locale,
+// it is a strong measure for closeness, especially when no other tie
+// breaker rule applies. One could also argue it is inconsistent, for
+// example, when pt-AO matches pt (which CLDR equates with pt-BR), even
+// though its parent is pt-PT according to the inheritance rules.
+//
+// Implementation Details:
+// There are several performance considerations worth pointing out. Most notably,
+// we preprocess as much as possible (within reason) at the time of creation of a
+// matcher. This includes:
+// - creating a per-language map, which includes data for the raw base language
+// and its canonicalized variant (if applicable),
+// - expanding entries for the equivalence classes defined in CLDR's
+// languageMatch data.
+// The per-language map ensures that typically only a very small number of tags
+// need to be considered. The pre-expansion of canonicalized subtags and
+// equivalence classes reduces the amount of map lookups that need to be done at
+// runtime.
+
+// matcher keeps a set of supported language tags, indexed by language.
+type matcher struct {
+ default_ *haveTag
+ supported []*haveTag
+ index map[langID]*matchHeader
+ passSettings bool
+ preferSameScript bool
+}
+
+// matchHeader has the lists of tags for exact matches and matches based on
+// maximized and canonicalized tags for a given language.
+type matchHeader struct {
+ haveTags []*haveTag
+ original bool
+}
+
+// haveTag holds a supported Tag and its maximized script and region. The maximized
+// or canonicalized language is not stored as it is not needed during matching.
+type haveTag struct {
+ tag Tag
+
+ // index of this tag in the original list of supported tags.
+ index int
+
+ // conf is the maximum confidence that can result from matching this haveTag.
+ // When conf < Exact this means it was inserted after applying a CLDR equivalence rule.
+ conf Confidence
+
+ // Maximized region and script.
+ maxRegion regionID
+ maxScript scriptID
+
+ // altScript may be checked as an alternative match to maxScript. If altScript
+ // matches, the confidence level for this match is Low. Theoretically there
+ // could be multiple alternative scripts. This does not occur in practice.
+ altScript scriptID
+
+ // nextMax is the index of the next haveTag with the same maximized tags.
+ nextMax uint16
+}
+
+func makeHaveTag(tag Tag, index int) (haveTag, langID) {
+ max := tag
+ if tag.lang != 0 || tag.region != 0 || tag.script != 0 {
+ max, _ = max.canonicalize(All)
+ max, _ = addTags(max)
+ max.remakeString()
+ }
+ return haveTag{tag, index, Exact, max.region, max.script, altScript(max.lang, max.script), 0}, max.lang
+}
+
+// altScript returns an alternative script that may match the given script with
+// a low confidence. At the moment, the langMatch data allows for at most one
+// script to map to another and we rely on this to keep the code simple.
+func altScript(l langID, s scriptID) scriptID {
+ for _, alt := range matchScript {
+ // TODO: also match cases where language is not the same.
+ if (langID(alt.wantLang) == l || langID(alt.haveLang) == l) &&
+ scriptID(alt.haveScript) == s {
+ return scriptID(alt.wantScript)
+ }
+ }
+ return 0
+}
+
+// addIfNew adds a haveTag to the list of tags only if it is a unique tag.
+// Tags that have the same maximized values are linked by index.
+func (h *matchHeader) addIfNew(n haveTag, exact bool) {
+ h.original = h.original || exact
+ // Don't add new exact matches.
+ for _, v := range h.haveTags {
+ if v.tag.equalsRest(n.tag) {
+ return
+ }
+ }
+ // Allow duplicate maximized tags, but create a linked list to allow quickly
+ // comparing the equivalents and bail out.
+ for i, v := range h.haveTags {
+ if v.maxScript == n.maxScript &&
+ v.maxRegion == n.maxRegion &&
+ v.tag.variantOrPrivateTagStr() == n.tag.variantOrPrivateTagStr() {
+ for h.haveTags[i].nextMax != 0 {
+ i = int(h.haveTags[i].nextMax)
+ }
+ h.haveTags[i].nextMax = uint16(len(h.haveTags))
+ break
+ }
+ }
+ h.haveTags = append(h.haveTags, &n)
+}
+
+// header returns the matchHeader for the given language. It creates one if
+// it doesn't already exist.
+func (m *matcher) header(l langID) *matchHeader {
+ if h := m.index[l]; h != nil {
+ return h
+ }
+ h := &matchHeader{}
+ m.index[l] = h
+ return h
+}
+
+func toConf(d uint8) Confidence {
+ if d <= 10 {
+ return High
+ }
+ if d < 30 {
+ return Low
+ }
+ return No
+}
+
+// newMatcher builds an index for the given supported tags and returns it as
+// a matcher. It also expands the index by considering various equivalence classes
+// for a given tag.
+func newMatcher(supported []Tag, options []MatchOption) *matcher {
+ m := &matcher{
+ index: make(map[langID]*matchHeader),
+ preferSameScript: true,
+ }
+ for _, o := range options {
+ o(m)
+ }
+ if len(supported) == 0 {
+ m.default_ = &haveTag{}
+ return m
+ }
+ // Add supported languages to the index. Add exact matches first to give
+ // them precedence.
+ for i, tag := range supported {
+ pair, _ := makeHaveTag(tag, i)
+ m.header(tag.lang).addIfNew(pair, true)
+ m.supported = append(m.supported, &pair)
+ }
+ m.default_ = m.header(supported[0].lang).haveTags[0]
+ // Keep these in two different loops to support the case that two equivalent
+ // languages are distinguished, such as iw and he.
+ for i, tag := range supported {
+ pair, max := makeHaveTag(tag, i)
+ if max != tag.lang {
+ m.header(max).addIfNew(pair, true)
+ }
+ }
+
+ // update is used to add indexes in the map for equivalent languages.
+ // update will only add entries to original indexes, thus not computing any
+ // transitive relations.
+ update := func(want, have uint16, conf Confidence) {
+ if hh := m.index[langID(have)]; hh != nil {
+ if !hh.original {
+ return
+ }
+ hw := m.header(langID(want))
+ for _, ht := range hh.haveTags {
+ v := *ht
+ if conf < v.conf {
+ v.conf = conf
+ }
+ v.nextMax = 0 // this value needs to be recomputed
+ if v.altScript != 0 {
+ v.altScript = altScript(langID(want), v.maxScript)
+ }
+ hw.addIfNew(v, conf == Exact && hh.original)
+ }
+ }
+ }
+
+ // Add entries for languages with mutual intelligibility as defined by CLDR's
+ // languageMatch data.
+ for _, ml := range matchLang {
+ update(ml.want, ml.have, toConf(ml.distance))
+ if !ml.oneway {
+ update(ml.have, ml.want, toConf(ml.distance))
+ }
+ }
+
+ // Add entries for possible canonicalizations. This is an optimization to
+ // ensure that only one map lookup needs to be done at runtime per desired tag.
+ // First we match deprecated equivalents. If they are perfect equivalents
+ // (their canonicalization simply substitutes a different language code, but
+ // nothing else), the match confidence is Exact, otherwise it is High.
+ for i, lm := range langAliasMap {
+ // If deprecated codes match and there is no fiddling with the script or
+ // or region, we consider it an exact match.
+ conf := Exact
+ if langAliasTypes[i] != langMacro {
+ if !isExactEquivalent(langID(lm.from)) {
+ conf = High
+ }
+ update(lm.to, lm.from, conf)
+ }
+ update(lm.from, lm.to, conf)
+ }
+ return m
+}
+
+// getBest gets the best matching tag in m for any of the given tags, taking into
+// account the order of preference of the given tags.
+func (m *matcher) getBest(want ...Tag) (got *haveTag, orig Tag, c Confidence) {
+ best := bestMatch{}
+ for i, w := range want {
+ var max Tag
+ // Check for exact match first.
+ h := m.index[w.lang]
+ if w.lang != 0 {
+ if h == nil {
+ continue
+ }
+ // Base language is defined.
+ max, _ = w.canonicalize(Legacy | Deprecated | Macro)
+ // A region that is added through canonicalization is stronger than
+ // a maximized region: set it in the original (e.g. mo -> ro-MD).
+ if w.region != max.region {
+ w.region = max.region
+ }
+ // TODO: should we do the same for scripts?
+ // See test case: en, sr, nl ; sh ; sr
+ max, _ = addTags(max)
+ } else {
+ // Base language is not defined.
+ if h != nil {
+ for i := range h.haveTags {
+ have := h.haveTags[i]
+ if have.tag.equalsRest(w) {
+ return have, w, Exact
+ }
+ }
+ }
+ if w.script == 0 && w.region == 0 {
+ // We skip all tags matching und for approximate matching, including
+ // private tags.
+ continue
+ }
+ max, _ = addTags(w)
+ if h = m.index[max.lang]; h == nil {
+ continue
+ }
+ }
+ pin := true
+ for _, t := range want[i+1:] {
+ if w.lang == t.lang {
+ pin = false
+ break
+ }
+ }
+ // Check for match based on maximized tag.
+ for i := range h.haveTags {
+ have := h.haveTags[i]
+ best.update(have, w, max.script, max.region, pin)
+ if best.conf == Exact {
+ for have.nextMax != 0 {
+ have = h.haveTags[have.nextMax]
+ best.update(have, w, max.script, max.region, pin)
+ }
+ return best.have, best.want, best.conf
+ }
+ }
+ }
+ if best.conf <= No {
+ if len(want) != 0 {
+ return nil, want[0], No
+ }
+ return nil, Tag{}, No
+ }
+ return best.have, best.want, best.conf
+}
+
+// bestMatch accumulates the best match so far.
+type bestMatch struct {
+ have *haveTag
+ want Tag
+ conf Confidence
+ pinnedRegion regionID
+ pinLanguage bool
+ sameRegionGroup bool
+ // Cached results from applying tie-breaking rules.
+ origLang bool
+ origReg bool
+ paradigmReg bool
+ regGroupDist uint8
+ origScript bool
+}
+
+// update updates the existing best match if the new pair is considered to be a
+// better match. To determine if the given pair is a better match, it first
+// computes the rough confidence level. If this surpasses the current match, it
+// will replace it and update the tie-breaker rule cache. If there is a tie, it
+// proceeds with applying a series of tie-breaker rules. If there is no
+// conclusive winner after applying the tie-breaker rules, it leaves the current
+// match as the preferred match.
+//
+// If pin is true and have and tag are a strong match, it will henceforth only
+// consider matches for this language. This corresponds to the nothing that most
+// users have a strong preference for the first defined language. A user can
+// still prefer a second language over a dialect of the preferred language by
+// explicitly specifying dialects, e.g. "en, nl, en-GB". In this case pin should
+// be false.
+func (m *bestMatch) update(have *haveTag, tag Tag, maxScript scriptID, maxRegion regionID, pin bool) {
+ // Bail if the maximum attainable confidence is below that of the current best match.
+ c := have.conf
+ if c < m.conf {
+ return
+ }
+ // Don't change the language once we already have found an exact match.
+ if m.pinLanguage && tag.lang != m.want.lang {
+ return
+ }
+ // Pin the region group if we are comparing tags for the same language.
+ if tag.lang == m.want.lang && m.sameRegionGroup {
+ _, sameGroup := regionGroupDist(m.pinnedRegion, have.maxRegion, have.maxScript, m.want.lang)
+ if !sameGroup {
+ return
+ }
+ }
+ if c == Exact && have.maxScript == maxScript {
+ // If there is another language and then another entry of this language,
+ // don't pin anything, otherwise pin the language.
+ m.pinLanguage = pin
+ }
+ if have.tag.equalsRest(tag) {
+ } else if have.maxScript != maxScript {
+ // There is usually very little comprehension between different scripts.
+ // In a few cases there may still be Low comprehension. This possibility
+ // is pre-computed and stored in have.altScript.
+ if Low < m.conf || have.altScript != maxScript {
+ return
+ }
+ c = Low
+ } else if have.maxRegion != maxRegion {
+ if High < c {
+ // There is usually a small difference between languages across regions.
+ c = High
+ }
+ }
+
+ // We store the results of the computations of the tie-breaker rules along
+ // with the best match. There is no need to do the checks once we determine
+ // we have a winner, but we do still need to do the tie-breaker computations.
+ // We use "beaten" to keep track if we still need to do the checks.
+ beaten := false // true if the new pair defeats the current one.
+ if c != m.conf {
+ if c < m.conf {
+ return
+ }
+ beaten = true
+ }
+
+ // Tie-breaker rules:
+ // We prefer if the pre-maximized language was specified and identical.
+ origLang := have.tag.lang == tag.lang && tag.lang != 0
+ if !beaten && m.origLang != origLang {
+ if m.origLang {
+ return
+ }
+ beaten = true
+ }
+
+ // We prefer if the pre-maximized region was specified and identical.
+ origReg := have.tag.region == tag.region && tag.region != 0
+ if !beaten && m.origReg != origReg {
+ if m.origReg {
+ return
+ }
+ beaten = true
+ }
+
+ regGroupDist, sameGroup := regionGroupDist(have.maxRegion, maxRegion, maxScript, tag.lang)
+ if !beaten && m.regGroupDist != regGroupDist {
+ if regGroupDist > m.regGroupDist {
+ return
+ }
+ beaten = true
+ }
+
+ paradigmReg := isParadigmLocale(tag.lang, have.maxRegion)
+ if !beaten && m.paradigmReg != paradigmReg {
+ if !paradigmReg {
+ return
+ }
+ beaten = true
+ }
+
+ // Next we prefer if the pre-maximized script was specified and identical.
+ origScript := have.tag.script == tag.script && tag.script != 0
+ if !beaten && m.origScript != origScript {
+ if m.origScript {
+ return
+ }
+ beaten = true
+ }
+
+ // Update m to the newly found best match.
+ if beaten {
+ m.have = have
+ m.want = tag
+ m.conf = c
+ m.pinnedRegion = maxRegion
+ m.sameRegionGroup = sameGroup
+ m.origLang = origLang
+ m.origReg = origReg
+ m.paradigmReg = paradigmReg
+ m.origScript = origScript
+ m.regGroupDist = regGroupDist
+ }
+}
+
+func isParadigmLocale(lang langID, r regionID) bool {
+ for _, e := range paradigmLocales {
+ if langID(e[0]) == lang && (r == regionID(e[1]) || r == regionID(e[2])) {
+ return true
+ }
+ }
+ return false
+}
+
+// regionGroupDist computes the distance between two regions based on their
+// CLDR grouping.
+func regionGroupDist(a, b regionID, script scriptID, lang langID) (dist uint8, same bool) {
+ const defaultDistance = 4
+
+ aGroup := uint(regionToGroups[a]) << 1
+ bGroup := uint(regionToGroups[b]) << 1
+ for _, ri := range matchRegion {
+ if langID(ri.lang) == lang && (ri.script == 0 || scriptID(ri.script) == script) {
+ group := uint(1 << (ri.group &^ 0x80))
+ if 0x80&ri.group == 0 {
+ if aGroup&bGroup&group != 0 { // Both regions are in the group.
+ return ri.distance, ri.distance == defaultDistance
+ }
+ } else {
+ if (aGroup|bGroup)&group == 0 { // Both regions are not in the group.
+ return ri.distance, ri.distance == defaultDistance
+ }
+ }
+ }
+ }
+ return defaultDistance, true
+}
+
+func (t Tag) variants() string {
+ if t.pVariant == 0 {
+ return ""
+ }
+ return t.str[t.pVariant:t.pExt]
+}
+
+// variantOrPrivateTagStr returns variants or private use tags.
+func (t Tag) variantOrPrivateTagStr() string {
+ if t.pExt > 0 {
+ return t.str[t.pVariant:t.pExt]
+ }
+ return t.str[t.pVariant:]
+}
+
+// equalsRest compares everything except the language.
+func (a Tag) equalsRest(b Tag) bool {
+ // TODO: don't include extensions in this comparison. To do this efficiently,
+ // though, we should handle private tags separately.
+ return a.script == b.script && a.region == b.region && a.variantOrPrivateTagStr() == b.variantOrPrivateTagStr()
+}
+
+// isExactEquivalent returns true if canonicalizing the language will not alter
+// the script or region of a tag.
+func isExactEquivalent(l langID) bool {
+ for _, o := range notEquivalent {
+ if o == l {
+ return false
+ }
+ }
+ return true
+}
+
+var notEquivalent []langID
+
+func init() {
+ // Create a list of all languages for which canonicalization may alter the
+ // script or region.
+ for _, lm := range langAliasMap {
+ tag := Tag{lang: langID(lm.from)}
+ if tag, _ = tag.canonicalize(All); tag.script != 0 || tag.region != 0 {
+ notEquivalent = append(notEquivalent, langID(lm.from))
+ }
+ }
+ // Maximize undefined regions of paradigm locales.
+ for i, v := range paradigmLocales {
+ max, _ := addTags(Tag{lang: langID(v[0])})
+ if v[1] == 0 {
+ paradigmLocales[i][1] = uint16(max.region)
+ }
+ if v[2] == 0 {
+ paradigmLocales[i][2] = uint16(max.region)
+ }
+ }
+}
diff --git a/language/internal/match_test.go b/language/internal/match_test.go
new file mode 100644
index 0000000..5481af5
--- /dev/null
+++ b/language/internal/match_test.go
@@ -0,0 +1,505 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language
+
+import (
+ "bytes"
+ "flag"
+ "fmt"
+ "os"
+ "path"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "golang.org/x/text/internal/testtext"
+ "golang.org/x/text/internal/ucd"
+)
+
+var verbose = flag.Bool("verbose", false, "set to true to print the internal tables of matchers")
+
+func TestCompliance(t *testing.T) {
+ filepath.Walk("../testdata", func(file string, info os.FileInfo, err error) error {
+ if info.IsDir() {
+ return nil
+ }
+ r, err := os.Open(file)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ucd.Parse(r, func(p *ucd.Parser) {
+ name := strings.Replace(path.Join(p.String(0), p.String(1)), " ", "", -1)
+ if skip[name] {
+ return
+ }
+ t.Run(info.Name()+"/"+name, func(t *testing.T) {
+ supported := makeTagList(p.String(0))
+ desired := makeTagList(p.String(1))
+ gotCombined, index, conf := NewMatcher(supported).Match(desired...)
+
+ gotMatch := supported[index]
+ wantMatch := mk(p.String(2))
+ if gotMatch != wantMatch {
+ t.Fatalf("match: got %q; want %q (%v)", gotMatch, wantMatch, conf)
+ }
+ wantCombined, err := Raw.Parse(p.String(3))
+ if err == nil && gotCombined != wantCombined {
+ t.Errorf("combined: got %q; want %q (%v)", gotCombined, wantCombined, conf)
+ }
+ })
+ })
+ return nil
+ })
+}
+
+var skip = map[string]bool{
+ // TODO: bugs
+ // Honor the wildcard match. This may only be useful to select non-exact
+ // stuff.
+ "mul,af/nl": true, // match: got "af"; want "mul"
+
+ // TODO: include other extensions.
+ // combined: got "en-GB-u-ca-buddhist-nu-arab"; want "en-GB-fonipa-t-m0-iso-i0-pinyin-u-ca-buddhist-nu-arab"
+ "und,en-GB-u-sd-gbsct/en-fonipa-u-nu-Arab-ca-buddhist-t-m0-iso-i0-pinyin": true,
+
+ // Inconsistencies with Mark Davis' implementation where it is not clear
+ // which is better.
+
+ // Inconsistencies in combined. I think the Go approach is more appropriate.
+ // We could use -u-rg- and -u-va- as alternative.
+ "und,fr/fr-BE-fonipa": true, // combined: got "fr"; want "fr-BE-fonipa"
+ "und,fr-CA/fr-BE-fonipa": true, // combined: got "fr-CA"; want "fr-BE-fonipa"
+ "und,fr-fonupa/fr-BE-fonipa": true, // combined: got "fr-fonupa"; want "fr-BE-fonipa"
+ "und,no/nn-BE-fonipa": true, // combined: got "no"; want "no-BE-fonipa"
+ "50,und,fr-CA-fonupa/fr-BE-fonipa": true, // combined: got "fr-CA-fonupa"; want "fr-BE-fonipa"
+
+ // The initial number is a threshold. As we don't use scoring, we will not
+ // implement this.
+ "50,und,fr-Cyrl-CA-fonupa/fr-BE-fonipa": true,
+ // match: got "und"; want "fr-Cyrl-CA-fonupa"
+ // combined: got "und"; want "fr-Cyrl-BE-fonipa"
+
+ // Other interesting cases to test:
+ // - Should same language or same script have the preference if there is
+ // usually no understanding of the other script?
+ // - More specific region in desired may replace enclosing supported.
+}
+
+func makeTagList(s string) (tags []Tag) {
+ for _, s := range strings.Split(s, ",") {
+ tags = append(tags, mk(strings.TrimSpace(s)))
+ }
+ return tags
+}
+
+func TestMatchStrings(t *testing.T) {
+ testCases := []struct {
+ supported string
+ desired string // strings separted by |
+ tag string
+ index int
+ }{{
+ supported: "en",
+ desired: "",
+ tag: "en",
+ index: 0,
+ }, {
+ supported: "en",
+ desired: "nl",
+ tag: "en",
+ index: 0,
+ }, {
+ supported: "en,nl",
+ desired: "nl",
+ tag: "nl",
+ index: 1,
+ }, {
+ supported: "en,nl",
+ desired: "nl|en",
+ tag: "nl",
+ index: 1,
+ }, {
+ supported: "en-GB,nl",
+ desired: "en ; q=0.1,nl",
+ tag: "nl",
+ index: 1,
+ }, {
+ supported: "en-GB,nl",
+ desired: "en;q=0.005 | dk; q=0.1,nl ",
+ tag: "en-GB",
+ index: 0,
+ }, {
+ // do not match faulty tags with und
+ supported: "en,und",
+ desired: "|en",
+ tag: "en",
+ index: 0,
+ }}
+ for _, tc := range testCases {
+ t.Run(path.Join(tc.supported, tc.desired), func(t *testing.T) {
+ m := NewMatcher(makeTagList(tc.supported))
+ tag, index := MatchStrings(m, strings.Split(tc.desired, "|")...)
+ if tag.String() != tc.tag || index != tc.index {
+ t.Errorf("got %v, %d; want %v, %d", tag, index, tc.tag, tc.index)
+ }
+ })
+ }
+}
+
+func TestAddLikelySubtags(t *testing.T) {
+ tests := []struct{ in, out string }{
+ {"aa", "aa-Latn-ET"},
+ {"aa-Latn", "aa-Latn-ET"},
+ {"aa-Arab", "aa-Arab-ET"},
+ {"aa-Arab-ER", "aa-Arab-ER"},
+ {"kk", "kk-Cyrl-KZ"},
+ {"kk-CN", "kk-Arab-CN"},
+ {"cmn", "cmn"},
+ {"zh-AU", "zh-Hant-AU"},
+ {"zh-VN", "zh-Hant-VN"},
+ {"zh-SG", "zh-Hans-SG"},
+ {"zh-Hant", "zh-Hant-TW"},
+ {"zh-Hani", "zh-Hani-CN"},
+ {"und-Hani", "zh-Hani-CN"},
+ {"und", "en-Latn-US"},
+ {"und-GB", "en-Latn-GB"},
+ {"und-CW", "pap-Latn-CW"},
+ {"und-YT", "fr-Latn-YT"},
+ {"und-Arab", "ar-Arab-EG"},
+ {"und-AM", "hy-Armn-AM"},
+ {"und-TW", "zh-Hant-TW"},
+ {"und-002", "en-Latn-NG"},
+ {"und-Latn-002", "en-Latn-NG"},
+ {"en-Latn-002", "en-Latn-NG"},
+ {"en-002", "en-Latn-NG"},
+ {"en-001", "en-Latn-US"},
+ {"und-003", "en-Latn-US"},
+ {"und-GB", "en-Latn-GB"},
+ {"Latn-001", "en-Latn-US"},
+ {"en-001", "en-Latn-US"},
+ {"es-419", "es-Latn-419"},
+ {"he-145", "he-Hebr-IL"},
+ {"ky-145", "ky-Latn-TR"},
+ {"kk", "kk-Cyrl-KZ"},
+ // Don't specialize duplicate and ambiguous matches.
+ {"kk-034", "kk-Arab-034"}, // Matches IR and AF. Both are Arab.
+ {"ku-145", "ku-Latn-TR"}, // Matches IQ, TR, and LB, but kk -> TR.
+ {"und-Arab-CC", "ms-Arab-CC"},
+ {"und-Arab-GB", "ks-Arab-GB"},
+ {"und-Hans-CC", "zh-Hans-CC"},
+ {"und-CC", "en-Latn-CC"},
+ {"sr", "sr-Cyrl-RS"},
+ {"sr-151", "sr-Latn-151"}, // Matches RO and RU.
+ // We would like addLikelySubtags to generate the same results if the input
+ // only changes by adding tags that would otherwise have been added
+ // by the expansion.
+ // In other words:
+ // und-AA -> xx-Scrp-AA implies und-Scrp-AA -> xx-Scrp-AA
+ // und-AA -> xx-Scrp-AA implies xx-AA -> xx-Scrp-AA
+ // und-Scrp -> xx-Scrp-AA implies und-Scrp-AA -> xx-Scrp-AA
+ // und-Scrp -> xx-Scrp-AA implies xx-Scrp -> xx-Scrp-AA
+ // xx -> xx-Scrp-AA implies xx-Scrp -> xx-Scrp-AA
+ // xx -> xx-Scrp-AA implies xx-AA -> xx-Scrp-AA
+ //
+ // The algorithm specified in
+ // http://unicode.org/reports/tr35/tr35-9.html#Supplemental_Data,
+ // Section C.10, does not handle the first case. For example,
+ // the CLDR data contains an entry und-BJ -> fr-Latn-BJ, but not
+ // there is no rule for und-Latn-BJ. According to spec, und-Latn-BJ
+ // would expand to en-Latn-BJ, violating the aforementioned principle.
+ // We deviate from the spec by letting und-Scrp-AA expand to xx-Scrp-AA
+ // if a rule of the form und-AA -> xx-Scrp-AA is defined.
+ // Note that as of version 23, CLDR has some explicitly specified
+ // entries that do not conform to these rules. The implementation
+ // will not correct these explicit inconsistencies. A later versions of CLDR
+ // is supposed to fix this.
+ {"und-Latn-BJ", "fr-Latn-BJ"},
+ {"und-Bugi-ID", "bug-Bugi-ID"},
+ // regions, scripts and languages without definitions
+ {"und-Arab-AA", "ar-Arab-AA"},
+ {"und-Afak-RE", "fr-Afak-RE"},
+ {"und-Arab-GB", "ks-Arab-GB"},
+ {"abp-Arab-GB", "abp-Arab-GB"},
+ // script has preference over region
+ {"und-Arab-NL", "ar-Arab-NL"},
+ {"zza", "zza-Latn-TR"},
+ // preserve variants and extensions
+ {"de-1901", "de-Latn-DE-1901"},
+ {"de-x-abc", "de-Latn-DE-x-abc"},
+ {"de-1901-x-abc", "de-Latn-DE-1901-x-abc"},
+ {"x-abc", "x-abc"}, // TODO: is this the desired behavior?
+ }
+ for i, tt := range tests {
+ in, _ := Parse(tt.in)
+ out, _ := Parse(tt.out)
+ in, _ = in.addLikelySubtags()
+ if in.String() != out.String() {
+ t.Errorf("%d: add(%s) was %s; want %s", i, tt.in, in, tt.out)
+ }
+ }
+}
+func TestMinimize(t *testing.T) {
+ tests := []struct{ in, out string }{
+ {"aa", "aa"},
+ {"aa-Latn", "aa"},
+ {"aa-Latn-ET", "aa"},
+ {"aa-ET", "aa"},
+ {"aa-Arab", "aa-Arab"},
+ {"aa-Arab-ER", "aa-Arab-ER"},
+ {"aa-Arab-ET", "aa-Arab"},
+ {"und", "und"},
+ {"und-Latn", "und"},
+ {"und-Latn-US", "und"},
+ {"en-Latn-US", "en"},
+ {"cmn", "cmn"},
+ {"cmn-Hans", "cmn-Hans"},
+ {"cmn-Hant", "cmn-Hant"},
+ {"zh-AU", "zh-AU"},
+ {"zh-VN", "zh-VN"},
+ {"zh-SG", "zh-SG"},
+ {"zh-Hant", "zh-Hant"},
+ {"zh-Hant-TW", "zh-TW"},
+ {"zh-Hans", "zh"},
+ {"zh-Hani", "zh-Hani"},
+ {"und-Hans", "und-Hans"},
+ {"und-Hani", "und-Hani"},
+
+ {"und-CW", "und-CW"},
+ {"und-YT", "und-YT"},
+ {"und-Arab", "und-Arab"},
+ {"und-AM", "und-AM"},
+ {"und-Arab-CC", "und-Arab-CC"},
+ {"und-CC", "und-CC"},
+ {"und-Latn-BJ", "und-BJ"},
+ {"und-Bugi-ID", "und-Bugi"},
+ {"bug-Bugi-ID", "bug-Bugi"},
+ // regions, scripts and languages without definitions
+ {"und-Arab-AA", "und-Arab-AA"},
+ // preserve variants and extensions
+ {"de-Latn-1901", "de-1901"},
+ {"de-Latn-x-abc", "de-x-abc"},
+ {"de-DE-1901-x-abc", "de-1901-x-abc"},
+ {"x-abc", "x-abc"}, // TODO: is this the desired behavior?
+ }
+ for i, tt := range tests {
+ in, _ := Parse(tt.in)
+ out, _ := Parse(tt.out)
+ min, _ := in.minimize()
+ if min.String() != out.String() {
+ t.Errorf("%d: min(%s) was %s; want %s", i, tt.in, min, tt.out)
+ }
+ max, _ := min.addLikelySubtags()
+ if x, _ := in.addLikelySubtags(); x.String() != max.String() {
+ t.Errorf("%d: max(min(%s)) = %s; want %s", i, tt.in, max, x)
+ }
+ }
+}
+
+func TestRegionGroups(t *testing.T) {
+ testCases := []struct {
+ a, b string
+ distance uint8
+ }{
+ {"zh-TW", "zh-HK", 5},
+ {"zh-MO", "zh-HK", 4},
+ {"es-ES", "es-AR", 5},
+ {"es-ES", "es", 4},
+ {"es-419", "es-MX", 4},
+ {"es-AR", "es-MX", 4},
+ {"es-ES", "es-MX", 5},
+ {"es-PT", "es-MX", 5},
+ }
+ for _, tc := range testCases {
+ a := MustParse(tc.a)
+ aScript, _ := a.Script()
+ b := MustParse(tc.b)
+ bScript, _ := b.Script()
+
+ if aScript != bScript {
+ t.Errorf("scripts differ: %q vs %q", aScript, bScript)
+ continue
+ }
+ d, _ := regionGroupDist(a.region, b.region, aScript.scriptID, a.lang)
+ if d != tc.distance {
+ t.Errorf("got %q; want %q", d, tc.distance)
+ }
+ }
+}
+
+func TestIsParadigmLocale(t *testing.T) {
+ testCases := map[string]bool{
+ "en-US": true,
+ "en-GB": true,
+ "en-VI": false,
+ "es-GB": false,
+ "es-ES": true,
+ "es-419": true,
+ }
+ for str, want := range testCases {
+ tag := Make(str)
+ got := isParadigmLocale(tag.lang, tag.region)
+ if got != want {
+ t.Errorf("isPL(%q) = %v; want %v", str, got, want)
+ }
+ }
+}
+
+// Implementation of String methods for various types for debugging purposes.
+
+func (m *matcher) String() string {
+ w := &bytes.Buffer{}
+ fmt.Fprintln(w, "Default:", m.default_)
+ for tag, h := range m.index {
+ fmt.Fprintf(w, " %s: %v\n", tag, h)
+ }
+ return w.String()
+}
+
+func (h *matchHeader) String() string {
+ w := &bytes.Buffer{}
+ fmt.Fprint(w, "haveTag: ")
+ for _, h := range h.haveTags {
+ fmt.Fprintf(w, "%v, ", h)
+ }
+ return w.String()
+}
+
+func (t haveTag) String() string {
+ return fmt.Sprintf("%v:%d:%v:%v-%v|%v", t.tag, t.index, t.conf, t.maxRegion, t.maxScript, t.altScript)
+}
+
+func TestBestMatchAlloc(t *testing.T) {
+ m := NewMatcher(makeTagList("en sr nl"))
+ // Go allocates when creating a list of tags from a single tag!
+ list := []Tag{English}
+ avg := testtext.AllocsPerRun(1, func() {
+ m.Match(list...)
+ })
+ if avg > 0 {
+ t.Errorf("got %f; want 0", avg)
+ }
+}
+
+var benchHave = []Tag{
+ mk("en"),
+ mk("en-GB"),
+ mk("za"),
+ mk("zh-Hant"),
+ mk("zh-Hans-CN"),
+ mk("zh"),
+ mk("zh-HK"),
+ mk("ar-MK"),
+ mk("en-CA"),
+ mk("fr-CA"),
+ mk("fr-US"),
+ mk("fr-CH"),
+ mk("fr"),
+ mk("lt"),
+ mk("lv"),
+ mk("iw"),
+ mk("iw-NL"),
+ mk("he"),
+ mk("he-IT"),
+ mk("tlh"),
+ mk("ja"),
+ mk("ja-Jpan"),
+ mk("ja-Jpan-JP"),
+ mk("de"),
+ mk("de-CH"),
+ mk("de-AT"),
+ mk("de-DE"),
+ mk("sr"),
+ mk("sr-Latn"),
+ mk("sr-Cyrl"),
+ mk("sr-ME"),
+}
+
+var benchWant = [][]Tag{
+ []Tag{
+ mk("en"),
+ },
+ []Tag{
+ mk("en-AU"),
+ mk("de-HK"),
+ mk("nl"),
+ mk("fy"),
+ mk("lv"),
+ },
+ []Tag{
+ mk("en-AU"),
+ mk("de-HK"),
+ mk("nl"),
+ mk("fy"),
+ },
+ []Tag{
+ mk("ja-Hant"),
+ mk("da-HK"),
+ mk("nl"),
+ mk("zh-TW"),
+ },
+ []Tag{
+ mk("ja-Hant"),
+ mk("da-HK"),
+ mk("nl"),
+ mk("hr"),
+ },
+}
+
+func BenchmarkMatch(b *testing.B) {
+ m := newMatcher(benchHave, nil)
+ for i := 0; i < b.N; i++ {
+ for _, want := range benchWant {
+ m.getBest(want...)
+ }
+ }
+}
+
+func BenchmarkMatchExact(b *testing.B) {
+ want := mk("en")
+ m := newMatcher(benchHave, nil)
+ for i := 0; i < b.N; i++ {
+ m.getBest(want)
+ }
+}
+
+func BenchmarkMatchAltLanguagePresent(b *testing.B) {
+ want := mk("hr")
+ m := newMatcher(benchHave, nil)
+ for i := 0; i < b.N; i++ {
+ m.getBest(want)
+ }
+}
+
+func BenchmarkMatchAltLanguageNotPresent(b *testing.B) {
+ want := mk("nn")
+ m := newMatcher(benchHave, nil)
+ for i := 0; i < b.N; i++ {
+ m.getBest(want)
+ }
+}
+
+func BenchmarkMatchAltScriptPresent(b *testing.B) {
+ want := mk("zh-Hant-CN")
+ m := newMatcher(benchHave, nil)
+ for i := 0; i < b.N; i++ {
+ m.getBest(want)
+ }
+}
+
+func BenchmarkMatchAltScriptNotPresent(b *testing.B) {
+ want := mk("fr-Cyrl")
+ m := newMatcher(benchHave, nil)
+ for i := 0; i < b.N; i++ {
+ m.getBest(want)
+ }
+}
+
+func BenchmarkMatchLimitedExact(b *testing.B) {
+ want := []Tag{mk("he-NL"), mk("iw-NL")}
+ m := newMatcher(benchHave, nil)
+ for i := 0; i < b.N; i++ {
+ m.getBest(want...)
+ }
+}
diff --git a/language/internal/parse.go b/language/internal/parse.go
new file mode 100644
index 0000000..fca2d30
--- /dev/null
+++ b/language/internal/parse.go
@@ -0,0 +1,859 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "sort"
+ "strconv"
+ "strings"
+
+ "golang.org/x/text/internal/tag"
+)
+
+// isAlpha returns true if the byte is not a digit.
+// b must be an ASCII letter or digit.
+func isAlpha(b byte) bool {
+ return b > '9'
+}
+
+// isAlphaNum returns true if the string contains only ASCII letters or digits.
+func isAlphaNum(s []byte) bool {
+ for _, c := range s {
+ if !('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9') {
+ return false
+ }
+ }
+ return true
+}
+
+// errSyntax is returned by any of the parsing functions when the
+// input is not well-formed, according to BCP 47.
+// TODO: return the position at which the syntax error occurred?
+var errSyntax = errors.New("language: tag is not well-formed")
+
+// ValueError is returned by any of the parsing functions when the
+// input is well-formed but the respective subtag is not recognized
+// as a valid value.
+type ValueError struct {
+ v [8]byte
+}
+
+func mkErrInvalid(s []byte) error {
+ var e ValueError
+ copy(e.v[:], s)
+ return e
+}
+
+func (e ValueError) tag() []byte {
+ n := bytes.IndexByte(e.v[:], 0)
+ if n == -1 {
+ n = 8
+ }
+ return e.v[:n]
+}
+
+// Error implements the error interface.
+func (e ValueError) Error() string {
+ return fmt.Sprintf("language: subtag %q is well-formed but unknown", e.tag())
+}
+
+// Subtag returns the subtag for which the error occurred.
+func (e ValueError) Subtag() string {
+ return string(e.tag())
+}
+
+// scanner is used to scan BCP 47 tokens, which are separated by _ or -.
+type scanner struct {
+ b []byte
+ bytes [max99thPercentileSize]byte
+ token []byte
+ start int // start position of the current token
+ end int // end position of the current token
+ next int // next point for scan
+ err error
+ done bool
+}
+
+func makeScannerString(s string) scanner {
+ scan := scanner{}
+ if len(s) <= len(scan.bytes) {
+ scan.b = scan.bytes[:copy(scan.bytes[:], s)]
+ } else {
+ scan.b = []byte(s)
+ }
+ scan.init()
+ return scan
+}
+
+// makeScanner returns a scanner using b as the input buffer.
+// b is not copied and may be modified by the scanner routines.
+func makeScanner(b []byte) scanner {
+ scan := scanner{b: b}
+ scan.init()
+ return scan
+}
+
+func (s *scanner) init() {
+ for i, c := range s.b {
+ if c == '_' {
+ s.b[i] = '-'
+ }
+ }
+ s.scan()
+}
+
+// restToLower converts the string between start and end to lower case.
+func (s *scanner) toLower(start, end int) {
+ for i := start; i < end; i++ {
+ c := s.b[i]
+ if 'A' <= c && c <= 'Z' {
+ s.b[i] += 'a' - 'A'
+ }
+ }
+}
+
+func (s *scanner) setError(e error) {
+ if s.err == nil || (e == errSyntax && s.err != errSyntax) {
+ s.err = e
+ }
+}
+
+// resizeRange shrinks or grows the array at position oldStart such that
+// a new string of size newSize can fit between oldStart and oldEnd.
+// Sets the scan point to after the resized range.
+func (s *scanner) resizeRange(oldStart, oldEnd, newSize int) {
+ s.start = oldStart
+ if end := oldStart + newSize; end != oldEnd {
+ diff := end - oldEnd
+ if end < cap(s.b) {
+ b := make([]byte, len(s.b)+diff)
+ copy(b, s.b[:oldStart])
+ copy(b[end:], s.b[oldEnd:])
+ s.b = b
+ } else {
+ s.b = append(s.b[end:], s.b[oldEnd:]...)
+ }
+ s.next = end + (s.next - s.end)
+ s.end = end
+ }
+}
+
+// replace replaces the current token with repl.
+func (s *scanner) replace(repl string) {
+ s.resizeRange(s.start, s.end, len(repl))
+ copy(s.b[s.start:], repl)
+}
+
+// gobble removes the current token from the input.
+// Caller must call scan after calling gobble.
+func (s *scanner) gobble(e error) {
+ s.setError(e)
+ if s.start == 0 {
+ s.b = s.b[:+copy(s.b, s.b[s.next:])]
+ s.end = 0
+ } else {
+ s.b = s.b[:s.start-1+copy(s.b[s.start-1:], s.b[s.end:])]
+ s.end = s.start - 1
+ }
+ s.next = s.start
+}
+
+// deleteRange removes the given range from s.b before the current token.
+func (s *scanner) deleteRange(start, end int) {
+ s.setError(errSyntax)
+ s.b = s.b[:start+copy(s.b[start:], s.b[end:])]
+ diff := end - start
+ s.next -= diff
+ s.start -= diff
+ s.end -= diff
+}
+
+// scan parses the next token of a BCP 47 string. Tokens that are larger
+// than 8 characters or include non-alphanumeric characters result in an error
+// and are gobbled and removed from the output.
+// It returns the end position of the last token consumed.
+func (s *scanner) scan() (end int) {
+ end = s.end
+ s.token = nil
+ for s.start = s.next; s.next < len(s.b); {
+ i := bytes.IndexByte(s.b[s.next:], '-')
+ if i == -1 {
+ s.end = len(s.b)
+ s.next = len(s.b)
+ i = s.end - s.start
+ } else {
+ s.end = s.next + i
+ s.next = s.end + 1
+ }
+ token := s.b[s.start:s.end]
+ if i < 1 || i > 8 || !isAlphaNum(token) {
+ s.gobble(errSyntax)
+ continue
+ }
+ s.token = token
+ return end
+ }
+ if n := len(s.b); n > 0 && s.b[n-1] == '-' {
+ s.setError(errSyntax)
+ s.b = s.b[:len(s.b)-1]
+ }
+ s.done = true
+ return end
+}
+
+// acceptMinSize parses multiple tokens of the given size or greater.
+// It returns the end position of the last token consumed.
+func (s *scanner) acceptMinSize(min int) (end int) {
+ end = s.end
+ s.scan()
+ for ; len(s.token) >= min; s.scan() {
+ end = s.end
+ }
+ return end
+}
+
+// Parse parses the given BCP 47 string and returns a valid Tag. If parsing
+// failed it returns an error and any part of the tag that could be parsed.
+// If parsing succeeded but an unknown value was found, it returns
+// ValueError. The Tag returned in this case is just stripped of the unknown
+// value. All other values are preserved. It accepts tags in the BCP 47 format
+// and extensions to this standard defined in
+// http://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
+// The resulting tag is canonicalized using the default canonicalization type.
+func Parse(s string) (t Tag, err error) {
+ return Default.Parse(s)
+}
+
+// Parse parses the given BCP 47 string and returns a valid Tag. If parsing
+// failed it returns an error and any part of the tag that could be parsed.
+// If parsing succeeded but an unknown value was found, it returns
+// ValueError. The Tag returned in this case is just stripped of the unknown
+// value. All other values are preserved. It accepts tags in the BCP 47 format
+// and extensions to this standard defined in
+// http://www.unicode.org/reports/tr35/#Unicode_Language_and_Locale_Identifiers.
+// The resulting tag is canonicalized using the the canonicalization type c.
+func (c CanonType) Parse(s string) (t Tag, err error) {
+ // TODO: consider supporting old-style locale key-value pairs.
+ if s == "" {
+ return und, errSyntax
+ }
+ if len(s) <= maxAltTaglen {
+ b := [maxAltTaglen]byte{}
+ for i, c := range s {
+ // Generating invalid UTF-8 is okay as it won't match.
+ if 'A' <= c && c <= 'Z' {
+ c += 'a' - 'A'
+ } else if c == '_' {
+ c = '-'
+ }
+ b[i] = byte(c)
+ }
+ if t, ok := grandfathered(b); ok {
+ return t, nil
+ }
+ }
+ scan := makeScannerString(s)
+ t, err = parse(&scan, s)
+ t, changed := t.canonicalize(c)
+ if changed {
+ t.remakeString()
+ }
+ return t, err
+}
+
+func parse(scan *scanner, s string) (t Tag, err error) {
+ t = und
+ var end int
+ if n := len(scan.token); n <= 1 {
+ scan.toLower(0, len(scan.b))
+ if n == 0 || scan.token[0] != 'x' {
+ return t, errSyntax
+ }
+ end = parseExtensions(scan)
+ } else if n >= 4 {
+ return und, errSyntax
+ } else { // the usual case
+ t, end = parseTag(scan)
+ if n := len(scan.token); n == 1 {
+ t.pExt = uint16(end)
+ end = parseExtensions(scan)
+ } else if end < len(scan.b) {
+ scan.setError(errSyntax)
+ scan.b = scan.b[:end]
+ }
+ }
+ if int(t.pVariant) < len(scan.b) {
+ if end < len(s) {
+ s = s[:end]
+ }
+ if len(s) > 0 && tag.Compare(s, scan.b) == 0 {
+ t.str = s
+ } else {
+ t.str = string(scan.b)
+ }
+ } else {
+ t.pVariant, t.pExt = 0, 0
+ }
+ return t, scan.err
+}
+
+// parseTag parses language, script, region and variants.
+// It returns a Tag and the end position in the input that was parsed.
+func parseTag(scan *scanner) (t Tag, end int) {
+ var e error
+ // TODO: set an error if an unknown lang, script or region is encountered.
+ t.lang, e = getLangID(scan.token)
+ scan.setError(e)
+ scan.replace(t.lang.String())
+ langStart := scan.start
+ end = scan.scan()
+ for len(scan.token) == 3 && isAlpha(scan.token[0]) {
+ // From http://tools.ietf.org/html/bcp47, <lang>-<extlang> tags are equivalent
+ // to a tag of the form <extlang>.
+ lang, e := getLangID(scan.token)
+ if lang != 0 {
+ t.lang = lang
+ copy(scan.b[langStart:], lang.String())
+ scan.b[langStart+3] = '-'
+ scan.start = langStart + 4
+ }
+ scan.gobble(e)
+ end = scan.scan()
+ }
+ if len(scan.token) == 4 && isAlpha(scan.token[0]) {
+ t.script, e = getScriptID(script, scan.token)
+ if t.script == 0 {
+ scan.gobble(e)
+ }
+ end = scan.scan()
+ }
+ if n := len(scan.token); n >= 2 && n <= 3 {
+ t.region, e = getRegionID(scan.token)
+ if t.region == 0 {
+ scan.gobble(e)
+ } else {
+ scan.replace(t.region.String())
+ }
+ end = scan.scan()
+ }
+ scan.toLower(scan.start, len(scan.b))
+ t.pVariant = byte(end)
+ end = parseVariants(scan, end, t)
+ t.pExt = uint16(end)
+ return t, end
+}
+
+var separator = []byte{'-'}
+
+// parseVariants scans tokens as long as each token is a valid variant string.
+// Duplicate variants are removed.
+func parseVariants(scan *scanner, end int, t Tag) int {
+ start := scan.start
+ varIDBuf := [4]uint8{}
+ variantBuf := [4][]byte{}
+ varID := varIDBuf[:0]
+ variant := variantBuf[:0]
+ last := -1
+ needSort := false
+ for ; len(scan.token) >= 4; scan.scan() {
+ // TODO: measure the impact of needing this conversion and redesign
+ // the data structure if there is an issue.
+ v, ok := variantIndex[string(scan.token)]
+ if !ok {
+ // unknown variant
+ // TODO: allow user-defined variants?
+ scan.gobble(mkErrInvalid(scan.token))
+ continue
+ }
+ varID = append(varID, v)
+ variant = append(variant, scan.token)
+ if !needSort {
+ if last < int(v) {
+ last = int(v)
+ } else {
+ needSort = true
+ // There is no legal combinations of more than 7 variants
+ // (and this is by no means a useful sequence).
+ const maxVariants = 8
+ if len(varID) > maxVariants {
+ break
+ }
+ }
+ }
+ end = scan.end
+ }
+ if needSort {
+ sort.Sort(variantsSort{varID, variant})
+ k, l := 0, -1
+ for i, v := range varID {
+ w := int(v)
+ if l == w {
+ // Remove duplicates.
+ continue
+ }
+ varID[k] = varID[i]
+ variant[k] = variant[i]
+ k++
+ l = w
+ }
+ if str := bytes.Join(variant[:k], separator); len(str) == 0 {
+ end = start - 1
+ } else {
+ scan.resizeRange(start, end, len(str))
+ copy(scan.b[scan.start:], str)
+ end = scan.end
+ }
+ }
+ return end
+}
+
+type variantsSort struct {
+ i []uint8
+ v [][]byte
+}
+
+func (s variantsSort) Len() int {
+ return len(s.i)
+}
+
+func (s variantsSort) Swap(i, j int) {
+ s.i[i], s.i[j] = s.i[j], s.i[i]
+ s.v[i], s.v[j] = s.v[j], s.v[i]
+}
+
+func (s variantsSort) Less(i, j int) bool {
+ return s.i[i] < s.i[j]
+}
+
+type bytesSort [][]byte
+
+func (b bytesSort) Len() int {
+ return len(b)
+}
+
+func (b bytesSort) Swap(i, j int) {
+ b[i], b[j] = b[j], b[i]
+}
+
+func (b bytesSort) Less(i, j int) bool {
+ return bytes.Compare(b[i], b[j]) == -1
+}
+
+// parseExtensions parses and normalizes the extensions in the buffer.
+// It returns the last position of scan.b that is part of any extension.
+// It also trims scan.b to remove excess parts accordingly.
+func parseExtensions(scan *scanner) int {
+ start := scan.start
+ exts := [][]byte{}
+ private := []byte{}
+ end := scan.end
+ for len(scan.token) == 1 {
+ extStart := scan.start
+ ext := scan.token[0]
+ end = parseExtension(scan)
+ extension := scan.b[extStart:end]
+ if len(extension) < 3 || (ext != 'x' && len(extension) < 4) {
+ scan.setError(errSyntax)
+ end = extStart
+ continue
+ } else if start == extStart && (ext == 'x' || scan.start == len(scan.b)) {
+ scan.b = scan.b[:end]
+ return end
+ } else if ext == 'x' {
+ private = extension
+ break
+ }
+ exts = append(exts, extension)
+ }
+ sort.Sort(bytesSort(exts))
+ if len(private) > 0 {
+ exts = append(exts, private)
+ }
+ scan.b = scan.b[:start]
+ if len(exts) > 0 {
+ scan.b = append(scan.b, bytes.Join(exts, separator)...)
+ } else if start > 0 {
+ // Strip trailing '-'.
+ scan.b = scan.b[:start-1]
+ }
+ return end
+}
+
+// parseExtension parses a single extension and returns the position of
+// the extension end.
+func parseExtension(scan *scanner) int {
+ start, end := scan.start, scan.end
+ switch scan.token[0] {
+ case 'u':
+ attrStart := end
+ scan.scan()
+ for last := []byte{}; len(scan.token) > 2; scan.scan() {
+ if bytes.Compare(scan.token, last) != -1 {
+ // Attributes are unsorted. Start over from scratch.
+ p := attrStart + 1
+ scan.next = p
+ attrs := [][]byte{}
+ for scan.scan(); len(scan.token) > 2; scan.scan() {
+ attrs = append(attrs, scan.token)
+ end = scan.end
+ }
+ sort.Sort(bytesSort(attrs))
+ copy(scan.b[p:], bytes.Join(attrs, separator))
+ break
+ }
+ last = scan.token
+ end = scan.end
+ }
+ var last, key []byte
+ for attrEnd := end; len(scan.token) == 2; last = key {
+ key = scan.token
+ keyEnd := scan.end
+ end = scan.acceptMinSize(3)
+ // TODO: check key value validity
+ if keyEnd == end || bytes.Compare(key, last) != 1 {
+ // We have an invalid key or the keys are not sorted.
+ // Start scanning keys from scratch and reorder.
+ p := attrEnd + 1
+ scan.next = p
+ keys := [][]byte{}
+ for scan.scan(); len(scan.token) == 2; {
+ keyStart, keyEnd := scan.start, scan.end
+ end = scan.acceptMinSize(3)
+ if keyEnd != end {
+ keys = append(keys, scan.b[keyStart:end])
+ } else {
+ scan.setError(errSyntax)
+ end = keyStart
+ }
+ }
+ sort.Sort(bytesSort(keys))
+ reordered := bytes.Join(keys, separator)
+ if e := p + len(reordered); e < end {
+ scan.deleteRange(e, end)
+ end = e
+ }
+ copy(scan.b[p:], bytes.Join(keys, separator))
+ break
+ }
+ }
+ case 't':
+ scan.scan()
+ if n := len(scan.token); n >= 2 && n <= 3 && isAlpha(scan.token[1]) {
+ _, end = parseTag(scan)
+ scan.toLower(start, end)
+ }
+ for len(scan.token) == 2 && !isAlpha(scan.token[1]) {
+ end = scan.acceptMinSize(3)
+ }
+ case 'x':
+ end = scan.acceptMinSize(1)
+ default:
+ end = scan.acceptMinSize(2)
+ }
+ return end
+}
+
+// Compose creates a Tag from individual parts, which may be of type Tag, Base,
+// Script, Region, Variant, []Variant, Extension, []Extension or error. If a
+// Base, Script or Region or slice of type Variant or Extension is passed more
+// than once, the latter will overwrite the former. Variants and Extensions are
+// accumulated, but if two extensions of the same type are passed, the latter
+// will replace the former. A Tag overwrites all former values and typically
+// only makes sense as the first argument. The resulting tag is returned after
+// canonicalizing using the Default CanonType. If one or more errors are
+// encountered, one of the errors is returned.
+func Compose(part ...interface{}) (t Tag, err error) {
+ return Default.Compose(part...)
+}
+
+// Compose creates a Tag from individual parts, which may be of type Tag, Base,
+// Script, Region, Variant, []Variant, Extension, []Extension or error. If a
+// Base, Script or Region or slice of type Variant or Extension is passed more
+// than once, the latter will overwrite the former. Variants and Extensions are
+// accumulated, but if two extensions of the same type are passed, the latter
+// will replace the former. A Tag overwrites all former values and typically
+// only makes sense as the first argument. The resulting tag is returned after
+// canonicalizing using CanonType c. If one or more errors are encountered,
+// one of the errors is returned.
+func (c CanonType) Compose(part ...interface{}) (t Tag, err error) {
+ var b builder
+ if err = b.update(part...); err != nil {
+ return und, err
+ }
+ t, _ = b.tag.canonicalize(c)
+
+ if len(b.ext) > 0 || len(b.variant) > 0 {
+ sort.Sort(sortVariant(b.variant))
+ sort.Strings(b.ext)
+ if b.private != "" {
+ b.ext = append(b.ext, b.private)
+ }
+ n := maxCoreSize + tokenLen(b.variant...) + tokenLen(b.ext...)
+ buf := make([]byte, n)
+ p := t.genCoreBytes(buf)
+ t.pVariant = byte(p)
+ p += appendTokens(buf[p:], b.variant...)
+ t.pExt = uint16(p)
+ p += appendTokens(buf[p:], b.ext...)
+ t.str = string(buf[:p])
+ } else if b.private != "" {
+ t.str = b.private
+ t.remakeString()
+ }
+ return
+}
+
+type builder struct {
+ tag Tag
+
+ private string // the x extension
+ ext []string
+ variant []string
+
+ err error
+}
+
+func (b *builder) addExt(e string) {
+ if e == "" {
+ } else if e[0] == 'x' {
+ b.private = e
+ } else {
+ b.ext = append(b.ext, e)
+ }
+}
+
+var errInvalidArgument = errors.New("invalid Extension or Variant")
+
+func (b *builder) update(part ...interface{}) (err error) {
+ replace := func(l *[]string, s string, eq func(a, b string) bool) bool {
+ if s == "" {
+ b.err = errInvalidArgument
+ return true
+ }
+ for i, v := range *l {
+ if eq(v, s) {
+ (*l)[i] = s
+ return true
+ }
+ }
+ return false
+ }
+ for _, x := range part {
+ switch v := x.(type) {
+ case Tag:
+ b.tag.lang = v.lang
+ b.tag.region = v.region
+ b.tag.script = v.script
+ if v.str != "" {
+ b.variant = nil
+ for x, s := "", v.str[v.pVariant:v.pExt]; s != ""; {
+ x, s = nextToken(s)
+ b.variant = append(b.variant, x)
+ }
+ b.ext, b.private = nil, ""
+ for i, e := int(v.pExt), ""; i < len(v.str); {
+ i, e = getExtension(v.str, i)
+ b.addExt(e)
+ }
+ }
+ case Base:
+ b.tag.lang = v.langID
+ case Script:
+ b.tag.script = v.scriptID
+ case Region:
+ b.tag.region = v.regionID
+ case Variant:
+ if !replace(&b.variant, v.variant, func(a, b string) bool { return a == b }) {
+ b.variant = append(b.variant, v.variant)
+ }
+ case Extension:
+ if !replace(&b.ext, v.s, func(a, b string) bool { return a[0] == b[0] }) {
+ b.addExt(v.s)
+ }
+ case []Variant:
+ b.variant = nil
+ for _, x := range v {
+ b.update(x)
+ }
+ case []Extension:
+ b.ext, b.private = nil, ""
+ for _, e := range v {
+ b.update(e)
+ }
+ // TODO: support parsing of raw strings based on morphology or just extensions?
+ case error:
+ err = v
+ }
+ }
+ return
+}
+
+func tokenLen(token ...string) (n int) {
+ for _, t := range token {
+ n += len(t) + 1
+ }
+ return
+}
+
+func appendTokens(b []byte, token ...string) int {
+ p := 0
+ for _, t := range token {
+ b[p] = '-'
+ copy(b[p+1:], t)
+ p += 1 + len(t)
+ }
+ return p
+}
+
+type sortVariant []string
+
+func (s sortVariant) Len() int {
+ return len(s)
+}
+
+func (s sortVariant) Swap(i, j int) {
+ s[j], s[i] = s[i], s[j]
+}
+
+func (s sortVariant) Less(i, j int) bool {
+ return variantIndex[s[i]] < variantIndex[s[j]]
+}
+
+func findExt(list []string, x byte) int {
+ for i, e := range list {
+ if e[0] == x {
+ return i
+ }
+ }
+ return -1
+}
+
+// getExtension returns the name, body and end position of the extension.
+func getExtension(s string, p int) (end int, ext string) {
+ if s[p] == '-' {
+ p++
+ }
+ if s[p] == 'x' {
+ return len(s), s[p:]
+ }
+ end = nextExtension(s, p)
+ return end, s[p:end]
+}
+
+// nextExtension finds the next extension within the string, searching
+// for the -<char>- pattern from position p.
+// In the fast majority of cases, language tags will have at most
+// one extension and extensions tend to be small.
+func nextExtension(s string, p int) int {
+ for n := len(s) - 3; p < n; {
+ if s[p] == '-' {
+ if s[p+2] == '-' {
+ return p
+ }
+ p += 3
+ } else {
+ p++
+ }
+ }
+ return len(s)
+}
+
+var errInvalidWeight = errors.New("ParseAcceptLanguage: invalid weight")
+
+// ParseAcceptLanguage parses the contents of an Accept-Language header as
+// defined in http://www.ietf.org/rfc/rfc2616.txt and returns a list of Tags and
+// a list of corresponding quality weights. It is more permissive than RFC 2616
+// and may return non-nil slices even if the input is not valid.
+// The Tags will be sorted by highest weight first and then by first occurrence.
+// Tags with a weight of zero will be dropped. An error will be returned if the
+// input could not be parsed.
+func ParseAcceptLanguage(s string) (tag []Tag, q []float32, err error) {
+ var entry string
+ for s != "" {
+ if entry, s = split(s, ','); entry == "" {
+ continue
+ }
+
+ entry, weight := split(entry, ';')
+
+ // Scan the language.
+ t, err := Parse(entry)
+ if err != nil {
+ id, ok := acceptFallback[entry]
+ if !ok {
+ return nil, nil, err
+ }
+ t = Tag{lang: id}
+ }
+
+ // Scan the optional weight.
+ w := 1.0
+ if weight != "" {
+ weight = consume(weight, 'q')
+ weight = consume(weight, '=')
+ // consume returns the empty string when a token could not be
+ // consumed, resulting in an error for ParseFloat.
+ if w, err = strconv.ParseFloat(weight, 32); err != nil {
+ return nil, nil, errInvalidWeight
+ }
+ // Drop tags with a quality weight of 0.
+ if w <= 0 {
+ continue
+ }
+ }
+
+ tag = append(tag, t)
+ q = append(q, float32(w))
+ }
+ sortStable(&tagSort{tag, q})
+ return tag, q, nil
+}
+
+// consume removes a leading token c from s and returns the result or the empty
+// string if there is no such token.
+func consume(s string, c byte) string {
+ if s == "" || s[0] != c {
+ return ""
+ }
+ return strings.TrimSpace(s[1:])
+}
+
+func split(s string, c byte) (head, tail string) {
+ if i := strings.IndexByte(s, c); i >= 0 {
+ return strings.TrimSpace(s[:i]), strings.TrimSpace(s[i+1:])
+ }
+ return strings.TrimSpace(s), ""
+}
+
+// Add hack mapping to deal with a small number of cases that that occur
+// in Accept-Language (with reasonable frequency).
+var acceptFallback = map[string]langID{
+ "english": _en,
+ "deutsch": _de,
+ "italian": _it,
+ "french": _fr,
+ "*": _mul, // defined in the spec to match all languages.
+}
+
+type tagSort struct {
+ tag []Tag
+ q []float32
+}
+
+func (s *tagSort) Len() int {
+ return len(s.q)
+}
+
+func (s *tagSort) Less(i, j int) bool {
+ return s.q[i] > s.q[j]
+}
+
+func (s *tagSort) Swap(i, j int) {
+ s.tag[i], s.tag[j] = s.tag[j], s.tag[i]
+ s.q[i], s.q[j] = s.q[j], s.q[i]
+}
diff --git a/language/internal/parse_test.go b/language/internal/parse_test.go
new file mode 100644
index 0000000..9b40eb4
--- /dev/null
+++ b/language/internal/parse_test.go
@@ -0,0 +1,517 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language
+
+import (
+ "bytes"
+ "strings"
+ "testing"
+
+ "golang.org/x/text/internal/tag"
+)
+
+type scanTest struct {
+ ok bool // true if scanning does not result in an error
+ in string
+ tok []string // the expected tokens
+}
+
+var tests = []scanTest{
+ {true, "", []string{}},
+ {true, "1", []string{"1"}},
+ {true, "en", []string{"en"}},
+ {true, "root", []string{"root"}},
+ {true, "maxchars", []string{"maxchars"}},
+ {false, "bad/", []string{}},
+ {false, "morethan8", []string{}},
+ {false, "-", []string{}},
+ {false, "----", []string{}},
+ {false, "_", []string{}},
+ {true, "en-US", []string{"en", "US"}},
+ {true, "en_US", []string{"en", "US"}},
+ {false, "en-US-", []string{"en", "US"}},
+ {false, "en-US--", []string{"en", "US"}},
+ {false, "en-US---", []string{"en", "US"}},
+ {false, "en--US", []string{"en", "US"}},
+ {false, "-en-US", []string{"en", "US"}},
+ {false, "-en--US-", []string{"en", "US"}},
+ {false, "-en--US-", []string{"en", "US"}},
+ {false, "en-.-US", []string{"en", "US"}},
+ {false, ".-en--US-.", []string{"en", "US"}},
+ {false, "en-u.-US", []string{"en", "US"}},
+ {true, "en-u1-US", []string{"en", "u1", "US"}},
+ {true, "maxchar1_maxchar2-maxchar3", []string{"maxchar1", "maxchar2", "maxchar3"}},
+ {false, "moreThan8-moreThan8-e", []string{"e"}},
+}
+
+func TestScan(t *testing.T) {
+ for i, tt := range tests {
+ scan := makeScannerString(tt.in)
+ for j := 0; !scan.done; j++ {
+ if j >= len(tt.tok) {
+ t.Errorf("%d: extra token %q", i, scan.token)
+ } else if tag.Compare(tt.tok[j], scan.token) != 0 {
+ t.Errorf("%d: token %d: found %q; want %q", i, j, scan.token, tt.tok[j])
+ break
+ }
+ scan.scan()
+ }
+ if s := strings.Join(tt.tok, "-"); tag.Compare(s, bytes.Replace(scan.b, b("_"), b("-"), -1)) != 0 {
+ t.Errorf("%d: input: found %q; want %q", i, scan.b, s)
+ }
+ if (scan.err == nil) != tt.ok {
+ t.Errorf("%d: ok: found %v; want %v", i, scan.err == nil, tt.ok)
+ }
+ }
+}
+
+func TestAcceptMinSize(t *testing.T) {
+ for i, tt := range tests {
+ // count number of successive tokens with a minimum size.
+ for sz := 1; sz <= 8; sz++ {
+ scan := makeScannerString(tt.in)
+ scan.end, scan.next = 0, 0
+ end := scan.acceptMinSize(sz)
+ n := 0
+ for i := 0; i < len(tt.tok) && len(tt.tok[i]) >= sz; i++ {
+ n += len(tt.tok[i])
+ if i > 0 {
+ n++
+ }
+ }
+ if end != n {
+ t.Errorf("%d:%d: found len %d; want %d", i, sz, end, n)
+ }
+ }
+ }
+}
+
+type parseTest struct {
+ i int // the index of this test
+ in string
+ lang, script, region string
+ variants, ext string
+ extList []string // only used when more than one extension is present
+ invalid bool
+ rewrite bool // special rewrite not handled by parseTag
+ changed bool // string needed to be reformatted
+}
+
+func parseTests() []parseTest {
+ tests := []parseTest{
+ {in: "root", lang: "und"},
+ {in: "und", lang: "und"},
+ {in: "en", lang: "en"},
+ {in: "xy", lang: "und", invalid: true},
+ {in: "en-ZY", lang: "en", invalid: true},
+ {in: "gsw", lang: "gsw"},
+ {in: "sr_Latn", lang: "sr", script: "Latn"},
+ {in: "af-Arab", lang: "af", script: "Arab"},
+ {in: "nl-BE", lang: "nl", region: "BE"},
+ {in: "es-419", lang: "es", region: "419"},
+ {in: "und-001", lang: "und", region: "001"},
+ {in: "de-latn-be", lang: "de", script: "Latn", region: "BE"},
+ // Variants
+ {in: "de-1901", lang: "de", variants: "1901"},
+ // Accept with unsuppressed script.
+ {in: "de-Latn-1901", lang: "de", script: "Latn", variants: "1901"},
+ // Specialized.
+ {in: "sl-rozaj", lang: "sl", variants: "rozaj"},
+ {in: "sl-rozaj-lipaw", lang: "sl", variants: "rozaj-lipaw"},
+ {in: "sl-rozaj-biske", lang: "sl", variants: "rozaj-biske"},
+ {in: "sl-rozaj-biske-1994", lang: "sl", variants: "rozaj-biske-1994"},
+ {in: "sl-rozaj-1994", lang: "sl", variants: "rozaj-1994"},
+ // Maximum number of variants while adhering to prefix rules.
+ {in: "sl-rozaj-biske-1994-alalc97-fonipa-fonupa-fonxsamp", lang: "sl", variants: "rozaj-biske-1994-alalc97-fonipa-fonupa-fonxsamp"},
+
+ // Sorting.
+ {in: "sl-1994-biske-rozaj", lang: "sl", variants: "rozaj-biske-1994", changed: true},
+ {in: "sl-rozaj-biske-1994-alalc97-fonupa-fonipa-fonxsamp", lang: "sl", variants: "rozaj-biske-1994-alalc97-fonipa-fonupa-fonxsamp", changed: true},
+ {in: "nl-fonxsamp-alalc97-fonipa-fonupa", lang: "nl", variants: "alalc97-fonipa-fonupa-fonxsamp", changed: true},
+
+ // Duplicates variants are removed, but not an error.
+ {in: "nl-fonupa-fonupa", lang: "nl", variants: "fonupa"},
+
+ // Variants that do not have correct prefixes. We still accept these.
+ {in: "de-Cyrl-1901", lang: "de", script: "Cyrl", variants: "1901"},
+ {in: "sl-rozaj-lipaw-1994", lang: "sl", variants: "rozaj-lipaw-1994"},
+ {in: "sl-1994-biske-rozaj-1994-biske-rozaj", lang: "sl", variants: "rozaj-biske-1994", changed: true},
+ {in: "de-Cyrl-1901", lang: "de", script: "Cyrl", variants: "1901"},
+
+ // Invalid variant.
+ {in: "de-1902", lang: "de", variants: "", invalid: true},
+
+ {in: "EN_CYRL", lang: "en", script: "Cyrl"},
+ // private use and extensions
+ {in: "x-a-b-c-d", ext: "x-a-b-c-d"},
+ {in: "x_A.-B-C_D", ext: "x-b-c-d", invalid: true, changed: true},
+ {in: "x-aa-bbbb-cccccccc-d", ext: "x-aa-bbbb-cccccccc-d"},
+ {in: "en-c_cc-b-bbb-a-aaa", lang: "en", changed: true, extList: []string{"a-aaa", "b-bbb", "c-cc"}},
+ {in: "en-x_cc-b-bbb-a-aaa", lang: "en", ext: "x-cc-b-bbb-a-aaa", changed: true},
+ {in: "en-c_cc-b-bbb-a-aaa-x-x", lang: "en", changed: true, extList: []string{"a-aaa", "b-bbb", "c-cc", "x-x"}},
+ {in: "en-v-c", lang: "en", ext: "", invalid: true},
+ {in: "en-v-abcdefghi", lang: "en", ext: "", invalid: true},
+ {in: "en-v-abc-x", lang: "en", ext: "v-abc", invalid: true},
+ {in: "en-v-abc-x-", lang: "en", ext: "v-abc", invalid: true},
+ {in: "en-v-abc-w-x-xx", lang: "en", extList: []string{"v-abc", "x-xx"}, invalid: true, changed: true},
+ {in: "en-v-abc-w-y-yx", lang: "en", extList: []string{"v-abc", "y-yx"}, invalid: true, changed: true},
+ {in: "en-v-c-abc", lang: "en", ext: "c-abc", invalid: true, changed: true},
+ {in: "en-v-w-abc", lang: "en", ext: "w-abc", invalid: true, changed: true},
+ {in: "en-v-x-abc", lang: "en", ext: "x-abc", invalid: true, changed: true},
+ {in: "en-v-x-a", lang: "en", ext: "x-a", invalid: true, changed: true},
+ {in: "en-9-aa-0-aa-z-bb-x-a", lang: "en", extList: []string{"0-aa", "9-aa", "z-bb", "x-a"}, changed: true},
+ {in: "en-u-c", lang: "en", ext: "", invalid: true},
+ {in: "en-u-co-phonebk", lang: "en", ext: "u-co-phonebk"},
+ {in: "en-u-co-phonebk-ca", lang: "en", ext: "u-co-phonebk", invalid: true},
+ {in: "en-u-nu-arabic-co-phonebk-ca", lang: "en", ext: "u-co-phonebk-nu-arabic", invalid: true, changed: true},
+ {in: "en-u-nu-arabic-co-phonebk-ca-x", lang: "en", ext: "u-co-phonebk-nu-arabic", invalid: true, changed: true},
+ {in: "en-u-nu-arabic-co-phonebk-ca-s", lang: "en", ext: "u-co-phonebk-nu-arabic", invalid: true, changed: true},
+ {in: "en-u-nu-arabic-co-phonebk-ca-a12345678", lang: "en", ext: "u-co-phonebk-nu-arabic", invalid: true, changed: true},
+ {in: "en-u-co-phonebook", lang: "en", ext: "", invalid: true},
+ {in: "en-u-co-phonebook-cu-xau", lang: "en", ext: "u-cu-xau", invalid: true, changed: true},
+ {in: "en-Cyrl-u-co-phonebk", lang: "en", script: "Cyrl", ext: "u-co-phonebk"},
+ {in: "en-US-u-co-phonebk", lang: "en", region: "US", ext: "u-co-phonebk"},
+ {in: "en-US-u-co-phonebk-cu-xau", lang: "en", region: "US", ext: "u-co-phonebk-cu-xau"},
+ {in: "en-scotland-u-co-phonebk", lang: "en", variants: "scotland", ext: "u-co-phonebk"},
+ {in: "en-u-cu-xua-co-phonebk", lang: "en", ext: "u-co-phonebk-cu-xua", changed: true},
+ {in: "en-u-def-abc-cu-xua-co-phonebk", lang: "en", ext: "u-abc-def-co-phonebk-cu-xua", changed: true},
+ {in: "en-u-def-abc", lang: "en", ext: "u-abc-def", changed: true},
+ {in: "en-u-cu-xua-co-phonebk-a-cd", lang: "en", extList: []string{"a-cd", "u-co-phonebk-cu-xua"}, changed: true},
+ // Invalid "u" extension. Drop invalid parts.
+ {in: "en-u-cu-co-phonebk", lang: "en", extList: []string{"u-co-phonebk"}, invalid: true, changed: true},
+ {in: "en-u-cu-xau-co", lang: "en", extList: []string{"u-cu-xau"}, invalid: true},
+ // We allow duplicate keys as the LDML spec does not explicitly prohibit it.
+ // TODO: Consider eliminating duplicates and returning an error.
+ {in: "en-u-cu-xau-co-phonebk-cu-xau", lang: "en", ext: "u-co-phonebk-cu-xau-cu-xau", changed: true},
+ {in: "en-t-en-Cyrl-NL-fonipa", lang: "en", ext: "t-en-cyrl-nl-fonipa", changed: true},
+ {in: "en-t-en-Cyrl-NL-fonipa-t0-abc-def", lang: "en", ext: "t-en-cyrl-nl-fonipa-t0-abc-def", changed: true},
+ {in: "en-t-t0-abcd", lang: "en", ext: "t-t0-abcd"},
+ // Not necessary to have changed here.
+ {in: "en-t-nl-abcd", lang: "en", ext: "t-nl", invalid: true},
+ {in: "en-t-nl-latn", lang: "en", ext: "t-nl-latn"},
+ {in: "en-t-t0-abcd-x-a", lang: "en", extList: []string{"t-t0-abcd", "x-a"}},
+ // invalid
+ {in: "", lang: "und", invalid: true},
+ {in: "-", lang: "und", invalid: true},
+ {in: "x", lang: "und", invalid: true},
+ {in: "x-", lang: "und", invalid: true},
+ {in: "x--", lang: "und", invalid: true},
+ {in: "a-a-b-c-d", lang: "und", invalid: true},
+ {in: "en-", lang: "en", invalid: true},
+ {in: "enne-", lang: "und", invalid: true},
+ {in: "en.", lang: "und", invalid: true},
+ {in: "en.-latn", lang: "und", invalid: true},
+ {in: "en.-en", lang: "en", invalid: true},
+ {in: "x-a-tooManyChars-c-d", ext: "x-a-c-d", invalid: true, changed: true},
+ {in: "a-tooManyChars-c-d", lang: "und", invalid: true},
+ // TODO: check key-value validity
+ // { in: "en-u-cu-xd", lang: "en", ext: "u-cu-xd", invalid: true },
+ {in: "en-t-abcd", lang: "en", invalid: true},
+ {in: "en-Latn-US-en", lang: "en", script: "Latn", region: "US", invalid: true},
+ // rewrites (more tests in TestGrandfathered)
+ {in: "zh-min-nan", lang: "nan"},
+ {in: "zh-yue", lang: "yue"},
+ {in: "zh-xiang", lang: "hsn", rewrite: true},
+ {in: "zh-guoyu", lang: "cmn", rewrite: true},
+ {in: "iw", lang: "iw"},
+ {in: "sgn-BE-FR", lang: "sfb", rewrite: true},
+ {in: "i-klingon", lang: "tlh", rewrite: true},
+ }
+ for i, tt := range tests {
+ tests[i].i = i
+ if tt.extList != nil {
+ tests[i].ext = strings.Join(tt.extList, "-")
+ }
+ if tt.ext != "" && tt.extList == nil {
+ tests[i].extList = []string{tt.ext}
+ }
+ }
+ return tests
+}
+
+func TestParseExtensions(t *testing.T) {
+ for i, tt := range parseTests() {
+ if tt.ext == "" || tt.rewrite {
+ continue
+ }
+ scan := makeScannerString(tt.in)
+ if len(scan.b) > 1 && scan.b[1] != '-' {
+ scan.end = nextExtension(string(scan.b), 0)
+ scan.next = scan.end + 1
+ scan.scan()
+ }
+ start := scan.start
+ scan.toLower(start, len(scan.b))
+ parseExtensions(&scan)
+ ext := string(scan.b[start:])
+ if ext != tt.ext {
+ t.Errorf("%d(%s): ext was %v; want %v", i, tt.in, ext, tt.ext)
+ }
+ if changed := !strings.HasPrefix(tt.in[start:], ext); changed != tt.changed {
+ t.Errorf("%d(%s): changed was %v; want %v", i, tt.in, changed, tt.changed)
+ }
+ }
+}
+
+// partChecks runs checks for each part by calling the function returned by f.
+func partChecks(t *testing.T, f func(*parseTest) (Tag, bool)) {
+ for i, tt := range parseTests() {
+ tag, skip := f(&tt)
+ if skip {
+ continue
+ }
+ if l, _ := getLangID(b(tt.lang)); l != tag.lang {
+ t.Errorf("%d: lang was %q; want %q", i, tag.lang, l)
+ }
+ if sc, _ := getScriptID(script, b(tt.script)); sc != tag.script {
+ t.Errorf("%d: script was %q; want %q", i, tag.script, sc)
+ }
+ if r, _ := getRegionID(b(tt.region)); r != tag.region {
+ t.Errorf("%d: region was %q; want %q", i, tag.region, r)
+ }
+ if tag.str == "" {
+ continue
+ }
+ p := int(tag.pVariant)
+ if p < int(tag.pExt) {
+ p++
+ }
+ if s, g := tag.str[p:tag.pExt], tt.variants; s != g {
+ t.Errorf("%d: variants was %q; want %q", i, s, g)
+ }
+ p = int(tag.pExt)
+ if p > 0 && p < len(tag.str) {
+ p++
+ }
+ if s, g := (tag.str)[p:], tt.ext; s != g {
+ t.Errorf("%d: extensions were %q; want %q", i, s, g)
+ }
+ }
+}
+
+func TestParseTag(t *testing.T) {
+ partChecks(t, func(tt *parseTest) (id Tag, skip bool) {
+ if strings.HasPrefix(tt.in, "x-") || tt.rewrite {
+ return Tag{}, true
+ }
+ scan := makeScannerString(tt.in)
+ id, end := parseTag(&scan)
+ id.str = string(scan.b[:end])
+ tt.ext = ""
+ tt.extList = []string{}
+ return id, false
+ })
+}
+
+func TestParse(t *testing.T) {
+ partChecks(t, func(tt *parseTest) (id Tag, skip bool) {
+ id, err := Raw.Parse(tt.in)
+ ext := ""
+ if id.str != "" {
+ if strings.HasPrefix(id.str, "x-") {
+ ext = id.str
+ } else if int(id.pExt) < len(id.str) && id.pExt > 0 {
+ ext = id.str[id.pExt+1:]
+ }
+ }
+ if tag, _ := Raw.Parse(id.String()); tag.String() != id.String() {
+ t.Errorf("%d:%s: reparse was %q; want %q", tt.i, tt.in, id.String(), tag.String())
+ }
+ if ext != tt.ext {
+ t.Errorf("%d:%s: ext was %q; want %q", tt.i, tt.in, ext, tt.ext)
+ }
+ changed := id.str != "" && !strings.HasPrefix(tt.in, id.str)
+ if changed != tt.changed {
+ t.Errorf("%d:%s: changed was %v; want %v", tt.i, tt.in, changed, tt.changed)
+ }
+ if (err != nil) != tt.invalid {
+ t.Errorf("%d:%s: invalid was %v; want %v. Error: %v", tt.i, tt.in, err != nil, tt.invalid, err)
+ }
+ return id, false
+ })
+}
+
+func TestErrors(t *testing.T) {
+ mkInvalid := func(s string) error {
+ return mkErrInvalid([]byte(s))
+ }
+ tests := []struct {
+ in string
+ out error
+ }{
+ // invalid subtags.
+ {"ac", mkInvalid("ac")},
+ {"AC", mkInvalid("ac")},
+ {"aa-Uuuu", mkInvalid("Uuuu")},
+ {"aa-AB", mkInvalid("AB")},
+ // ill-formed wins over invalid.
+ {"ac-u", errSyntax},
+ {"ac-u-ca", errSyntax},
+ {"ac-u-ca-co-pinyin", errSyntax},
+ {"noob", errSyntax},
+ }
+ for _, tt := range tests {
+ _, err := Parse(tt.in)
+ if err != tt.out {
+ t.Errorf("%s: was %q; want %q", tt.in, err, tt.out)
+ }
+ }
+}
+
+func TestCompose1(t *testing.T) {
+ partChecks(t, func(tt *parseTest) (id Tag, skip bool) {
+ l, _ := ParseBase(tt.lang)
+ s, _ := ParseScript(tt.script)
+ r, _ := ParseRegion(tt.region)
+ v := []Variant{}
+ for _, x := range strings.Split(tt.variants, "-") {
+ p, _ := ParseVariant(x)
+ v = append(v, p)
+ }
+ e := []Extension{}
+ for _, x := range tt.extList {
+ p, _ := ParseExtension(x)
+ e = append(e, p)
+ }
+ id, _ = Raw.Compose(l, s, r, v, e)
+ return id, false
+ })
+}
+
+func TestCompose2(t *testing.T) {
+ partChecks(t, func(tt *parseTest) (id Tag, skip bool) {
+ l, _ := ParseBase(tt.lang)
+ s, _ := ParseScript(tt.script)
+ r, _ := ParseRegion(tt.region)
+ p := []interface{}{l, s, r, s, r, l}
+ for _, x := range strings.Split(tt.variants, "-") {
+ v, _ := ParseVariant(x)
+ p = append(p, v)
+ }
+ for _, x := range tt.extList {
+ e, _ := ParseExtension(x)
+ p = append(p, e)
+ }
+ id, _ = Raw.Compose(p...)
+ return id, false
+ })
+}
+
+func TestCompose3(t *testing.T) {
+ partChecks(t, func(tt *parseTest) (id Tag, skip bool) {
+ id, _ = Raw.Parse(tt.in)
+ id, _ = Raw.Compose(id)
+ return id, false
+ })
+}
+
+func mk(s string) Tag {
+ return Raw.Make(s)
+}
+
+func TestParseAcceptLanguage(t *testing.T) {
+ type res struct {
+ t Tag
+ q float32
+ }
+ en := []res{{mk("en"), 1.0}}
+ tests := []struct {
+ out []res
+ in string
+ ok bool
+ }{
+ {en, "en", true},
+ {en, " en", true},
+ {en, "en ", true},
+ {en, " en ", true},
+ {en, "en,", true},
+ {en, ",en", true},
+ {en, ",,,en,,,", true},
+ {en, ",en;q=1", true},
+
+ // We allow an empty input, contrary to spec.
+ {nil, "", true},
+ {[]res{{mk("aa"), 1}}, "aa;", true}, // allow unspecified weight
+
+ // errors
+ {nil, ";", false},
+ {nil, "$", false},
+ {nil, "e;", false},
+ {nil, "x;", false},
+ {nil, "x", false},
+ {nil, "ac", false}, // non-existing language
+ {nil, "aa;q", false},
+ {nil, "aa;q=", false},
+ {nil, "aa;q=.", false},
+
+ // odd fallbacks
+ {
+ []res{{mk("en"), 0.1}},
+ " english ;q=.1",
+ true,
+ },
+ {
+ []res{{mk("it"), 1.0}, {mk("de"), 1.0}, {mk("fr"), 1.0}},
+ " italian, deutsch, french",
+ true,
+ },
+
+ // lists
+ {
+ []res{{mk("en"), 0.1}},
+ "en;q=.1",
+ true,
+ },
+ {
+ []res{{mk("mul"), 1.0}},
+ "*",
+ true,
+ },
+ {
+ []res{{mk("en"), 1.0}, {mk("de"), 1.0}},
+ "en,de",
+ true,
+ },
+ {
+ []res{{mk("en"), 1.0}, {mk("de"), .5}},
+ "en,de;q=0.5",
+ true,
+ },
+ {
+ []res{{mk("de"), 0.8}, {mk("en"), 0.5}},
+ " en ; q = 0.5 , , de;q=0.8",
+ true,
+ },
+ {
+ []res{{mk("en"), 1.0}, {mk("de"), 1.0}, {mk("fr"), 1.0}, {mk("tlh"), 1.0}},
+ "en,de,fr,i-klingon",
+ true,
+ },
+ // sorting
+ {
+ []res{{mk("tlh"), 0.4}, {mk("de"), 0.2}, {mk("fr"), 0.2}, {mk("en"), 0.1}},
+ "en;q=0.1,de;q=0.2,fr;q=0.2,i-klingon;q=0.4",
+ true,
+ },
+ // dropping
+ {
+ []res{{mk("fr"), 0.2}, {mk("en"), 0.1}},
+ "en;q=0.1,de;q=0,fr;q=0.2,i-klingon;q=0.0",
+ true,
+ },
+ }
+ for i, tt := range tests {
+ tags, qs, e := ParseAcceptLanguage(tt.in)
+ if e == nil != tt.ok {
+ t.Errorf("%d:%s:err: was %v; want %v", i, tt.in, e == nil, tt.ok)
+ }
+ for j, tag := range tags {
+ if out := tt.out[j]; !tag.equalTags(out.t) || qs[j] != out.q {
+ t.Errorf("%d:%s: was %s, %1f; want %s, %1f", i, tt.in, tag, qs[j], out.t, out.q)
+ break
+ }
+ }
+ }
+}
diff --git a/language/internal/tables.go b/language/internal/tables.go
new file mode 100644
index 0000000..b738d45
--- /dev/null
+++ b/language/internal/tables.go
@@ -0,0 +1,3686 @@
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+package language
+
+import "golang.org/x/text/internal/tag"
+
+// CLDRVersion is the CLDR version from which the tables in this package are derived.
+const CLDRVersion = "32"
+
+const numLanguages = 8665
+
+const numScripts = 242
+
+const numRegions = 357
+
+type fromTo struct {
+ from uint16
+ to uint16
+}
+
+const nonCanonicalUnd = 1201
+const (
+ _af = 22
+ _am = 39
+ _ar = 58
+ _az = 88
+ _bg = 126
+ _bn = 165
+ _ca = 215
+ _cs = 250
+ _da = 257
+ _de = 269
+ _el = 310
+ _en = 313
+ _es = 318
+ _et = 320
+ _fa = 328
+ _fi = 337
+ _fil = 339
+ _fr = 350
+ _gu = 420
+ _he = 444
+ _hi = 446
+ _hr = 465
+ _hu = 469
+ _hy = 471
+ _id = 481
+ _is = 504
+ _it = 505
+ _ja = 512
+ _ka = 528
+ _kk = 578
+ _km = 586
+ _kn = 593
+ _ko = 596
+ _ky = 650
+ _lo = 696
+ _lt = 704
+ _lv = 711
+ _mk = 767
+ _ml = 772
+ _mn = 779
+ _mo = 784
+ _mr = 795
+ _ms = 799
+ _mul = 806
+ _my = 817
+ _nb = 839
+ _ne = 849
+ _nl = 871
+ _no = 879
+ _pa = 925
+ _pl = 947
+ _pt = 960
+ _ro = 988
+ _ru = 994
+ _sh = 1031
+ _si = 1036
+ _sk = 1042
+ _sl = 1046
+ _sq = 1073
+ _sr = 1074
+ _sv = 1092
+ _sw = 1093
+ _ta = 1104
+ _te = 1121
+ _th = 1131
+ _tl = 1146
+ _tn = 1152
+ _tr = 1162
+ _uk = 1198
+ _ur = 1204
+ _uz = 1212
+ _vi = 1219
+ _zh = 1321
+ _zu = 1327
+ _jbo = 515
+ _ami = 1650
+ _bnn = 2357
+ _hak = 438
+ _tlh = 14467
+ _lb = 661
+ _nv = 899
+ _pwn = 12055
+ _tao = 14188
+ _tay = 14198
+ _tsu = 14662
+ _nn = 874
+ _sfb = 13629
+ _vgt = 15701
+ _sgg = 13660
+ _cmn = 3007
+ _nan = 835
+ _hsn = 467
+)
+
+const langPrivateStart = 0x2f72
+
+const langPrivateEnd = 0x3179
+
+// lang holds an alphabetically sorted list of ISO-639 language identifiers.
+// All entries are 4 bytes. The index of the identifier (divided by 4) is the language tag.
+// For 2-byte language identifiers, the two successive bytes have the following meaning:
+// - if the first letter of the 2- and 3-letter ISO codes are the same:
+// the second and third letter of the 3-letter ISO code.
+// - otherwise: a 0 and a by 2 bits right-shifted index into altLangISO3.
+// For 3-byte language identifiers the 4th byte is 0.
+const lang tag.Index = "" + // Size: 5324 bytes
+ "---\x00aaaraai\x00aak\x00aau\x00abbkabi\x00abq\x00abr\x00abt\x00aby\x00a" +
+ "cd\x00ace\x00ach\x00ada\x00ade\x00adj\x00ady\x00adz\x00aeveaeb\x00aey" +
+ "\x00affragc\x00agd\x00agg\x00agm\x00ago\x00agq\x00aha\x00ahl\x00aho\x00a" +
+ "jg\x00akkaakk\x00ala\x00ali\x00aln\x00alt\x00ammhamm\x00amn\x00amo\x00am" +
+ "p\x00anrganc\x00ank\x00ann\x00any\x00aoj\x00aom\x00aoz\x00apc\x00apd\x00" +
+ "ape\x00apr\x00aps\x00apz\x00arraarc\x00arh\x00arn\x00aro\x00arq\x00ars" +
+ "\x00ary\x00arz\x00assmasa\x00ase\x00asg\x00aso\x00ast\x00ata\x00atg\x00a" +
+ "tj\x00auy\x00avvaavl\x00avn\x00avt\x00avu\x00awa\x00awb\x00awo\x00awx" +
+ "\x00ayymayb\x00azzebaakbal\x00ban\x00bap\x00bar\x00bas\x00bav\x00bax\x00" +
+ "bba\x00bbb\x00bbc\x00bbd\x00bbj\x00bbp\x00bbr\x00bcf\x00bch\x00bci\x00bc" +
+ "m\x00bcn\x00bco\x00bcq\x00bcu\x00bdd\x00beelbef\x00beh\x00bej\x00bem\x00" +
+ "bet\x00bew\x00bex\x00bez\x00bfd\x00bfq\x00bft\x00bfy\x00bgulbgc\x00bgn" +
+ "\x00bgx\x00bhihbhb\x00bhg\x00bhi\x00bhk\x00bhl\x00bho\x00bhy\x00biisbib" +
+ "\x00big\x00bik\x00bim\x00bin\x00bio\x00biq\x00bjh\x00bji\x00bjj\x00bjn" +
+ "\x00bjo\x00bjr\x00bjt\x00bjz\x00bkc\x00bkm\x00bkq\x00bku\x00bkv\x00blt" +
+ "\x00bmambmh\x00bmk\x00bmq\x00bmu\x00bnenbng\x00bnm\x00bnp\x00boodboj\x00" +
+ "bom\x00bon\x00bpy\x00bqc\x00bqi\x00bqp\x00bqv\x00brrebra\x00brh\x00brx" +
+ "\x00brz\x00bsosbsj\x00bsq\x00bss\x00bst\x00bto\x00btt\x00btv\x00bua\x00b" +
+ "uc\x00bud\x00bug\x00buk\x00bum\x00buo\x00bus\x00buu\x00bvb\x00bwd\x00bwr" +
+ "\x00bxh\x00bye\x00byn\x00byr\x00bys\x00byv\x00byx\x00bza\x00bze\x00bzf" +
+ "\x00bzh\x00bzw\x00caatcan\x00cbj\x00cch\x00ccp\x00ceheceb\x00cfa\x00cgg" +
+ "\x00chhachk\x00chm\x00cho\x00chp\x00chr\x00cja\x00cjm\x00cjv\x00ckb\x00c" +
+ "kl\x00cko\x00cky\x00cla\x00cme\x00cmg\x00cooscop\x00cps\x00crrecrh\x00cr" +
+ "j\x00crk\x00crl\x00crm\x00crs\x00csescsb\x00csw\x00ctd\x00cuhucvhvcyymda" +
+ "andad\x00daf\x00dag\x00dah\x00dak\x00dar\x00dav\x00dbd\x00dbq\x00dcc\x00" +
+ "ddn\x00deeuded\x00den\x00dga\x00dgh\x00dgi\x00dgl\x00dgr\x00dgz\x00dia" +
+ "\x00dje\x00dnj\x00dob\x00doi\x00dop\x00dow\x00dri\x00drs\x00dsb\x00dtm" +
+ "\x00dtp\x00dts\x00dty\x00dua\x00duc\x00dud\x00dug\x00dvivdva\x00dww\x00d" +
+ "yo\x00dyu\x00dzzodzg\x00ebu\x00eeweefi\x00egl\x00egy\x00eka\x00eky\x00el" +
+ "llema\x00emi\x00enngenn\x00enq\x00eopoeri\x00es\x00\x05esu\x00etstetr" +
+ "\x00ett\x00etu\x00etx\x00euusewo\x00ext\x00faasfaa\x00fab\x00fag\x00fai" +
+ "\x00fan\x00ffulffi\x00ffm\x00fiinfia\x00fil\x00fit\x00fjijflr\x00fmp\x00" +
+ "foaofod\x00fon\x00for\x00fpe\x00fqs\x00frrafrc\x00frp\x00frr\x00frs\x00f" +
+ "ub\x00fud\x00fue\x00fuf\x00fuh\x00fuq\x00fur\x00fuv\x00fuy\x00fvr\x00fyr" +
+ "ygalegaa\x00gaf\x00gag\x00gah\x00gaj\x00gam\x00gan\x00gaw\x00gay\x00gba" +
+ "\x00gbf\x00gbm\x00gby\x00gbz\x00gcr\x00gdlagde\x00gdn\x00gdr\x00geb\x00g" +
+ "ej\x00gel\x00gez\x00gfk\x00ggn\x00ghs\x00gil\x00gim\x00gjk\x00gjn\x00gju" +
+ "\x00gkn\x00gkp\x00gllgglk\x00gmm\x00gmv\x00gnrngnd\x00gng\x00god\x00gof" +
+ "\x00goi\x00gom\x00gon\x00gor\x00gos\x00got\x00grb\x00grc\x00grt\x00grw" +
+ "\x00gsw\x00guujgub\x00guc\x00gud\x00gur\x00guw\x00gux\x00guz\x00gvlvgvf" +
+ "\x00gvr\x00gvs\x00gwc\x00gwi\x00gwt\x00gyi\x00haauhag\x00hak\x00ham\x00h" +
+ "aw\x00haz\x00hbb\x00hdy\x00heebhhy\x00hiinhia\x00hif\x00hig\x00hih\x00hi" +
+ "l\x00hla\x00hlu\x00hmd\x00hmt\x00hnd\x00hne\x00hnj\x00hnn\x00hno\x00homo" +
+ "hoc\x00hoj\x00hot\x00hrrvhsb\x00hsn\x00htathuunhui\x00hyyehzerianaian" +
+ "\x00iar\x00iba\x00ibb\x00iby\x00ica\x00ich\x00idndidd\x00idi\x00idu\x00i" +
+ "eleife\x00igboigb\x00ige\x00iiiiijj\x00ikpkikk\x00ikt\x00ikw\x00ikx\x00i" +
+ "lo\x00imo\x00inndinh\x00iodoiou\x00iri\x00isslittaiukuiw\x00\x03iwm\x00i" +
+ "ws\x00izh\x00izi\x00japnjab\x00jam\x00jbo\x00jbu\x00jen\x00jgk\x00jgo" +
+ "\x00ji\x00\x06jib\x00jmc\x00jml\x00jra\x00jut\x00jvavjwavkaatkaa\x00kab" +
+ "\x00kac\x00kad\x00kai\x00kaj\x00kam\x00kao\x00kbd\x00kbm\x00kbp\x00kbq" +
+ "\x00kbx\x00kby\x00kcg\x00kck\x00kcl\x00kct\x00kde\x00kdh\x00kdl\x00kdt" +
+ "\x00kea\x00ken\x00kez\x00kfo\x00kfr\x00kfy\x00kgonkge\x00kgf\x00kgp\x00k" +
+ "ha\x00khb\x00khn\x00khq\x00khs\x00kht\x00khw\x00khz\x00kiikkij\x00kiu" +
+ "\x00kiw\x00kjuakjd\x00kjg\x00kjs\x00kjy\x00kkazkkc\x00kkj\x00klalkln\x00" +
+ "klq\x00klt\x00klx\x00kmhmkmb\x00kmh\x00kmo\x00kms\x00kmu\x00kmw\x00knank" +
+ "nf\x00knp\x00koorkoi\x00kok\x00kol\x00kos\x00koz\x00kpe\x00kpf\x00kpo" +
+ "\x00kpr\x00kpx\x00kqb\x00kqf\x00kqs\x00kqy\x00kraukrc\x00kri\x00krj\x00k" +
+ "rl\x00krs\x00kru\x00ksasksb\x00ksd\x00ksf\x00ksh\x00ksj\x00ksr\x00ktb" +
+ "\x00ktm\x00kto\x00kuurkub\x00kud\x00kue\x00kuj\x00kum\x00kun\x00kup\x00k" +
+ "us\x00kvomkvg\x00kvr\x00kvx\x00kw\x00\x01kwj\x00kwo\x00kxa\x00kxc\x00kxm" +
+ "\x00kxp\x00kxw\x00kxz\x00kyirkye\x00kyx\x00kzr\x00laatlab\x00lad\x00lag" +
+ "\x00lah\x00laj\x00las\x00lbtzlbe\x00lbu\x00lbw\x00lcm\x00lcp\x00ldb\x00l" +
+ "ed\x00lee\x00lem\x00lep\x00leq\x00leu\x00lez\x00lguglgg\x00liimlia\x00li" +
+ "d\x00lif\x00lig\x00lih\x00lij\x00lis\x00ljp\x00lki\x00lkt\x00lle\x00lln" +
+ "\x00lmn\x00lmo\x00lmp\x00lninlns\x00lnu\x00loaoloj\x00lok\x00lol\x00lor" +
+ "\x00los\x00loz\x00lrc\x00ltitltg\x00luublua\x00luo\x00luy\x00luz\x00lvav" +
+ "lwl\x00lzh\x00lzz\x00mad\x00maf\x00mag\x00mai\x00mak\x00man\x00mas\x00ma" +
+ "w\x00maz\x00mbh\x00mbo\x00mbq\x00mbu\x00mbw\x00mci\x00mcp\x00mcq\x00mcr" +
+ "\x00mcu\x00mda\x00mde\x00mdf\x00mdh\x00mdj\x00mdr\x00mdx\x00med\x00mee" +
+ "\x00mek\x00men\x00mer\x00met\x00meu\x00mfa\x00mfe\x00mfn\x00mfo\x00mfq" +
+ "\x00mglgmgh\x00mgl\x00mgo\x00mgp\x00mgy\x00mhahmhi\x00mhl\x00mirimif\x00" +
+ "min\x00mis\x00miw\x00mkkdmki\x00mkl\x00mkp\x00mkw\x00mlalmle\x00mlp\x00m" +
+ "ls\x00mmo\x00mmu\x00mmx\x00mnonmna\x00mnf\x00mni\x00mnw\x00moolmoa\x00mo" +
+ "e\x00moh\x00mos\x00mox\x00mpp\x00mps\x00mpt\x00mpx\x00mql\x00mrarmrd\x00" +
+ "mrj\x00mro\x00mssamtltmtc\x00mtf\x00mti\x00mtr\x00mua\x00mul\x00mur\x00m" +
+ "us\x00mva\x00mvn\x00mvy\x00mwk\x00mwr\x00mwv\x00mxc\x00mxm\x00myyamyk" +
+ "\x00mym\x00myv\x00myw\x00myx\x00myz\x00mzk\x00mzm\x00mzn\x00mzp\x00mzw" +
+ "\x00mzz\x00naaunac\x00naf\x00nah\x00nak\x00nan\x00nap\x00naq\x00nas\x00n" +
+ "bobnca\x00nce\x00ncf\x00nch\x00nco\x00ncu\x00nddendc\x00nds\x00neepneb" +
+ "\x00new\x00nex\x00nfr\x00ngdonga\x00ngb\x00ngl\x00nhb\x00nhe\x00nhw\x00n" +
+ "if\x00nii\x00nij\x00nin\x00niu\x00niy\x00niz\x00njo\x00nkg\x00nko\x00nll" +
+ "dnmg\x00nmz\x00nnnonnf\x00nnh\x00nnk\x00nnm\x00noornod\x00noe\x00non\x00" +
+ "nop\x00nou\x00nqo\x00nrblnrb\x00nsk\x00nsn\x00nso\x00nss\x00ntm\x00ntr" +
+ "\x00nui\x00nup\x00nus\x00nuv\x00nux\x00nvavnwb\x00nxq\x00nxr\x00nyyanym" +
+ "\x00nyn\x00nzi\x00occiogc\x00ojjiokr\x00okv\x00omrmong\x00onn\x00ons\x00" +
+ "opm\x00orrioro\x00oru\x00osssosa\x00ota\x00otk\x00ozm\x00paanpag\x00pal" +
+ "\x00pam\x00pap\x00pau\x00pbi\x00pcd\x00pcm\x00pdc\x00pdt\x00ped\x00peo" +
+ "\x00pex\x00pfl\x00phl\x00phn\x00pilipil\x00pip\x00pka\x00pko\x00plolpla" +
+ "\x00pms\x00png\x00pnn\x00pnt\x00pon\x00ppo\x00pra\x00prd\x00prg\x00psusp" +
+ "ss\x00ptorptp\x00puu\x00pwa\x00quuequc\x00qug\x00rai\x00raj\x00rao\x00rc" +
+ "f\x00rej\x00rel\x00res\x00rgn\x00rhg\x00ria\x00rif\x00rjs\x00rkt\x00rmoh" +
+ "rmf\x00rmo\x00rmt\x00rmu\x00rnunrna\x00rng\x00roonrob\x00rof\x00roo\x00r" +
+ "ro\x00rtm\x00ruusrue\x00rug\x00rw\x00\x04rwk\x00rwo\x00ryu\x00saansaf" +
+ "\x00sah\x00saq\x00sas\x00sat\x00sav\x00saz\x00sba\x00sbe\x00sbp\x00scrds" +
+ "ck\x00scl\x00scn\x00sco\x00scs\x00sdndsdc\x00sdh\x00semesef\x00seh\x00se" +
+ "i\x00ses\x00sgagsga\x00sgs\x00sgw\x00sgz\x00sh\x00\x02shi\x00shk\x00shn" +
+ "\x00shu\x00siinsid\x00sig\x00sil\x00sim\x00sjr\x00sklkskc\x00skr\x00sks" +
+ "\x00sllvsld\x00sli\x00sll\x00sly\x00smmosma\x00smi\x00smj\x00smn\x00smp" +
+ "\x00smq\x00sms\x00snnasnc\x00snk\x00snp\x00snx\x00sny\x00soomsok\x00soq" +
+ "\x00sou\x00soy\x00spd\x00spl\x00sps\x00sqqisrrpsrb\x00srn\x00srr\x00srx" +
+ "\x00ssswssd\x00ssg\x00ssy\x00stotstk\x00stq\x00suunsua\x00sue\x00suk\x00" +
+ "sur\x00sus\x00svweswwaswb\x00swc\x00swg\x00swp\x00swv\x00sxn\x00sxw\x00s" +
+ "yl\x00syr\x00szl\x00taamtaj\x00tal\x00tan\x00taq\x00tbc\x00tbd\x00tbf" +
+ "\x00tbg\x00tbo\x00tbw\x00tbz\x00tci\x00tcy\x00tdd\x00tdg\x00tdh\x00teelt" +
+ "ed\x00tem\x00teo\x00tet\x00tfi\x00tggktgc\x00tgo\x00tgu\x00thhathl\x00th" +
+ "q\x00thr\x00tiirtif\x00tig\x00tik\x00tim\x00tio\x00tiv\x00tkuktkl\x00tkr" +
+ "\x00tkt\x00tlgltlf\x00tlx\x00tly\x00tmh\x00tmy\x00tnsntnh\x00toontof\x00" +
+ "tog\x00toq\x00tpi\x00tpm\x00tpz\x00tqo\x00trurtru\x00trv\x00trw\x00tssot" +
+ "sd\x00tsf\x00tsg\x00tsj\x00tsw\x00ttatttd\x00tte\x00ttj\x00ttr\x00tts" +
+ "\x00ttt\x00tuh\x00tul\x00tum\x00tuq\x00tvd\x00tvl\x00tvu\x00twwitwh\x00t" +
+ "wq\x00txg\x00tyahtya\x00tyv\x00tzm\x00ubu\x00udm\x00ugiguga\x00ukkruli" +
+ "\x00umb\x00und\x00unr\x00unx\x00urrduri\x00urt\x00urw\x00usa\x00utr\x00u" +
+ "vh\x00uvl\x00uzzbvag\x00vai\x00van\x00veenvec\x00vep\x00viievic\x00viv" +
+ "\x00vls\x00vmf\x00vmw\x00voolvot\x00vro\x00vun\x00vut\x00walnwae\x00waj" +
+ "\x00wal\x00wan\x00war\x00wbp\x00wbq\x00wbr\x00wci\x00wer\x00wgi\x00whg" +
+ "\x00wib\x00wiu\x00wiv\x00wja\x00wji\x00wls\x00wmo\x00wnc\x00wni\x00wnu" +
+ "\x00woolwob\x00wos\x00wrs\x00wsk\x00wtm\x00wuu\x00wuv\x00wwa\x00xav\x00x" +
+ "bi\x00xcr\x00xes\x00xhhoxla\x00xlc\x00xld\x00xmf\x00xmn\x00xmr\x00xna" +
+ "\x00xnr\x00xog\x00xon\x00xpr\x00xrb\x00xsa\x00xsi\x00xsm\x00xsr\x00xwe" +
+ "\x00yam\x00yao\x00yap\x00yas\x00yat\x00yav\x00yay\x00yaz\x00yba\x00ybb" +
+ "\x00yby\x00yer\x00ygr\x00ygw\x00yiidyko\x00yle\x00ylg\x00yll\x00yml\x00y" +
+ "ooryon\x00yrb\x00yre\x00yrl\x00yss\x00yua\x00yue\x00yuj\x00yut\x00yuw" +
+ "\x00zahazag\x00zbl\x00zdj\x00zea\x00zgh\x00zhhozhx\x00zia\x00zlm\x00zmi" +
+ "\x00zne\x00zuulzxx\x00zza\x00\xff\xff\xff\xff"
+
+const langNoIndexOffset = 1330
+
+// langNoIndex is a bit vector of all 3-letter language codes that are not used as an index
+// in lookup tables. The language ids for these language codes are derived directly
+// from the letters and are not consecutive.
+// Size: 2197 bytes, 2197 elements
+var langNoIndex = [2197]uint8{
+ // Entry 0 - 3F
+ 0xff, 0xf8, 0xed, 0xfe, 0xeb, 0xd3, 0x3b, 0xd2,
+ 0xfb, 0xbf, 0x7a, 0xfa, 0x37, 0x1d, 0x3c, 0x57,
+ 0x6e, 0x97, 0x73, 0x38, 0xfb, 0xea, 0xbf, 0x70,
+ 0xad, 0x03, 0xff, 0xff, 0xcf, 0x05, 0x84, 0x62,
+ 0xe9, 0xbf, 0xfd, 0xbf, 0xbf, 0xf7, 0xfd, 0x77,
+ 0x0f, 0xff, 0xef, 0x6f, 0xff, 0xfb, 0xdf, 0xe2,
+ 0xc9, 0xf8, 0x7f, 0x7e, 0x4d, 0xb8, 0x0a, 0x6a,
+ 0x7c, 0xea, 0xe3, 0xfa, 0x7a, 0xbf, 0x67, 0xff,
+ // Entry 40 - 7F
+ 0xff, 0xff, 0xff, 0xdf, 0x2a, 0x54, 0x91, 0xc0,
+ 0x5d, 0xe3, 0x97, 0x14, 0x07, 0x20, 0xdd, 0xed,
+ 0x9f, 0x3f, 0xc9, 0x21, 0xf8, 0x3f, 0x94, 0x35,
+ 0x7c, 0x5f, 0xff, 0x5f, 0x8e, 0x6e, 0xdf, 0xff,
+ 0xff, 0xff, 0x55, 0x7c, 0xd3, 0xfd, 0xbf, 0xb5,
+ 0x7b, 0xdf, 0x7f, 0xf7, 0xca, 0xfe, 0xdb, 0xa3,
+ 0xa8, 0xff, 0x1f, 0x67, 0x7d, 0xeb, 0xef, 0xce,
+ 0xff, 0xff, 0x9f, 0xff, 0xb7, 0xef, 0xfe, 0xcf,
+ // Entry 80 - BF
+ 0xdb, 0xff, 0xf3, 0xcd, 0xfb, 0x2f, 0xff, 0xff,
+ 0xbb, 0xee, 0xf7, 0xbd, 0xdb, 0xff, 0x5f, 0xf7,
+ 0xfd, 0xf2, 0xfd, 0xff, 0x5e, 0x2f, 0x3b, 0xba,
+ 0x7e, 0xff, 0xff, 0xfe, 0xf7, 0xff, 0xdd, 0xff,
+ 0xfd, 0xdf, 0xfb, 0xfe, 0x9d, 0xb4, 0xd3, 0xff,
+ 0xef, 0xff, 0xdf, 0xf7, 0x7f, 0xb7, 0xfd, 0xd5,
+ 0xa5, 0x77, 0x40, 0xff, 0x9c, 0xc1, 0x41, 0x2c,
+ 0x08, 0x20, 0x41, 0x00, 0x50, 0x40, 0x00, 0x80,
+ // Entry C0 - FF
+ 0xfb, 0x4a, 0xf2, 0x9f, 0xb4, 0x42, 0x41, 0x96,
+ 0x1b, 0x14, 0x08, 0xf2, 0x2b, 0xe7, 0x17, 0x56,
+ 0x05, 0x7d, 0x0e, 0x1c, 0x37, 0x71, 0xf3, 0xef,
+ 0x97, 0xff, 0x5d, 0x38, 0x64, 0x08, 0x00, 0x10,
+ 0xbc, 0x85, 0xaf, 0xdf, 0xff, 0xf7, 0x73, 0x35,
+ 0x3e, 0x87, 0xc7, 0xdf, 0xff, 0x00, 0x81, 0x00,
+ 0xb0, 0x05, 0x80, 0x00, 0x00, 0x00, 0x00, 0x03,
+ 0x40, 0x00, 0x40, 0x92, 0x21, 0x50, 0xb1, 0x5d,
+ // Entry 100 - 13F
+ 0xfd, 0xdc, 0xbe, 0x5e, 0x00, 0x00, 0x02, 0x64,
+ 0x0d, 0x19, 0x41, 0xdf, 0x79, 0x22, 0x00, 0x00,
+ 0x00, 0x5e, 0x64, 0xdc, 0x24, 0xe5, 0xd9, 0xe3,
+ 0xfe, 0xff, 0xfd, 0xcb, 0x9f, 0x14, 0x01, 0x0c,
+ 0x86, 0x00, 0xd1, 0x00, 0xf0, 0xc5, 0x67, 0x5f,
+ 0x56, 0x89, 0x5e, 0xb5, 0x6c, 0xaf, 0x03, 0x00,
+ 0x02, 0x00, 0x00, 0x00, 0xc0, 0x37, 0xda, 0x56,
+ 0x90, 0x69, 0x01, 0x2c, 0x96, 0x69, 0x20, 0xfb,
+ // Entry 140 - 17F
+ 0xff, 0x3f, 0x00, 0x00, 0x00, 0x01, 0x08, 0x16,
+ 0x01, 0x00, 0x00, 0xb0, 0x14, 0x03, 0x50, 0x06,
+ 0x0a, 0x00, 0x01, 0x00, 0x00, 0x00, 0x11, 0x09,
+ 0x00, 0x00, 0x60, 0x10, 0x00, 0x00, 0x00, 0x10,
+ 0x00, 0x00, 0x44, 0x00, 0x00, 0x10, 0x00, 0x04,
+ 0x08, 0x00, 0x00, 0x04, 0x00, 0x80, 0x28, 0x04,
+ 0x00, 0x00, 0x40, 0xd5, 0x2d, 0x00, 0x64, 0x35,
+ 0x24, 0x52, 0xf4, 0xd4, 0xbd, 0x62, 0xc9, 0x03,
+ // Entry 180 - 1BF
+ 0x00, 0x80, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x04, 0x13, 0x39, 0x01, 0xdd, 0x57, 0x98,
+ 0x21, 0x18, 0x81, 0x00, 0x00, 0x01, 0x40, 0x82,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x01, 0x40, 0x00, 0x44, 0x00, 0x00, 0x80, 0xea,
+ 0xa9, 0x39, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00,
+ // Entry 1C0 - 1FF
+ 0x00, 0x01, 0x28, 0x05, 0x00, 0x00, 0x00, 0x00,
+ 0x04, 0x20, 0x04, 0xa6, 0x00, 0x04, 0x00, 0x00,
+ 0x81, 0x50, 0x00, 0x00, 0x00, 0x11, 0x84, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x55,
+ 0x02, 0x10, 0x08, 0x04, 0x00, 0x00, 0x00, 0x40,
+ 0x30, 0x83, 0x01, 0x00, 0x00, 0x00, 0x11, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x1e, 0xcd, 0xbf, 0x7a, 0xbf,
+ // Entry 200 - 23F
+ 0xdf, 0xc3, 0x83, 0x82, 0xc0, 0xfb, 0x57, 0x27,
+ 0xcd, 0x55, 0xe7, 0x01, 0x00, 0x20, 0xb2, 0xc5,
+ 0xa4, 0x45, 0x25, 0x9b, 0x02, 0xdf, 0xe0, 0xdf,
+ 0x03, 0x44, 0x08, 0x10, 0x01, 0x04, 0x01, 0xe3,
+ 0x92, 0x54, 0xdb, 0x28, 0xd1, 0x5f, 0xf6, 0x6d,
+ 0x79, 0xed, 0x1c, 0x7d, 0x04, 0x08, 0x00, 0x01,
+ 0x21, 0x12, 0x64, 0x5f, 0xdd, 0x0e, 0x85, 0x4f,
+ 0x40, 0x40, 0x00, 0x04, 0xf1, 0xfd, 0x3d, 0x54,
+ // Entry 240 - 27F
+ 0xe8, 0x03, 0xb4, 0x27, 0x23, 0x0d, 0x00, 0x00,
+ 0x20, 0x7b, 0x38, 0x02, 0x05, 0x84, 0x00, 0xf0,
+ 0xbb, 0x7e, 0x5a, 0x00, 0x18, 0x04, 0x81, 0x00,
+ 0x00, 0x00, 0x80, 0x10, 0x90, 0x1c, 0x01, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x10, 0x40, 0x00, 0x04,
+ 0x08, 0xa0, 0x70, 0xa5, 0x0c, 0x40, 0x00, 0x00,
+ 0x11, 0x04, 0x04, 0x68, 0x00, 0x20, 0x70, 0xff,
+ 0x7b, 0x7f, 0x60, 0x00, 0x05, 0x9b, 0xdd, 0x66,
+ // Entry 280 - 2BF
+ 0x03, 0x00, 0x11, 0x00, 0x00, 0x00, 0x40, 0x05,
+ 0xb5, 0xb6, 0x80, 0x08, 0x04, 0x00, 0x04, 0x51,
+ 0xe2, 0xef, 0xfd, 0x3f, 0x05, 0x09, 0x08, 0x05,
+ 0x40, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00,
+ 0x08, 0x00, 0x00, 0x00, 0x00, 0x81, 0x00, 0x60,
+ 0xe7, 0x48, 0x00, 0x81, 0x20, 0xc0, 0x05, 0x80,
+ 0x03, 0x00, 0x00, 0x00, 0x8c, 0x50, 0x40, 0x04,
+ 0x84, 0x47, 0x84, 0x40, 0x20, 0x10, 0x00, 0x20,
+ // Entry 2C0 - 2FF
+ 0x02, 0x50, 0x80, 0x11, 0x00, 0x91, 0x6c, 0xe2,
+ 0x50, 0x27, 0x1d, 0x11, 0x29, 0x06, 0x59, 0xe9,
+ 0x33, 0x08, 0x00, 0x20, 0x04, 0x40, 0x10, 0x00,
+ 0x00, 0x00, 0x50, 0x44, 0x92, 0x49, 0xd6, 0x5d,
+ 0xa7, 0x81, 0x47, 0x97, 0xfb, 0x00, 0x10, 0x00,
+ 0x08, 0x00, 0x80, 0x00, 0x40, 0x04, 0x00, 0x01,
+ 0x02, 0x00, 0x01, 0x40, 0x80, 0x00, 0x00, 0x08,
+ 0xd8, 0xeb, 0xf6, 0x39, 0xc4, 0x89, 0x12, 0x00,
+ // Entry 300 - 33F
+ 0x00, 0x0c, 0x04, 0x01, 0x20, 0x20, 0xdd, 0xa0,
+ 0x01, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00,
+ 0x04, 0x10, 0xd0, 0x9d, 0x95, 0x13, 0x04, 0x80,
+ 0x00, 0x01, 0xd0, 0x12, 0x40, 0x00, 0x10, 0xb0,
+ 0x10, 0x62, 0x4c, 0xd2, 0x02, 0x01, 0x4a, 0x00,
+ 0x46, 0x04, 0x00, 0x08, 0x02, 0x00, 0x20, 0x80,
+ 0x00, 0x80, 0x06, 0x00, 0x08, 0x00, 0x00, 0x00,
+ 0x00, 0xf0, 0xd8, 0x6f, 0x15, 0x02, 0x08, 0x00,
+ // Entry 340 - 37F
+ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x10, 0x01,
+ 0x00, 0x10, 0x00, 0x00, 0x00, 0xf0, 0x84, 0xe3,
+ 0xdd, 0xbf, 0xf9, 0xf9, 0x3b, 0x7f, 0x7f, 0xdb,
+ 0xfd, 0xfc, 0xfe, 0xdf, 0xff, 0xfd, 0xff, 0xf6,
+ 0xfb, 0xfc, 0xf7, 0x1f, 0xff, 0xb3, 0x6c, 0xff,
+ 0xd9, 0xad, 0xdf, 0xfe, 0xef, 0xba, 0xdf, 0xff,
+ 0xff, 0xff, 0xb7, 0xdd, 0x7d, 0xbf, 0xab, 0x7f,
+ 0xfd, 0xfd, 0xdf, 0x2f, 0x9c, 0xdf, 0xf3, 0x6f,
+ // Entry 380 - 3BF
+ 0xdf, 0xdd, 0xff, 0xfb, 0xee, 0xd2, 0xab, 0x5f,
+ 0xd5, 0xdf, 0x7f, 0xff, 0xeb, 0xff, 0xe4, 0x4d,
+ 0xf9, 0xff, 0xfe, 0xf7, 0xfd, 0xdf, 0xfb, 0xbf,
+ 0xee, 0xdb, 0x6f, 0xef, 0xff, 0x7f, 0xff, 0xff,
+ 0xf7, 0x5f, 0xd3, 0x3b, 0xfd, 0xd9, 0xdf, 0xeb,
+ 0xbc, 0x08, 0x05, 0x24, 0xff, 0x07, 0x70, 0xfe,
+ 0xe6, 0x5e, 0x00, 0x08, 0x00, 0x83, 0x3d, 0x1b,
+ 0x06, 0xe6, 0x72, 0x60, 0xd1, 0x3c, 0x7f, 0x44,
+ // Entry 3C0 - 3FF
+ 0x02, 0x30, 0x9f, 0x7a, 0x16, 0xbd, 0x7f, 0x57,
+ 0xf2, 0xff, 0x31, 0xff, 0xf2, 0x1e, 0x90, 0xf7,
+ 0xf1, 0xf9, 0x45, 0x80, 0x01, 0x02, 0x00, 0x00,
+ 0x40, 0x54, 0x9f, 0x8a, 0xd9, 0xd9, 0x0e, 0x11,
+ 0x86, 0x51, 0xc0, 0xf3, 0xfb, 0x47, 0x00, 0x01,
+ 0x05, 0xd1, 0x50, 0x58, 0x00, 0x00, 0x00, 0x10,
+ 0x04, 0x02, 0x00, 0x00, 0x0a, 0x00, 0x17, 0xd2,
+ 0xb9, 0xfd, 0xfc, 0xba, 0xfe, 0xef, 0xc7, 0xbe,
+ // Entry 400 - 43F
+ 0x53, 0x6f, 0xdf, 0xe7, 0xdb, 0x65, 0xbb, 0x7f,
+ 0xfa, 0xff, 0x77, 0xf3, 0xef, 0xbf, 0xfd, 0xf7,
+ 0xdf, 0xdf, 0x9b, 0x7f, 0xff, 0xff, 0x7f, 0x6f,
+ 0xf7, 0xfb, 0xeb, 0xdf, 0xbc, 0xff, 0xbf, 0x6b,
+ 0x7b, 0xfb, 0xff, 0xce, 0x76, 0xbd, 0xf7, 0xf7,
+ 0xdf, 0xdc, 0xf7, 0xf7, 0xff, 0xdf, 0xf3, 0xfe,
+ 0xef, 0xff, 0xff, 0xff, 0xb6, 0x7f, 0x7f, 0xde,
+ 0xf7, 0xb9, 0xeb, 0x77, 0xff, 0xfb, 0xbf, 0xdf,
+ // Entry 440 - 47F
+ 0xfd, 0xfe, 0xfb, 0xff, 0xfe, 0xeb, 0x1f, 0x7d,
+ 0x2f, 0xfd, 0xb6, 0xb5, 0xa5, 0xfc, 0xff, 0xfd,
+ 0x7f, 0x4e, 0xbf, 0x8f, 0xae, 0xff, 0xee, 0xdf,
+ 0x7f, 0xf7, 0x73, 0x02, 0x02, 0x04, 0xfc, 0xf7,
+ 0xff, 0xb7, 0xd7, 0xef, 0xfe, 0xcd, 0xf5, 0xce,
+ 0xe2, 0x8e, 0xe7, 0xbf, 0xb7, 0xff, 0x56, 0xbd,
+ 0xcd, 0xff, 0xfb, 0xff, 0xdf, 0xd7, 0xea, 0xff,
+ 0xe5, 0x5f, 0x6d, 0x0f, 0xa7, 0x51, 0x06, 0xc4,
+ // Entry 480 - 4BF
+ 0x13, 0x50, 0x5d, 0xaf, 0xa6, 0xfd, 0x99, 0xfb,
+ 0x63, 0x1d, 0x53, 0xff, 0xef, 0xb7, 0x35, 0x20,
+ 0x14, 0x00, 0x55, 0x51, 0x82, 0x65, 0xf5, 0x41,
+ 0xe2, 0xff, 0xfc, 0xdf, 0x00, 0x05, 0xc5, 0x05,
+ 0x00, 0x22, 0x00, 0x74, 0x69, 0x10, 0x08, 0x04,
+ 0x41, 0x00, 0x01, 0x06, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x51, 0x20, 0x05, 0x04, 0x01, 0x00, 0x00,
+ 0x06, 0x01, 0x20, 0x00, 0x18, 0x01, 0x92, 0xb1,
+ // Entry 4C0 - 4FF
+ 0xfd, 0x47, 0x49, 0x06, 0x95, 0x06, 0x57, 0xed,
+ 0xfb, 0x4c, 0x1c, 0x6b, 0x83, 0x04, 0x62, 0x40,
+ 0x00, 0x11, 0x42, 0x00, 0x00, 0x00, 0x54, 0x83,
+ 0xb8, 0x4f, 0x10, 0x8c, 0x89, 0x46, 0xde, 0xf7,
+ 0x13, 0x31, 0x00, 0x20, 0x00, 0x00, 0x00, 0x90,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x10, 0x00,
+ 0x01, 0x00, 0x00, 0xf0, 0x5b, 0xf4, 0xbe, 0x3d,
+ 0xba, 0xcf, 0xf7, 0xaf, 0x42, 0x04, 0x84, 0x41,
+ // Entry 500 - 53F
+ 0x30, 0xff, 0x79, 0x72, 0x04, 0x00, 0x00, 0x49,
+ 0x2d, 0x14, 0x27, 0x57, 0xed, 0xf1, 0x3f, 0xe7,
+ 0x3f, 0x00, 0x00, 0x02, 0xc6, 0xa0, 0x1e, 0xf8,
+ 0xbb, 0xff, 0xfd, 0xfb, 0xb7, 0xfd, 0xe5, 0xf7,
+ 0xfd, 0xfc, 0xd5, 0xed, 0x47, 0xf4, 0x7e, 0x10,
+ 0x01, 0x01, 0x84, 0x6d, 0xff, 0xf7, 0xdd, 0xf9,
+ 0x5b, 0x05, 0x86, 0xed, 0xf5, 0x77, 0xbd, 0x3c,
+ 0x00, 0x00, 0x00, 0x42, 0x71, 0x42, 0x00, 0x40,
+ // Entry 540 - 57F
+ 0x00, 0x00, 0x01, 0x43, 0x19, 0x00, 0x08, 0x00,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ // Entry 580 - 5BF
+ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
+ 0xff, 0xab, 0xbd, 0xe7, 0x57, 0xee, 0x13, 0x5d,
+ 0x09, 0xc1, 0x40, 0x21, 0xfa, 0x17, 0x01, 0x80,
+ 0x00, 0x00, 0x00, 0x00, 0xf0, 0xce, 0xfb, 0xbf,
+ 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00,
+ 0x00, 0x30, 0x15, 0xa3, 0x10, 0x00, 0x00, 0x00,
+ 0x11, 0x04, 0x16, 0x00, 0x00, 0x02, 0x00, 0x81,
+ 0xa3, 0x01, 0x50, 0x00, 0x00, 0x83, 0x11, 0x40,
+ // Entry 5C0 - 5FF
+ 0x00, 0x00, 0x00, 0xf0, 0xdd, 0x7b, 0x3e, 0x02,
+ 0xaa, 0x10, 0x5d, 0x98, 0x52, 0x00, 0x80, 0x20,
+ 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x02, 0x02,
+ 0x19, 0x00, 0x10, 0x02, 0x10, 0x61, 0x5a, 0x9d,
+ 0x31, 0x00, 0x00, 0x00, 0x01, 0x10, 0x02, 0x20,
+ 0x00, 0x00, 0x01, 0x00, 0x42, 0x00, 0x20, 0x00,
+ 0x00, 0x1f, 0xdf, 0xd2, 0xb9, 0xff, 0xfd, 0x3f,
+ 0x1f, 0x98, 0xcf, 0x9c, 0xbf, 0xaf, 0x5f, 0xfe,
+ // Entry 600 - 63F
+ 0x7b, 0x4b, 0x40, 0x10, 0xe1, 0xfd, 0xaf, 0xd9,
+ 0xb7, 0xf6, 0xfb, 0xb3, 0xc7, 0xff, 0x6f, 0xf1,
+ 0x73, 0xb1, 0x7f, 0x9f, 0x7f, 0xbd, 0xfc, 0xb7,
+ 0xee, 0x1c, 0xfa, 0xcb, 0xef, 0xdd, 0xf9, 0xbd,
+ 0x6e, 0xae, 0x55, 0xfd, 0x6e, 0x81, 0x76, 0x1f,
+ 0xd4, 0x77, 0xf5, 0x7d, 0xfb, 0xff, 0xeb, 0xfe,
+ 0xbe, 0x5f, 0x46, 0x1b, 0xe9, 0x5f, 0x50, 0x18,
+ 0x02, 0xfa, 0xf7, 0x9d, 0x15, 0x97, 0x05, 0x0f,
+ // Entry 640 - 67F
+ 0x75, 0xc4, 0x7d, 0x81, 0x92, 0xf1, 0x57, 0x6c,
+ 0xff, 0xe4, 0xef, 0x6f, 0xff, 0xfc, 0xdd, 0xde,
+ 0xfc, 0xfd, 0x76, 0x5f, 0x7a, 0x1f, 0x00, 0x98,
+ 0x02, 0xfb, 0xa3, 0xef, 0xf3, 0xd6, 0xf2, 0xff,
+ 0xb9, 0xda, 0x7d, 0x50, 0x1e, 0x15, 0x7b, 0xb4,
+ 0xf5, 0x3e, 0xff, 0xff, 0xf1, 0xf7, 0xff, 0xe7,
+ 0x5f, 0xff, 0xff, 0x9e, 0xdb, 0xf6, 0xd7, 0xb9,
+ 0xef, 0x27, 0x80, 0xbb, 0xc5, 0xff, 0xff, 0xe3,
+ // Entry 680 - 6BF
+ 0x97, 0x9d, 0xbf, 0x9f, 0xf7, 0xc7, 0xfd, 0x37,
+ 0xce, 0x7f, 0x04, 0x1d, 0x53, 0x7f, 0xf8, 0xda,
+ 0x5d, 0xce, 0x7d, 0x06, 0xb9, 0xea, 0x69, 0xa0,
+ 0x1a, 0x20, 0x00, 0x30, 0x02, 0x04, 0x24, 0x08,
+ 0x04, 0x00, 0x00, 0x40, 0xd4, 0x02, 0x04, 0x00,
+ 0x00, 0x04, 0x00, 0x04, 0x00, 0x20, 0x01, 0x06,
+ 0x50, 0x00, 0x08, 0x00, 0x00, 0x00, 0x24, 0x00,
+ 0x04, 0x00, 0x10, 0xcc, 0x58, 0xd5, 0x0d, 0x0f,
+ // Entry 6C0 - 6FF
+ 0x14, 0x4d, 0xf1, 0x16, 0x44, 0xd1, 0x42, 0x08,
+ 0x40, 0x00, 0x00, 0x40, 0x00, 0x08, 0x00, 0x00,
+ 0x00, 0xdc, 0xfb, 0xcb, 0x0e, 0x58, 0x08, 0x41,
+ 0x04, 0x20, 0x04, 0x00, 0x30, 0x12, 0x40, 0x00,
+ 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x01, 0x00, 0x00, 0x00, 0x80, 0x10, 0x10, 0xab,
+ 0x6d, 0x93, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x80, 0x80, 0x25, 0x00, 0x00,
+ // Entry 700 - 73F
+ 0x00, 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00,
+ 0x80, 0x86, 0xc2, 0x00, 0x00, 0x00, 0x00, 0x01,
+ 0xdf, 0x18, 0x00, 0x00, 0x02, 0xf0, 0xfd, 0x79,
+ 0x3b, 0x00, 0x25, 0x00, 0x00, 0x00, 0x02, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00,
+ 0x03, 0x00, 0x09, 0x20, 0x00, 0x00, 0x01, 0x00,
+ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ // Entry 740 - 77F
+ 0x00, 0x00, 0x00, 0xef, 0xd5, 0xfd, 0xcf, 0x7e,
+ 0xb0, 0x11, 0x00, 0x00, 0x00, 0x92, 0x01, 0x44,
+ 0xcd, 0xf9, 0x5c, 0x00, 0x01, 0x00, 0x30, 0x04,
+ 0x04, 0x55, 0x00, 0x01, 0x04, 0xf4, 0x3f, 0x4a,
+ 0x01, 0x00, 0x00, 0xb0, 0x80, 0x00, 0x55, 0x55,
+ 0x97, 0x7c, 0x9f, 0x31, 0xcc, 0x68, 0xd1, 0x03,
+ 0xd5, 0x57, 0x27, 0x14, 0x01, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x2c, 0xf7, 0xcb, 0x1f, 0x14, 0x60,
+ // Entry 780 - 7BF
+ 0x03, 0x68, 0x01, 0x10, 0x8b, 0x38, 0x8a, 0x01,
+ 0x00, 0x00, 0x20, 0x00, 0x24, 0x44, 0x00, 0x00,
+ 0x10, 0x03, 0x11, 0x02, 0x01, 0x00, 0x00, 0xf0,
+ 0xf5, 0xff, 0xd5, 0x97, 0xbc, 0x70, 0xd6, 0x78,
+ 0x78, 0x15, 0x50, 0x01, 0xa4, 0x84, 0xa9, 0x41,
+ 0x00, 0x00, 0x00, 0x6b, 0x39, 0x52, 0x74, 0x00,
+ 0xe8, 0x30, 0x90, 0x6a, 0x92, 0x00, 0x00, 0x02,
+ 0xff, 0xef, 0xff, 0x4b, 0x85, 0x53, 0xf4, 0xed,
+ // Entry 7C0 - 7FF
+ 0xdd, 0xbf, 0x72, 0x19, 0xc7, 0x0c, 0xd5, 0x42,
+ 0x54, 0xdd, 0x77, 0x14, 0x00, 0x80, 0x40, 0x56,
+ 0xcc, 0x16, 0x9e, 0xea, 0x35, 0x7d, 0xef, 0xff,
+ 0xbd, 0xa4, 0xaf, 0x01, 0x44, 0x18, 0x01, 0x4d,
+ 0x4e, 0x4a, 0x08, 0x50, 0x28, 0x30, 0xe0, 0x80,
+ 0x10, 0x20, 0x24, 0x00, 0xff, 0x2f, 0xd3, 0x60,
+ 0xfe, 0x01, 0x02, 0x88, 0x0a, 0x40, 0x16, 0x01,
+ 0x01, 0x15, 0x2b, 0x3c, 0x01, 0x00, 0x00, 0x10,
+ // Entry 800 - 83F
+ 0x90, 0x49, 0x41, 0x02, 0x02, 0x01, 0xe1, 0xbf,
+ 0xbf, 0x03, 0x00, 0x00, 0x10, 0xd4, 0xa3, 0xd1,
+ 0x40, 0x9c, 0x44, 0xdf, 0xf5, 0x8f, 0x66, 0xb3,
+ 0x55, 0x20, 0xd4, 0xc1, 0xd8, 0x30, 0x3d, 0x80,
+ 0x00, 0x00, 0x00, 0x04, 0xd4, 0x11, 0xc5, 0x84,
+ 0x2e, 0x50, 0x00, 0x22, 0x50, 0x6e, 0xbd, 0x93,
+ 0x07, 0x00, 0x20, 0x10, 0x84, 0xb2, 0x45, 0x10,
+ 0x06, 0x44, 0x00, 0x00, 0x12, 0x02, 0x11, 0x00,
+ // Entry 840 - 87F
+ 0xf0, 0xfb, 0xfd, 0x3f, 0x05, 0x00, 0x12, 0x81,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x02,
+ 0x00, 0x00, 0x00, 0x00, 0x03, 0x30, 0x02, 0x28,
+ 0x84, 0x00, 0x21, 0xc0, 0x23, 0x24, 0x00, 0x00,
+ 0x00, 0xcb, 0xe4, 0x3a, 0x42, 0x88, 0x14, 0xf1,
+ 0xef, 0xff, 0x7f, 0x12, 0x01, 0x01, 0x84, 0x50,
+ 0x07, 0xfc, 0xff, 0xff, 0x0f, 0x01, 0x00, 0x40,
+ 0x10, 0x38, 0x01, 0x01, 0x1c, 0x12, 0x40, 0xe1,
+ // Entry 880 - 8BF
+ 0x76, 0x16, 0x08, 0x03, 0x10, 0x00, 0x00, 0x00,
+ 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x24,
+ 0x0a, 0x00, 0x80, 0x00, 0x00,
+}
+
+// altLangISO3 holds an alphabetically sorted list of 3-letter language code alternatives
+// to 2-letter language codes that cannot be derived using the method described above.
+// Each 3-letter code is followed by its 1-byte langID.
+const altLangISO3 tag.Index = "---\x00cor\x00hbs\x01heb\x02kin\x03spa\x04yid\x05\xff\xff\xff\xff"
+
+// altLangIndex is used to convert indexes in altLangISO3 to langIDs.
+// Size: 12 bytes, 6 elements
+var altLangIndex = [6]uint16{
+ 0x0281, 0x0407, 0x01fb, 0x03e5, 0x013e, 0x0208,
+}
+
+// langAliasMap maps langIDs to their suggested replacements.
+// Size: 656 bytes, 164 elements
+var langAliasMap = [164]fromTo{
+ 0: {from: 0x82, to: 0x88},
+ 1: {from: 0x187, to: 0x1ae},
+ 2: {from: 0x1f3, to: 0x1e1},
+ 3: {from: 0x1fb, to: 0x1bc},
+ 4: {from: 0x208, to: 0x512},
+ 5: {from: 0x20f, to: 0x20e},
+ 6: {from: 0x310, to: 0x3dc},
+ 7: {from: 0x347, to: 0x36f},
+ 8: {from: 0x407, to: 0x432},
+ 9: {from: 0x47a, to: 0x153},
+ 10: {from: 0x490, to: 0x451},
+ 11: {from: 0x4a2, to: 0x21},
+ 12: {from: 0x53e, to: 0x544},
+ 13: {from: 0x58f, to: 0x12d},
+ 14: {from: 0x630, to: 0x1eb1},
+ 15: {from: 0x651, to: 0x431},
+ 16: {from: 0x662, to: 0x431},
+ 17: {from: 0x6ed, to: 0x3a},
+ 18: {from: 0x6f8, to: 0x1d7},
+ 19: {from: 0x73e, to: 0x21a1},
+ 20: {from: 0x7b3, to: 0x56},
+ 21: {from: 0x7b9, to: 0x299b},
+ 22: {from: 0x7c5, to: 0x58},
+ 23: {from: 0x7e6, to: 0x145},
+ 24: {from: 0x80c, to: 0x5a},
+ 25: {from: 0x815, to: 0x8d},
+ 26: {from: 0x87e, to: 0x810},
+ 27: {from: 0x8c3, to: 0xee3},
+ 28: {from: 0x9ef, to: 0x331},
+ 29: {from: 0xa36, to: 0x2c5},
+ 30: {from: 0xa3d, to: 0xbf},
+ 31: {from: 0xabe, to: 0x3322},
+ 32: {from: 0xb38, to: 0x529},
+ 33: {from: 0xb75, to: 0x265a},
+ 34: {from: 0xb7e, to: 0xbc3},
+ 35: {from: 0xb9b, to: 0x44e},
+ 36: {from: 0xbbc, to: 0x4229},
+ 37: {from: 0xbbf, to: 0x529},
+ 38: {from: 0xbfe, to: 0x2da7},
+ 39: {from: 0xc2e, to: 0x3181},
+ 40: {from: 0xcb9, to: 0xf3},
+ 41: {from: 0xd08, to: 0xfa},
+ 42: {from: 0xdc8, to: 0x11a},
+ 43: {from: 0xdd7, to: 0x32d},
+ 44: {from: 0xdf8, to: 0xdfb},
+ 45: {from: 0xdfe, to: 0x531},
+ 46: {from: 0xedf, to: 0x205a},
+ 47: {from: 0xeee, to: 0x2e9a},
+ 48: {from: 0xf39, to: 0x367},
+ 49: {from: 0x10d0, to: 0x140},
+ 50: {from: 0x1104, to: 0x2d0},
+ 51: {from: 0x11a0, to: 0x1ec},
+ 52: {from: 0x1279, to: 0x21},
+ 53: {from: 0x1424, to: 0x15e},
+ 54: {from: 0x1470, to: 0x14e},
+ 55: {from: 0x151f, to: 0xd9b},
+ 56: {from: 0x1523, to: 0x390},
+ 57: {from: 0x1532, to: 0x19f},
+ 58: {from: 0x1580, to: 0x210},
+ 59: {from: 0x1583, to: 0x10d},
+ 60: {from: 0x15a3, to: 0x3caf},
+ 61: {from: 0x166a, to: 0x19b},
+ 62: {from: 0x16c8, to: 0x136},
+ 63: {from: 0x1700, to: 0x29f8},
+ 64: {from: 0x1718, to: 0x194},
+ 65: {from: 0x1727, to: 0xf3f},
+ 66: {from: 0x177a, to: 0x178},
+ 67: {from: 0x1809, to: 0x17b6},
+ 68: {from: 0x1816, to: 0x18f3},
+ 69: {from: 0x188a, to: 0x436},
+ 70: {from: 0x1979, to: 0x1d01},
+ 71: {from: 0x1a74, to: 0x2bb0},
+ 72: {from: 0x1a8a, to: 0x1f8},
+ 73: {from: 0x1b5a, to: 0x1fa},
+ 74: {from: 0x1b86, to: 0x1515},
+ 75: {from: 0x1d64, to: 0x2c9b},
+ 76: {from: 0x2038, to: 0x37b1},
+ 77: {from: 0x203d, to: 0x20dd},
+ 78: {from: 0x205a, to: 0x30b},
+ 79: {from: 0x20e3, to: 0x274},
+ 80: {from: 0x20ee, to: 0x263},
+ 81: {from: 0x20f2, to: 0x22d},
+ 82: {from: 0x20f9, to: 0x256},
+ 83: {from: 0x210f, to: 0x21eb},
+ 84: {from: 0x2135, to: 0x27d},
+ 85: {from: 0x2160, to: 0x913},
+ 86: {from: 0x2199, to: 0x121},
+ 87: {from: 0x21ce, to: 0x1561},
+ 88: {from: 0x21e6, to: 0x504},
+ 89: {from: 0x21f4, to: 0x49f},
+ 90: {from: 0x222d, to: 0x121},
+ 91: {from: 0x2237, to: 0x121},
+ 92: {from: 0x2262, to: 0x92a},
+ 93: {from: 0x2316, to: 0x3226},
+ 94: {from: 0x2382, to: 0x3365},
+ 95: {from: 0x2472, to: 0x2c7},
+ 96: {from: 0x24e4, to: 0x2ff},
+ 97: {from: 0x24f0, to: 0x2fa},
+ 98: {from: 0x24fa, to: 0x31f},
+ 99: {from: 0x2550, to: 0xb5b},
+ 100: {from: 0x25a9, to: 0xe2},
+ 101: {from: 0x263e, to: 0x2d0},
+ 102: {from: 0x26c9, to: 0x26b4},
+ 103: {from: 0x26f9, to: 0x3c8},
+ 104: {from: 0x2727, to: 0x3caf},
+ 105: {from: 0x2765, to: 0x26b4},
+ 106: {from: 0x2789, to: 0x4358},
+ 107: {from: 0x28ef, to: 0x2837},
+ 108: {from: 0x2914, to: 0x351},
+ 109: {from: 0x2986, to: 0x2da7},
+ 110: {from: 0x2b1a, to: 0x38d},
+ 111: {from: 0x2bfc, to: 0x395},
+ 112: {from: 0x2c3f, to: 0x3caf},
+ 113: {from: 0x2cfc, to: 0x3be},
+ 114: {from: 0x2d13, to: 0x597},
+ 115: {from: 0x2d47, to: 0x148},
+ 116: {from: 0x2d48, to: 0x148},
+ 117: {from: 0x2dff, to: 0x2f1},
+ 118: {from: 0x2e08, to: 0x19cc},
+ 119: {from: 0x2e1a, to: 0x2d95},
+ 120: {from: 0x2e21, to: 0x292},
+ 121: {from: 0x2e54, to: 0x7d},
+ 122: {from: 0x2e65, to: 0x2282},
+ 123: {from: 0x2ea0, to: 0x2e9b},
+ 124: {from: 0x2eef, to: 0x2ed7},
+ 125: {from: 0x3193, to: 0x3c4},
+ 126: {from: 0x3366, to: 0x338e},
+ 127: {from: 0x342a, to: 0x3dc},
+ 128: {from: 0x34ee, to: 0x18d0},
+ 129: {from: 0x35c8, to: 0x2c9b},
+ 130: {from: 0x35e6, to: 0x412},
+ 131: {from: 0x3658, to: 0x246},
+ 132: {from: 0x3676, to: 0x3f4},
+ 133: {from: 0x36fd, to: 0x445},
+ 134: {from: 0x37c0, to: 0x121},
+ 135: {from: 0x3816, to: 0x38f2},
+ 136: {from: 0x382b, to: 0x2c9b},
+ 137: {from: 0x382f, to: 0xa9},
+ 138: {from: 0x3832, to: 0x3228},
+ 139: {from: 0x386c, to: 0x39a6},
+ 140: {from: 0x3892, to: 0x3fc0},
+ 141: {from: 0x38a5, to: 0x39d7},
+ 142: {from: 0x38b4, to: 0x1fa4},
+ 143: {from: 0x38b5, to: 0x2e9a},
+ 144: {from: 0x395c, to: 0x47e},
+ 145: {from: 0x3b4e, to: 0xd91},
+ 146: {from: 0x3b78, to: 0x137},
+ 147: {from: 0x3c99, to: 0x4bc},
+ 148: {from: 0x3fbd, to: 0x100},
+ 149: {from: 0x4208, to: 0xa91},
+ 150: {from: 0x42be, to: 0x573},
+ 151: {from: 0x42f9, to: 0x3f60},
+ 152: {from: 0x4378, to: 0x25a},
+ 153: {from: 0x43cb, to: 0x36cb},
+ 154: {from: 0x43cd, to: 0x10f},
+ 155: {from: 0x44af, to: 0x3322},
+ 156: {from: 0x44e3, to: 0x512},
+ 157: {from: 0x45ca, to: 0x2409},
+ 158: {from: 0x45dd, to: 0x26dc},
+ 159: {from: 0x4610, to: 0x48ae},
+ 160: {from: 0x46ae, to: 0x46a0},
+ 161: {from: 0x473e, to: 0x4745},
+ 162: {from: 0x4916, to: 0x31f},
+ 163: {from: 0x49a7, to: 0x523},
+}
+
+// Size: 164 bytes, 164 elements
+var langAliasTypes = [164]langAliasType{
+ // Entry 0 - 3F
+ 1, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 1, 0, 0, 1, 2,
+ 1, 1, 2, 0, 1, 0, 1, 2, 1, 1, 0, 0, 2, 1, 1, 0,
+ 2, 0, 0, 1, 0, 1, 0, 0, 1, 2, 1, 1, 1, 1, 0, 0,
+ 2, 1, 1, 1, 1, 2, 1, 0, 1, 1, 2, 2, 0, 1, 2, 0,
+ // Entry 40 - 7F
+ 1, 0, 1, 1, 1, 1, 0, 0, 2, 1, 0, 0, 0, 0, 1, 1,
+ 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1,
+ 2, 2, 2, 0, 1, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1,
+ 0, 1, 0, 2, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 2,
+ // Entry 80 - BF
+ 0, 0, 2, 1, 1, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0,
+ 1, 1, 0, 1, 2, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0,
+ 0, 1, 1, 1,
+}
+
+const (
+ _Latn = 87
+ _Hani = 54
+ _Hans = 56
+ _Hant = 57
+ _Qaaa = 139
+ _Qaai = 147
+ _Qabx = 188
+ _Zinh = 236
+ _Zyyy = 241
+ _Zzzz = 242
+)
+
+// script is an alphabetically sorted list of ISO 15924 codes. The index
+// of the script in the string, divided by 4, is the internal scriptID.
+const script tag.Index = "" + // Size: 976 bytes
+ "----AdlmAfakAghbAhomArabAranArmiArmnAvstBaliBamuBassBatkBengBhksBlisBopo" +
+ "BrahBraiBugiBuhdCakmCansCariChamCherCirtCoptCpmnCprtCyrlCyrsDevaDogrDsrt" +
+ "DuplEgydEgyhEgypElbaEthiGeokGeorGlagGongGonmGothGranGrekGujrGuruHanbHang" +
+ "HaniHanoHansHantHatrHebrHiraHluwHmngHmnpHrktHungIndsItalJamoJavaJpanJurc" +
+ "KaliKanaKharKhmrKhojKitlKitsKndaKoreKpelKthiLanaLaooLatfLatgLatnLekeLepc" +
+ "LimbLinaLinbLisuLomaLyciLydiMahjMakaMandManiMarcMayaMedfMendMercMeroMlym" +
+ "ModiMongMoonMrooMteiMultMymrNarbNbatNewaNkdbNkgbNkooNshuOgamOlckOrkhOrya" +
+ "OsgeOsmaPalmPaucPermPhagPhliPhlpPhlvPhnxPiqdPlrdPrtiQaaaQaabQaacQaadQaae" +
+ "QaafQaagQaahQaaiQaajQaakQaalQaamQaanQaaoQaapQaaqQaarQaasQaatQaauQaavQaaw" +
+ "QaaxQaayQaazQabaQabbQabcQabdQabeQabfQabgQabhQabiQabjQabkQablQabmQabnQabo" +
+ "QabpQabqQabrQabsQabtQabuQabvQabwQabxRjngRoroRunrSamrSaraSarbSaurSgnwShaw" +
+ "ShrdShuiSiddSindSinhSoraSoyoSundSyloSyrcSyreSyrjSyrnTagbTakrTaleTaluTaml" +
+ "TangTavtTeluTengTfngTglgThaaThaiTibtTirhUgarVaiiVispWaraWchoWoleXpeoXsux" +
+ "YiiiZanbZinhZmthZsyeZsymZxxxZyyyZzzz\xff\xff\xff\xff"
+
+// suppressScript is an index from langID to the dominant script for that language,
+// if it exists. If a script is given, it should be suppressed from the language tag.
+// Size: 1330 bytes, 1330 elements
+var suppressScript = [1330]uint8{
+ // Entry 0 - 3F
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00,
+ // Entry 40 - 7F
+ 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0x00,
+ // Entry 80 - BF
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x0e, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ // Entry C0 - FF
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00,
+ // Entry 100 - 13F
+ 0x57, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0xde, 0x00, 0x00, 0x00, 0x00, 0xe0, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x31, 0x00,
+ 0x00, 0x57, 0x00, 0x00, 0x57, 0x00, 0x57, 0x00,
+ // Entry 140 - 17F
+ 0x57, 0x00, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00,
+ 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x57, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00,
+ 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x00,
+ 0x00, 0x57, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x57, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ // Entry 180 - 1BF
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x57, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x57, 0x32, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x3b, 0x00, 0x21, 0x00,
+ // Entry 1C0 - 1FF
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x57, 0x57, 0x00, 0x57, 0x57, 0x00, 0x08,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x00,
+ 0x57, 0x57, 0x00, 0x3b, 0x00, 0x00, 0x00, 0x00,
+ // Entry 200 - 23F
+ 0x46, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x2b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ // Entry 240 - 27F
+ 0x00, 0x00, 0x1f, 0x00, 0x00, 0x57, 0x00, 0x00,
+ 0x00, 0x00, 0x4b, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x4f, 0x00, 0x00, 0x50, 0x00, 0x21, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ // Entry 280 - 2BF
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00,
+ 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ // Entry 2C0 - 2FF
+ 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x21, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f,
+ // Entry 300 - 33F
+ 0x00, 0x00, 0x00, 0x00, 0x6b, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x21, 0x00, 0x00, 0x00, 0x57,
+ 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x72, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x00,
+ // Entry 340 - 37F
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57, 0x00,
+ 0x57, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57,
+ 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x00, 0x57,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x57, 0x00,
+ 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00,
+ // Entry 380 - 3BF
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x57, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x33, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00,
+ // Entry 3C0 - 3FF
+ 0x57, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00,
+ 0x00, 0x57, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x1f, 0x00, 0x00, 0x57, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ // Entry 400 - 43F
+ 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0xca, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x57, 0x00,
+ 0x00, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57,
+ 0x00, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x00,
+ // Entry 440 - 47F
+ 0x00, 0x00, 0x00, 0x00, 0x57, 0x57, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0xd7, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0xda, 0x00, 0x57, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0xdf, 0x00, 0x00, 0x00, 0x29,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57,
+ 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x57, 0x00,
+ // Entry 480 - 4BF
+ 0x57, 0x00, 0x57, 0x00, 0x00, 0x00, 0x57, 0x00,
+ 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x57, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1f, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ // Entry 4C0 - 4FF
+ 0x57, 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ // Entry 500 - 53F
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x3b, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x57,
+ 0x00, 0x00,
+}
+
+const (
+ _001 = 1
+ _419 = 31
+ _BR = 65
+ _CA = 73
+ _ES = 110
+ _GB = 123
+ _MD = 188
+ _PT = 238
+ _UK = 306
+ _US = 309
+ _ZZ = 357
+ _XA = 323
+ _XC = 325
+ _XK = 333
+)
+
+// isoRegionOffset needs to be added to the index of regionISO to obtain the regionID
+// for 2-letter ISO codes. (The first isoRegionOffset regionIDs are reserved for
+// the UN.M49 codes used for groups.)
+const isoRegionOffset = 32
+
+// regionTypes defines the status of a region for various standards.
+// Size: 358 bytes, 358 elements
+var regionTypes = [358]uint8{
+ // Entry 0 - 3F
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x05, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ // Entry 40 - 7F
+ 0x06, 0x06, 0x06, 0x06, 0x04, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x04, 0x06, 0x04,
+ 0x00, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x04,
+ 0x06, 0x04, 0x06, 0x06, 0x06, 0x06, 0x00, 0x06,
+ 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x00,
+ 0x06, 0x04, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ // Entry 80 - BF
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x00, 0x04, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x00, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ // Entry C0 - FF
+ 0x06, 0x00, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x00,
+ 0x06, 0x06, 0x06, 0x06, 0x00, 0x06, 0x04, 0x06,
+ 0x06, 0x06, 0x06, 0x00, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x00,
+ 0x06, 0x06, 0x00, 0x06, 0x05, 0x05, 0x05, 0x05,
+ 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05,
+ // Entry 100 - 13F
+ 0x05, 0x05, 0x06, 0x00, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x04, 0x06,
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x02, 0x06, 0x04, 0x06, 0x06, 0x06,
+ 0x06, 0x06, 0x00, 0x06, 0x06, 0x06, 0x06, 0x06,
+ // Entry 140 - 17F
+ 0x06, 0x00, 0x06, 0x05, 0x05, 0x05, 0x05, 0x05,
+ 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05,
+ 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05, 0x05,
+ 0x05, 0x05, 0x05, 0x05, 0x05, 0x04, 0x06, 0x06,
+ 0x04, 0x06, 0x06, 0x04, 0x06, 0x05,
+}
+
+// regionISO holds a list of alphabetically sorted 2-letter ISO region codes.
+// Each 2-letter codes is followed by two bytes with the following meaning:
+// - [A-Z}{2}: the first letter of the 2-letter code plus these two
+// letters form the 3-letter ISO code.
+// - 0, n: index into altRegionISO3.
+const regionISO tag.Index = "" + // Size: 1308 bytes
+ "AAAAACSCADNDAEREAFFGAGTGAIIAALLBAMRMANNTAOGOAQTAARRGASSMATUTAUUSAWBWAXLA" +
+ "AZZEBAIHBBRBBDGDBEELBFFABGGRBHHRBIDIBJENBLLMBMMUBNRNBOOLBQESBRRABSHSBTTN" +
+ "BUURBVVTBWWABYLRBZLZCAANCCCKCDODCFAFCGOGCHHECIIVCKOKCLHLCMMRCNHNCOOLCPPT" +
+ "CRRICS\x00\x00CTTECUUBCVPVCWUWCXXRCYYPCZZEDDDRDEEUDGGADJJIDKNKDMMADOOMDY" +
+ "HYDZZAEA ECCUEESTEGGYEHSHERRIESSPETTHEU\x00\x03EZ FIINFJJIFKLKFMSMFORO" +
+ "FQ\x00\x18FRRAFXXXGAABGBBRGDRDGEEOGFUFGGGYGHHAGIIBGLRLGMMBGNINGPLPGQNQGR" +
+ "RCGS\x00\x06GTTMGUUMGWNBGYUYHKKGHMMDHNNDHRRVHTTIHUUNHVVOIC IDDNIERLILSR" +
+ "IMMNINNDIOOTIQRQIRRNISSLITTAJEEYJMAMJOORJPPNJTTNKEENKGGZKHHMKIIRKM\x00" +
+ "\x09KNNAKP\x00\x0cKRORKWWTKY\x00\x0fKZAZLAAOLBBNLCCALIIELKKALRBRLSSOLTTU" +
+ "LUUXLVVALYBYMAARMCCOMDDAMENEMFAFMGDGMHHLMIIDMKKDMLLIMMMRMNNGMOACMPNPMQTQ" +
+ "MRRTMSSRMTLTMUUSMVDVMWWIMXEXMYYSMZOZNAAMNCCLNEERNFFKNGGANHHBNIICNLLDNOOR" +
+ "NPPLNQ\x00\x1eNRRUNTTZNUIUNZZLOMMNPAANPCCIPEERPFYFPGNGPHHLPKAKPLOLPM\x00" +
+ "\x12PNCNPRRIPSSEPTRTPUUSPWLWPYRYPZCZQAATQMMMQNNNQOOOQPPPQQQQQRRRQSSSQTTT" +
+ "QU\x00\x03QVVVQWWWQXXXQYYYQZZZREEURHHOROOURS\x00\x15RUUSRWWASAAUSBLBSCYC" +
+ "SDDNSEWESGGPSHHNSIVNSJJMSKVKSLLESMMRSNENSOOMSRURSSSDSTTPSUUNSVLVSXXMSYYR" +
+ "SZWZTAAATCCATDCDTF\x00\x18TGGOTHHATJJKTKKLTLLSTMKMTNUNTOONTPMPTRURTTTOTV" +
+ "UVTWWNTZZAUAKRUGGAUK UMMIUN USSAUYRYUZZBVAATVCCTVDDRVEENVGGBVIIRVNNMVU" +
+ "UTWFLFWKAKWSSMXAAAXBBBXCCCXDDDXEEEXFFFXGGGXHHHXIIIXJJJXKKKXLLLXMMMXNNNXO" +
+ "OOXPPPXQQQXRRRXSSSXTTTXUUUXVVVXWWWXXXXXYYYXZZZYDMDYEEMYT\x00\x1bYUUGZAAF" +
+ "ZMMBZRARZWWEZZZZ\xff\xff\xff\xff"
+
+// altRegionISO3 holds a list of 3-letter region codes that cannot be
+// mapped to 2-letter codes using the default algorithm. This is a short list.
+const altRegionISO3 string = "SCGQUUSGSCOMPRKCYMSPMSRBATFMYTATN"
+
+// altRegionIDs holds a list of regionIDs the positions of which match those
+// of the 3-letter ISO codes in altRegionISO3.
+// Size: 22 bytes, 11 elements
+var altRegionIDs = [11]uint16{
+ 0x0057, 0x0070, 0x0088, 0x00a8, 0x00aa, 0x00ad, 0x00ea, 0x0105,
+ 0x0121, 0x015f, 0x00dc,
+}
+
+// Size: 80 bytes, 20 elements
+var regionOldMap = [20]fromTo{
+ 0: {from: 0x44, to: 0xc4},
+ 1: {from: 0x58, to: 0xa7},
+ 2: {from: 0x5f, to: 0x60},
+ 3: {from: 0x66, to: 0x3b},
+ 4: {from: 0x79, to: 0x78},
+ 5: {from: 0x93, to: 0x37},
+ 6: {from: 0xa3, to: 0x133},
+ 7: {from: 0xc1, to: 0x133},
+ 8: {from: 0xd7, to: 0x13f},
+ 9: {from: 0xdc, to: 0x2b},
+ 10: {from: 0xef, to: 0x133},
+ 11: {from: 0xf2, to: 0xe2},
+ 12: {from: 0xfc, to: 0x70},
+ 13: {from: 0x103, to: 0x164},
+ 14: {from: 0x12a, to: 0x126},
+ 15: {from: 0x132, to: 0x7b},
+ 16: {from: 0x13a, to: 0x13e},
+ 17: {from: 0x141, to: 0x133},
+ 18: {from: 0x15d, to: 0x15e},
+ 19: {from: 0x163, to: 0x4b},
+}
+
+// m49 maps regionIDs to UN.M49 codes. The first isoRegionOffset entries are
+// codes indicating collections of regions.
+// Size: 716 bytes, 358 elements
+var m49 = [358]int16{
+ // Entry 0 - 3F
+ 0, 1, 2, 3, 5, 9, 11, 13,
+ 14, 15, 17, 18, 19, 21, 29, 30,
+ 34, 35, 39, 53, 54, 57, 61, 142,
+ 143, 145, 150, 151, 154, 155, 202, 419,
+ 958, 0, 20, 784, 4, 28, 660, 8,
+ 51, 530, 24, 10, 32, 16, 40, 36,
+ 533, 248, 31, 70, 52, 50, 56, 854,
+ 100, 48, 108, 204, 652, 60, 96, 68,
+ // Entry 40 - 7F
+ 535, 76, 44, 64, 104, 74, 72, 112,
+ 84, 124, 166, 180, 140, 178, 756, 384,
+ 184, 152, 120, 156, 170, 0, 188, 891,
+ 296, 192, 132, 531, 162, 196, 203, 278,
+ 276, 0, 262, 208, 212, 214, 204, 12,
+ 0, 218, 233, 818, 732, 232, 724, 231,
+ 967, 0, 246, 242, 238, 583, 234, 0,
+ 250, 249, 266, 826, 308, 268, 254, 831,
+ // Entry 80 - BF
+ 288, 292, 304, 270, 324, 312, 226, 300,
+ 239, 320, 316, 624, 328, 344, 334, 340,
+ 191, 332, 348, 854, 0, 360, 372, 376,
+ 833, 356, 86, 368, 364, 352, 380, 832,
+ 388, 400, 392, 581, 404, 417, 116, 296,
+ 174, 659, 408, 410, 414, 136, 398, 418,
+ 422, 662, 438, 144, 430, 426, 440, 442,
+ 428, 434, 504, 492, 498, 499, 663, 450,
+ // Entry C0 - FF
+ 584, 581, 807, 466, 104, 496, 446, 580,
+ 474, 478, 500, 470, 480, 462, 454, 484,
+ 458, 508, 516, 540, 562, 574, 566, 548,
+ 558, 528, 578, 524, 10, 520, 536, 570,
+ 554, 512, 591, 0, 604, 258, 598, 608,
+ 586, 616, 666, 612, 630, 275, 620, 581,
+ 585, 600, 591, 634, 959, 960, 961, 962,
+ 963, 964, 965, 966, 967, 968, 969, 970,
+ // Entry 100 - 13F
+ 971, 972, 638, 716, 642, 688, 643, 646,
+ 682, 90, 690, 729, 752, 702, 654, 705,
+ 744, 703, 694, 674, 686, 706, 740, 728,
+ 678, 810, 222, 534, 760, 748, 0, 796,
+ 148, 260, 768, 764, 762, 772, 626, 795,
+ 788, 776, 626, 792, 780, 798, 158, 834,
+ 804, 800, 826, 581, 0, 840, 858, 860,
+ 336, 670, 704, 862, 92, 850, 704, 548,
+ // Entry 140 - 17F
+ 876, 581, 882, 973, 974, 975, 976, 977,
+ 978, 979, 980, 981, 982, 983, 984, 985,
+ 986, 987, 988, 989, 990, 991, 992, 993,
+ 994, 995, 996, 997, 998, 720, 887, 175,
+ 891, 710, 894, 180, 716, 999,
+}
+
+// m49Index gives indexes into fromM49 based on the three most significant bits
+// of a 10-bit UN.M49 code. To search an UN.M49 code in fromM49, search in
+// fromM49[m49Index[msb39(code)]:m49Index[msb3(code)+1]]
+// for an entry where the first 7 bits match the 7 lsb of the UN.M49 code.
+// The region code is stored in the 9 lsb of the indexed value.
+// Size: 18 bytes, 9 elements
+var m49Index = [9]int16{
+ 0, 59, 108, 143, 181, 220, 259, 291,
+ 333,
+}
+
+// fromM49 contains entries to map UN.M49 codes to regions. See m49Index for details.
+// Size: 666 bytes, 333 elements
+var fromM49 = [333]uint16{
+ // Entry 0 - 3F
+ 0x0201, 0x0402, 0x0603, 0x0824, 0x0a04, 0x1027, 0x1205, 0x142b,
+ 0x1606, 0x1867, 0x1a07, 0x1c08, 0x1e09, 0x202d, 0x220a, 0x240b,
+ 0x260c, 0x2822, 0x2a0d, 0x302a, 0x3825, 0x3a0e, 0x3c0f, 0x3e32,
+ 0x402c, 0x4410, 0x4611, 0x482f, 0x4e12, 0x502e, 0x5842, 0x6039,
+ 0x6435, 0x6628, 0x6834, 0x6a13, 0x6c14, 0x7036, 0x7215, 0x783d,
+ 0x7a16, 0x8043, 0x883f, 0x8c33, 0x9046, 0x9445, 0x9841, 0xa848,
+ 0xac9a, 0xb509, 0xb93c, 0xc03e, 0xc838, 0xd0c4, 0xd83a, 0xe047,
+ 0xe8a6, 0xf052, 0xf849, 0x085a, 0x10ad, 0x184c, 0x1c17, 0x1e18,
+ // Entry 40 - 7F
+ 0x20b3, 0x2219, 0x2920, 0x2c1a, 0x2e1b, 0x3051, 0x341c, 0x361d,
+ 0x3853, 0x3d2e, 0x445c, 0x4c4a, 0x5454, 0x5ca8, 0x5f5f, 0x644d,
+ 0x684b, 0x7050, 0x7856, 0x7e90, 0x8059, 0x885d, 0x941e, 0x965e,
+ 0x983b, 0xa063, 0xa864, 0xac65, 0xb469, 0xbd1a, 0xc486, 0xcc6f,
+ 0xce6f, 0xd06d, 0xd26a, 0xd476, 0xdc74, 0xde88, 0xe473, 0xec72,
+ 0xf031, 0xf279, 0xf478, 0xfc7e, 0x04e5, 0x0921, 0x0c62, 0x147a,
+ 0x187d, 0x1c83, 0x26ed, 0x2860, 0x2c5f, 0x3060, 0x4080, 0x4881,
+ 0x50a7, 0x5887, 0x6082, 0x687c, 0x7085, 0x788a, 0x8089, 0x8884,
+ // Entry 80 - BF
+ 0x908c, 0x9891, 0x9c8e, 0xa138, 0xa88f, 0xb08d, 0xb892, 0xc09d,
+ 0xc899, 0xd095, 0xd89c, 0xe09b, 0xe896, 0xf097, 0xf89e, 0x004f,
+ 0x08a0, 0x10a2, 0x1cae, 0x20a1, 0x28a4, 0x30aa, 0x34ab, 0x3cac,
+ 0x42a5, 0x44af, 0x461f, 0x4cb0, 0x54b5, 0x58b8, 0x5cb4, 0x64b9,
+ 0x6cb2, 0x70b6, 0x74b7, 0x7cc6, 0x84bf, 0x8cce, 0x94d0, 0x9ccd,
+ 0xa4c3, 0xaccb, 0xb4c8, 0xbcc9, 0xc0cc, 0xc8cf, 0xd8bb, 0xe0c5,
+ 0xe4bc, 0xe6bd, 0xe8ca, 0xf0ba, 0xf8d1, 0x00e1, 0x08d2, 0x10dd,
+ 0x18db, 0x20d9, 0x2429, 0x265b, 0x2a30, 0x2d1b, 0x2e40, 0x30de,
+ // Entry C0 - FF
+ 0x38d3, 0x493f, 0x54e0, 0x5cd8, 0x64d4, 0x6cd6, 0x74df, 0x7cd5,
+ 0x84da, 0x88c7, 0x8b33, 0x8e75, 0x90c0, 0x92f0, 0x94e8, 0x9ee2,
+ 0xace6, 0xb0f1, 0xb8e4, 0xc0e7, 0xc8eb, 0xd0e9, 0xd8ee, 0xe08b,
+ 0xe526, 0xecec, 0xf4f3, 0xfd02, 0x0504, 0x0706, 0x0d07, 0x183c,
+ 0x1d0e, 0x26a9, 0x2826, 0x2cb1, 0x2ebe, 0x34ea, 0x3d39, 0x4513,
+ 0x4d18, 0x5508, 0x5d14, 0x6105, 0x650a, 0x6d12, 0x7d0d, 0x7f11,
+ 0x813e, 0x830f, 0x8515, 0x8d61, 0x9964, 0xa15d, 0xa86e, 0xb117,
+ 0xb30b, 0xb86c, 0xc10b, 0xc916, 0xd110, 0xd91d, 0xe10c, 0xe84e,
+ // Entry 100 - 13F
+ 0xf11c, 0xf524, 0xf923, 0x0122, 0x0925, 0x1129, 0x192c, 0x2023,
+ 0x2928, 0x312b, 0x3727, 0x391f, 0x3d2d, 0x4131, 0x4930, 0x4ec2,
+ 0x5519, 0x646b, 0x747b, 0x7e7f, 0x809f, 0x8298, 0x852f, 0x9135,
+ 0xa53d, 0xac37, 0xb536, 0xb937, 0xbd3b, 0xd940, 0xe542, 0xed5e,
+ 0xef5e, 0xf657, 0xfd62, 0x7c20, 0x7ef4, 0x80f5, 0x82f6, 0x84f7,
+ 0x86f8, 0x88f9, 0x8afa, 0x8cfb, 0x8e70, 0x90fd, 0x92fe, 0x94ff,
+ 0x9700, 0x9901, 0x9b43, 0x9d44, 0x9f45, 0xa146, 0xa347, 0xa548,
+ 0xa749, 0xa94a, 0xab4b, 0xad4c, 0xaf4d, 0xb14e, 0xb34f, 0xb550,
+ // Entry 140 - 17F
+ 0xb751, 0xb952, 0xbb53, 0xbd54, 0xbf55, 0xc156, 0xc357, 0xc558,
+ 0xc759, 0xc95a, 0xcb5b, 0xcd5c, 0xcf65,
+}
+
+// Size: 1615 bytes
+var variantIndex = map[string]uint8{
+ "1606nict": 0x0,
+ "1694acad": 0x1,
+ "1901": 0x2,
+ "1959acad": 0x3,
+ "1994": 0x4d,
+ "1996": 0x4,
+ "abl1943": 0x5,
+ "akuapem": 0x6,
+ "alalc97": 0x4f,
+ "aluku": 0x7,
+ "ao1990": 0x8,
+ "arevela": 0x9,
+ "arevmda": 0xa,
+ "asante": 0xb,
+ "baku1926": 0xc,
+ "balanka": 0xd,
+ "barla": 0xe,
+ "basiceng": 0xf,
+ "bauddha": 0x10,
+ "biscayan": 0x11,
+ "biske": 0x48,
+ "bohoric": 0x12,
+ "boont": 0x13,
+ "colb1945": 0x14,
+ "cornu": 0x15,
+ "dajnko": 0x16,
+ "ekavsk": 0x17,
+ "emodeng": 0x18,
+ "fonipa": 0x50,
+ "fonnapa": 0x51,
+ "fonupa": 0x52,
+ "fonxsamp": 0x53,
+ "hepburn": 0x19,
+ "heploc": 0x4e,
+ "hognorsk": 0x1a,
+ "hsistemo": 0x1b,
+ "ijekavsk": 0x1c,
+ "itihasa": 0x1d,
+ "jauer": 0x1e,
+ "jyutping": 0x1f,
+ "kkcor": 0x20,
+ "kociewie": 0x21,
+ "kscor": 0x22,
+ "laukika": 0x23,
+ "lipaw": 0x49,
+ "luna1918": 0x24,
+ "metelko": 0x25,
+ "monoton": 0x26,
+ "ndyuka": 0x27,
+ "nedis": 0x28,
+ "newfound": 0x29,
+ "njiva": 0x4a,
+ "nulik": 0x2a,
+ "osojs": 0x4b,
+ "oxendict": 0x2b,
+ "pahawh2": 0x2c,
+ "pahawh3": 0x2d,
+ "pahawh4": 0x2e,
+ "pamaka": 0x2f,
+ "petr1708": 0x30,
+ "pinyin": 0x31,
+ "polyton": 0x32,
+ "puter": 0x33,
+ "rigik": 0x34,
+ "rozaj": 0x35,
+ "rumgr": 0x36,
+ "scotland": 0x37,
+ "scouse": 0x38,
+ "simple": 0x54,
+ "solba": 0x4c,
+ "sotav": 0x39,
+ "spanglis": 0x3a,
+ "surmiran": 0x3b,
+ "sursilv": 0x3c,
+ "sutsilv": 0x3d,
+ "tarask": 0x3e,
+ "uccor": 0x3f,
+ "ucrcor": 0x40,
+ "ulster": 0x41,
+ "unifon": 0x42,
+ "vaidika": 0x43,
+ "valencia": 0x44,
+ "vallader": 0x45,
+ "wadegile": 0x46,
+ "xsistemo": 0x47,
+}
+
+// variantNumSpecialized is the number of specialized variants in variants.
+const variantNumSpecialized = 79
+
+// nRegionGroups is the number of region groups.
+const nRegionGroups = 33
+
+type likelyLangRegion struct {
+ lang uint16
+ region uint16
+}
+
+// likelyScript is a lookup table, indexed by scriptID, for the most likely
+// languages and regions given a script.
+// Size: 976 bytes, 244 elements
+var likelyScript = [244]likelyLangRegion{
+ 1: {lang: 0x14e, region: 0x84},
+ 3: {lang: 0x2a2, region: 0x106},
+ 4: {lang: 0x1f, region: 0x99},
+ 5: {lang: 0x3a, region: 0x6b},
+ 7: {lang: 0x3b, region: 0x9c},
+ 8: {lang: 0x1d7, region: 0x28},
+ 9: {lang: 0x13, region: 0x9c},
+ 10: {lang: 0x5b, region: 0x95},
+ 11: {lang: 0x60, region: 0x52},
+ 12: {lang: 0xb9, region: 0xb4},
+ 13: {lang: 0x63, region: 0x95},
+ 14: {lang: 0xa5, region: 0x35},
+ 15: {lang: 0x3e9, region: 0x99},
+ 17: {lang: 0x529, region: 0x12e},
+ 18: {lang: 0x3b1, region: 0x99},
+ 19: {lang: 0x15e, region: 0x78},
+ 20: {lang: 0xc2, region: 0x95},
+ 21: {lang: 0x9d, region: 0xe7},
+ 22: {lang: 0xdb, region: 0x35},
+ 23: {lang: 0xf3, region: 0x49},
+ 24: {lang: 0x4f0, region: 0x12b},
+ 25: {lang: 0xe7, region: 0x13e},
+ 26: {lang: 0xe5, region: 0x135},
+ 28: {lang: 0xf1, region: 0x6b},
+ 30: {lang: 0x1a0, region: 0x5d},
+ 31: {lang: 0x3e2, region: 0x106},
+ 33: {lang: 0x1be, region: 0x99},
+ 36: {lang: 0x15e, region: 0x78},
+ 39: {lang: 0x133, region: 0x6b},
+ 40: {lang: 0x431, region: 0x27},
+ 41: {lang: 0x27, region: 0x6f},
+ 43: {lang: 0x210, region: 0x7d},
+ 44: {lang: 0xfe, region: 0x38},
+ 46: {lang: 0x19b, region: 0x99},
+ 47: {lang: 0x19e, region: 0x130},
+ 48: {lang: 0x3e9, region: 0x99},
+ 49: {lang: 0x136, region: 0x87},
+ 50: {lang: 0x1a4, region: 0x99},
+ 51: {lang: 0x39d, region: 0x99},
+ 52: {lang: 0x529, region: 0x12e},
+ 53: {lang: 0x254, region: 0xab},
+ 54: {lang: 0x529, region: 0x53},
+ 55: {lang: 0x1cb, region: 0xe7},
+ 56: {lang: 0x529, region: 0x53},
+ 57: {lang: 0x529, region: 0x12e},
+ 58: {lang: 0x2fd, region: 0x9b},
+ 59: {lang: 0x1bc, region: 0x97},
+ 60: {lang: 0x200, region: 0xa2},
+ 61: {lang: 0x1c5, region: 0x12b},
+ 62: {lang: 0x1ca, region: 0xaf},
+ 65: {lang: 0x1d5, region: 0x92},
+ 67: {lang: 0x142, region: 0x9e},
+ 68: {lang: 0x254, region: 0xab},
+ 69: {lang: 0x20e, region: 0x95},
+ 70: {lang: 0x200, region: 0xa2},
+ 72: {lang: 0x135, region: 0xc4},
+ 73: {lang: 0x200, region: 0xa2},
+ 74: {lang: 0x3bb, region: 0xe8},
+ 75: {lang: 0x24a, region: 0xa6},
+ 76: {lang: 0x3fa, region: 0x99},
+ 79: {lang: 0x251, region: 0x99},
+ 80: {lang: 0x254, region: 0xab},
+ 82: {lang: 0x88, region: 0x99},
+ 83: {lang: 0x370, region: 0x123},
+ 84: {lang: 0x2b8, region: 0xaf},
+ 89: {lang: 0x29f, region: 0x99},
+ 90: {lang: 0x2a8, region: 0x99},
+ 91: {lang: 0x28f, region: 0x87},
+ 92: {lang: 0x1a0, region: 0x87},
+ 93: {lang: 0x2ac, region: 0x53},
+ 95: {lang: 0x4f4, region: 0x12b},
+ 96: {lang: 0x4f5, region: 0x12b},
+ 97: {lang: 0x1be, region: 0x99},
+ 99: {lang: 0x337, region: 0x9c},
+ 100: {lang: 0x4f7, region: 0x53},
+ 101: {lang: 0xa9, region: 0x53},
+ 104: {lang: 0x2e8, region: 0x112},
+ 105: {lang: 0x4f8, region: 0x10b},
+ 106: {lang: 0x4f8, region: 0x10b},
+ 107: {lang: 0x304, region: 0x99},
+ 108: {lang: 0x31b, region: 0x99},
+ 109: {lang: 0x30b, region: 0x53},
+ 111: {lang: 0x31e, region: 0x35},
+ 112: {lang: 0x30e, region: 0x99},
+ 113: {lang: 0x414, region: 0xe8},
+ 114: {lang: 0x331, region: 0xc4},
+ 115: {lang: 0x4f9, region: 0x108},
+ 116: {lang: 0x3b, region: 0xa1},
+ 117: {lang: 0x353, region: 0xdb},
+ 120: {lang: 0x2d0, region: 0x84},
+ 121: {lang: 0x52a, region: 0x53},
+ 122: {lang: 0x403, region: 0x96},
+ 123: {lang: 0x3ee, region: 0x99},
+ 124: {lang: 0x39b, region: 0xc5},
+ 125: {lang: 0x395, region: 0x99},
+ 126: {lang: 0x399, region: 0x135},
+ 127: {lang: 0x429, region: 0x115},
+ 128: {lang: 0x3b, region: 0x11c},
+ 129: {lang: 0xfd, region: 0xc4},
+ 130: {lang: 0x27d, region: 0x106},
+ 131: {lang: 0x2c9, region: 0x53},
+ 132: {lang: 0x39f, region: 0x9c},
+ 133: {lang: 0x39f, region: 0x53},
+ 135: {lang: 0x3ad, region: 0xb0},
+ 137: {lang: 0x1c6, region: 0x53},
+ 138: {lang: 0x4fd, region: 0x9c},
+ 189: {lang: 0x3cb, region: 0x95},
+ 191: {lang: 0x372, region: 0x10c},
+ 192: {lang: 0x420, region: 0x97},
+ 194: {lang: 0x4ff, region: 0x15e},
+ 195: {lang: 0x3f0, region: 0x99},
+ 196: {lang: 0x45, region: 0x135},
+ 197: {lang: 0x139, region: 0x7b},
+ 198: {lang: 0x3e9, region: 0x99},
+ 200: {lang: 0x3e9, region: 0x99},
+ 201: {lang: 0x3fa, region: 0x99},
+ 202: {lang: 0x40c, region: 0xb3},
+ 203: {lang: 0x433, region: 0x99},
+ 204: {lang: 0xef, region: 0xc5},
+ 205: {lang: 0x43e, region: 0x95},
+ 206: {lang: 0x44d, region: 0x35},
+ 207: {lang: 0x44e, region: 0x9b},
+ 211: {lang: 0x45a, region: 0xe7},
+ 212: {lang: 0x11a, region: 0x99},
+ 213: {lang: 0x45e, region: 0x53},
+ 214: {lang: 0x232, region: 0x53},
+ 215: {lang: 0x450, region: 0x99},
+ 216: {lang: 0x4a5, region: 0x53},
+ 217: {lang: 0x9f, region: 0x13e},
+ 218: {lang: 0x461, region: 0x99},
+ 220: {lang: 0x528, region: 0xba},
+ 221: {lang: 0x153, region: 0xe7},
+ 222: {lang: 0x128, region: 0xcd},
+ 223: {lang: 0x46b, region: 0x123},
+ 224: {lang: 0xa9, region: 0x53},
+ 225: {lang: 0x2ce, region: 0x99},
+ 226: {lang: 0x4ad, region: 0x11c},
+ 227: {lang: 0x4be, region: 0xb4},
+ 229: {lang: 0x1ce, region: 0x99},
+ 232: {lang: 0x3a9, region: 0x9c},
+ 233: {lang: 0x22, region: 0x9b},
+ 234: {lang: 0x1ea, region: 0x53},
+ 235: {lang: 0xef, region: 0xc5},
+}
+
+type likelyScriptRegion struct {
+ region uint16
+ script uint8
+ flags uint8
+}
+
+// likelyLang is a lookup table, indexed by langID, for the most likely
+// scripts and regions given incomplete information. If more entries exist for a
+// given language, region and script are the index and size respectively
+// of the list in likelyLangList.
+// Size: 5320 bytes, 1330 elements
+var likelyLang = [1330]likelyScriptRegion{
+ 0: {region: 0x135, script: 0x57, flags: 0x0},
+ 1: {region: 0x6f, script: 0x57, flags: 0x0},
+ 2: {region: 0x165, script: 0x57, flags: 0x0},
+ 3: {region: 0x165, script: 0x57, flags: 0x0},
+ 4: {region: 0x165, script: 0x57, flags: 0x0},
+ 5: {region: 0x7d, script: 0x1f, flags: 0x0},
+ 6: {region: 0x165, script: 0x57, flags: 0x0},
+ 7: {region: 0x165, script: 0x1f, flags: 0x0},
+ 8: {region: 0x80, script: 0x57, flags: 0x0},
+ 9: {region: 0x165, script: 0x57, flags: 0x0},
+ 10: {region: 0x165, script: 0x57, flags: 0x0},
+ 11: {region: 0x165, script: 0x57, flags: 0x0},
+ 12: {region: 0x95, script: 0x57, flags: 0x0},
+ 13: {region: 0x131, script: 0x57, flags: 0x0},
+ 14: {region: 0x80, script: 0x57, flags: 0x0},
+ 15: {region: 0x165, script: 0x57, flags: 0x0},
+ 16: {region: 0x165, script: 0x57, flags: 0x0},
+ 17: {region: 0x106, script: 0x1f, flags: 0x0},
+ 18: {region: 0x165, script: 0x57, flags: 0x0},
+ 19: {region: 0x9c, script: 0x9, flags: 0x0},
+ 20: {region: 0x128, script: 0x5, flags: 0x0},
+ 21: {region: 0x165, script: 0x57, flags: 0x0},
+ 22: {region: 0x161, script: 0x57, flags: 0x0},
+ 23: {region: 0x165, script: 0x57, flags: 0x0},
+ 24: {region: 0x165, script: 0x57, flags: 0x0},
+ 25: {region: 0x165, script: 0x57, flags: 0x0},
+ 26: {region: 0x165, script: 0x57, flags: 0x0},
+ 27: {region: 0x165, script: 0x57, flags: 0x0},
+ 28: {region: 0x52, script: 0x57, flags: 0x0},
+ 29: {region: 0x165, script: 0x57, flags: 0x0},
+ 30: {region: 0x165, script: 0x57, flags: 0x0},
+ 31: {region: 0x99, script: 0x4, flags: 0x0},
+ 32: {region: 0x165, script: 0x57, flags: 0x0},
+ 33: {region: 0x80, script: 0x57, flags: 0x0},
+ 34: {region: 0x9b, script: 0xe9, flags: 0x0},
+ 35: {region: 0x165, script: 0x57, flags: 0x0},
+ 36: {region: 0x165, script: 0x57, flags: 0x0},
+ 37: {region: 0x14d, script: 0x57, flags: 0x0},
+ 38: {region: 0x106, script: 0x1f, flags: 0x0},
+ 39: {region: 0x6f, script: 0x29, flags: 0x0},
+ 40: {region: 0x165, script: 0x57, flags: 0x0},
+ 41: {region: 0x165, script: 0x57, flags: 0x0},
+ 42: {region: 0xd6, script: 0x57, flags: 0x0},
+ 43: {region: 0x165, script: 0x57, flags: 0x0},
+ 45: {region: 0x165, script: 0x57, flags: 0x0},
+ 46: {region: 0x165, script: 0x57, flags: 0x0},
+ 47: {region: 0x165, script: 0x57, flags: 0x0},
+ 48: {region: 0x165, script: 0x57, flags: 0x0},
+ 49: {region: 0x165, script: 0x57, flags: 0x0},
+ 50: {region: 0x165, script: 0x57, flags: 0x0},
+ 51: {region: 0x95, script: 0x57, flags: 0x0},
+ 52: {region: 0x165, script: 0x5, flags: 0x0},
+ 53: {region: 0x122, script: 0x5, flags: 0x0},
+ 54: {region: 0x165, script: 0x57, flags: 0x0},
+ 55: {region: 0x165, script: 0x57, flags: 0x0},
+ 56: {region: 0x165, script: 0x57, flags: 0x0},
+ 57: {region: 0x165, script: 0x57, flags: 0x0},
+ 58: {region: 0x6b, script: 0x5, flags: 0x0},
+ 59: {region: 0x0, script: 0x3, flags: 0x1},
+ 60: {region: 0x165, script: 0x57, flags: 0x0},
+ 61: {region: 0x51, script: 0x57, flags: 0x0},
+ 62: {region: 0x3f, script: 0x57, flags: 0x0},
+ 63: {region: 0x67, script: 0x5, flags: 0x0},
+ 65: {region: 0xba, script: 0x5, flags: 0x0},
+ 66: {region: 0x6b, script: 0x5, flags: 0x0},
+ 67: {region: 0x99, script: 0xe, flags: 0x0},
+ 68: {region: 0x12f, script: 0x57, flags: 0x0},
+ 69: {region: 0x135, script: 0xc4, flags: 0x0},
+ 70: {region: 0x165, script: 0x57, flags: 0x0},
+ 71: {region: 0x165, script: 0x57, flags: 0x0},
+ 72: {region: 0x6e, script: 0x57, flags: 0x0},
+ 73: {region: 0x165, script: 0x57, flags: 0x0},
+ 74: {region: 0x165, script: 0x57, flags: 0x0},
+ 75: {region: 0x49, script: 0x57, flags: 0x0},
+ 76: {region: 0x165, script: 0x57, flags: 0x0},
+ 77: {region: 0x106, script: 0x1f, flags: 0x0},
+ 78: {region: 0x165, script: 0x5, flags: 0x0},
+ 79: {region: 0x165, script: 0x57, flags: 0x0},
+ 80: {region: 0x165, script: 0x57, flags: 0x0},
+ 81: {region: 0x165, script: 0x57, flags: 0x0},
+ 82: {region: 0x99, script: 0x21, flags: 0x0},
+ 83: {region: 0x165, script: 0x57, flags: 0x0},
+ 84: {region: 0x165, script: 0x57, flags: 0x0},
+ 85: {region: 0x165, script: 0x57, flags: 0x0},
+ 86: {region: 0x3f, script: 0x57, flags: 0x0},
+ 87: {region: 0x165, script: 0x57, flags: 0x0},
+ 88: {region: 0x3, script: 0x5, flags: 0x1},
+ 89: {region: 0x106, script: 0x1f, flags: 0x0},
+ 90: {region: 0xe8, script: 0x5, flags: 0x0},
+ 91: {region: 0x95, script: 0x57, flags: 0x0},
+ 92: {region: 0xdb, script: 0x21, flags: 0x0},
+ 93: {region: 0x2e, script: 0x57, flags: 0x0},
+ 94: {region: 0x52, script: 0x57, flags: 0x0},
+ 95: {region: 0x165, script: 0x57, flags: 0x0},
+ 96: {region: 0x52, script: 0xb, flags: 0x0},
+ 97: {region: 0x165, script: 0x57, flags: 0x0},
+ 98: {region: 0x165, script: 0x57, flags: 0x0},
+ 99: {region: 0x95, script: 0x57, flags: 0x0},
+ 100: {region: 0x165, script: 0x57, flags: 0x0},
+ 101: {region: 0x52, script: 0x57, flags: 0x0},
+ 102: {region: 0x165, script: 0x57, flags: 0x0},
+ 103: {region: 0x165, script: 0x57, flags: 0x0},
+ 104: {region: 0x165, script: 0x57, flags: 0x0},
+ 105: {region: 0x165, script: 0x57, flags: 0x0},
+ 106: {region: 0x4f, script: 0x57, flags: 0x0},
+ 107: {region: 0x165, script: 0x57, flags: 0x0},
+ 108: {region: 0x165, script: 0x57, flags: 0x0},
+ 109: {region: 0x165, script: 0x57, flags: 0x0},
+ 110: {region: 0x165, script: 0x29, flags: 0x0},
+ 111: {region: 0x165, script: 0x57, flags: 0x0},
+ 112: {region: 0x165, script: 0x57, flags: 0x0},
+ 113: {region: 0x47, script: 0x1f, flags: 0x0},
+ 114: {region: 0x165, script: 0x57, flags: 0x0},
+ 115: {region: 0x165, script: 0x57, flags: 0x0},
+ 116: {region: 0x10b, script: 0x5, flags: 0x0},
+ 117: {region: 0x162, script: 0x57, flags: 0x0},
+ 118: {region: 0x165, script: 0x57, flags: 0x0},
+ 119: {region: 0x95, script: 0x57, flags: 0x0},
+ 120: {region: 0x165, script: 0x57, flags: 0x0},
+ 121: {region: 0x12f, script: 0x57, flags: 0x0},
+ 122: {region: 0x52, script: 0x57, flags: 0x0},
+ 123: {region: 0x99, script: 0xd7, flags: 0x0},
+ 124: {region: 0xe8, script: 0x5, flags: 0x0},
+ 125: {region: 0x99, script: 0x21, flags: 0x0},
+ 126: {region: 0x38, script: 0x1f, flags: 0x0},
+ 127: {region: 0x99, script: 0x21, flags: 0x0},
+ 128: {region: 0xe8, script: 0x5, flags: 0x0},
+ 129: {region: 0x12b, script: 0x31, flags: 0x0},
+ 131: {region: 0x99, script: 0x21, flags: 0x0},
+ 132: {region: 0x165, script: 0x57, flags: 0x0},
+ 133: {region: 0x99, script: 0x21, flags: 0x0},
+ 134: {region: 0xe7, script: 0x57, flags: 0x0},
+ 135: {region: 0x165, script: 0x57, flags: 0x0},
+ 136: {region: 0x99, script: 0x21, flags: 0x0},
+ 137: {region: 0x165, script: 0x57, flags: 0x0},
+ 138: {region: 0x13f, script: 0x57, flags: 0x0},
+ 139: {region: 0x165, script: 0x57, flags: 0x0},
+ 140: {region: 0x165, script: 0x57, flags: 0x0},
+ 141: {region: 0xe7, script: 0x57, flags: 0x0},
+ 142: {region: 0x165, script: 0x57, flags: 0x0},
+ 143: {region: 0xd6, script: 0x57, flags: 0x0},
+ 144: {region: 0x165, script: 0x57, flags: 0x0},
+ 145: {region: 0x165, script: 0x57, flags: 0x0},
+ 146: {region: 0x165, script: 0x57, flags: 0x0},
+ 147: {region: 0x165, script: 0x29, flags: 0x0},
+ 148: {region: 0x99, script: 0x21, flags: 0x0},
+ 149: {region: 0x95, script: 0x57, flags: 0x0},
+ 150: {region: 0x165, script: 0x57, flags: 0x0},
+ 151: {region: 0x165, script: 0x57, flags: 0x0},
+ 152: {region: 0x114, script: 0x57, flags: 0x0},
+ 153: {region: 0x165, script: 0x57, flags: 0x0},
+ 154: {region: 0x165, script: 0x57, flags: 0x0},
+ 155: {region: 0x52, script: 0x57, flags: 0x0},
+ 156: {region: 0x165, script: 0x57, flags: 0x0},
+ 157: {region: 0xe7, script: 0x57, flags: 0x0},
+ 158: {region: 0x165, script: 0x57, flags: 0x0},
+ 159: {region: 0x13e, script: 0xd9, flags: 0x0},
+ 160: {region: 0xc3, script: 0x57, flags: 0x0},
+ 161: {region: 0x165, script: 0x57, flags: 0x0},
+ 162: {region: 0x165, script: 0x57, flags: 0x0},
+ 163: {region: 0xc3, script: 0x57, flags: 0x0},
+ 164: {region: 0x165, script: 0x57, flags: 0x0},
+ 165: {region: 0x35, script: 0xe, flags: 0x0},
+ 166: {region: 0x165, script: 0x57, flags: 0x0},
+ 167: {region: 0x165, script: 0x57, flags: 0x0},
+ 168: {region: 0x165, script: 0x57, flags: 0x0},
+ 169: {region: 0x53, script: 0xe0, flags: 0x0},
+ 170: {region: 0x165, script: 0x57, flags: 0x0},
+ 171: {region: 0x165, script: 0x57, flags: 0x0},
+ 172: {region: 0x165, script: 0x57, flags: 0x0},
+ 173: {region: 0x99, script: 0xe, flags: 0x0},
+ 174: {region: 0x165, script: 0x57, flags: 0x0},
+ 175: {region: 0x9c, script: 0x5, flags: 0x0},
+ 176: {region: 0x165, script: 0x57, flags: 0x0},
+ 177: {region: 0x4f, script: 0x57, flags: 0x0},
+ 178: {region: 0x78, script: 0x57, flags: 0x0},
+ 179: {region: 0x99, script: 0x21, flags: 0x0},
+ 180: {region: 0xe8, script: 0x5, flags: 0x0},
+ 181: {region: 0x99, script: 0x21, flags: 0x0},
+ 182: {region: 0x165, script: 0x57, flags: 0x0},
+ 183: {region: 0x33, script: 0x57, flags: 0x0},
+ 184: {region: 0x165, script: 0x57, flags: 0x0},
+ 185: {region: 0xb4, script: 0xc, flags: 0x0},
+ 186: {region: 0x52, script: 0x57, flags: 0x0},
+ 187: {region: 0x165, script: 0x29, flags: 0x0},
+ 188: {region: 0xe7, script: 0x57, flags: 0x0},
+ 189: {region: 0x165, script: 0x57, flags: 0x0},
+ 190: {region: 0xe8, script: 0x21, flags: 0x0},
+ 191: {region: 0x106, script: 0x1f, flags: 0x0},
+ 192: {region: 0x15f, script: 0x57, flags: 0x0},
+ 193: {region: 0x165, script: 0x57, flags: 0x0},
+ 194: {region: 0x95, script: 0x57, flags: 0x0},
+ 195: {region: 0x165, script: 0x57, flags: 0x0},
+ 196: {region: 0x52, script: 0x57, flags: 0x0},
+ 197: {region: 0x165, script: 0x57, flags: 0x0},
+ 198: {region: 0x165, script: 0x57, flags: 0x0},
+ 199: {region: 0x165, script: 0x57, flags: 0x0},
+ 200: {region: 0x86, script: 0x57, flags: 0x0},
+ 201: {region: 0x165, script: 0x57, flags: 0x0},
+ 202: {region: 0x165, script: 0x57, flags: 0x0},
+ 203: {region: 0x165, script: 0x57, flags: 0x0},
+ 204: {region: 0x165, script: 0x57, flags: 0x0},
+ 205: {region: 0x6d, script: 0x29, flags: 0x0},
+ 206: {region: 0x165, script: 0x57, flags: 0x0},
+ 207: {region: 0x165, script: 0x57, flags: 0x0},
+ 208: {region: 0x52, script: 0x57, flags: 0x0},
+ 209: {region: 0x165, script: 0x57, flags: 0x0},
+ 210: {region: 0x165, script: 0x57, flags: 0x0},
+ 211: {region: 0xc3, script: 0x57, flags: 0x0},
+ 212: {region: 0x165, script: 0x57, flags: 0x0},
+ 213: {region: 0x165, script: 0x57, flags: 0x0},
+ 214: {region: 0x165, script: 0x57, flags: 0x0},
+ 215: {region: 0x6e, script: 0x57, flags: 0x0},
+ 216: {region: 0x165, script: 0x57, flags: 0x0},
+ 217: {region: 0x165, script: 0x57, flags: 0x0},
+ 218: {region: 0xd6, script: 0x57, flags: 0x0},
+ 219: {region: 0x35, script: 0x16, flags: 0x0},
+ 220: {region: 0x106, script: 0x1f, flags: 0x0},
+ 221: {region: 0xe7, script: 0x57, flags: 0x0},
+ 222: {region: 0x165, script: 0x57, flags: 0x0},
+ 223: {region: 0x131, script: 0x57, flags: 0x0},
+ 224: {region: 0x8a, script: 0x57, flags: 0x0},
+ 225: {region: 0x75, script: 0x57, flags: 0x0},
+ 226: {region: 0x106, script: 0x1f, flags: 0x0},
+ 227: {region: 0x135, script: 0x57, flags: 0x0},
+ 228: {region: 0x49, script: 0x57, flags: 0x0},
+ 229: {region: 0x135, script: 0x1a, flags: 0x0},
+ 230: {region: 0xa6, script: 0x5, flags: 0x0},
+ 231: {region: 0x13e, script: 0x19, flags: 0x0},
+ 232: {region: 0x165, script: 0x57, flags: 0x0},
+ 233: {region: 0x9b, script: 0x5, flags: 0x0},
+ 234: {region: 0x165, script: 0x57, flags: 0x0},
+ 235: {region: 0x165, script: 0x57, flags: 0x0},
+ 236: {region: 0x165, script: 0x57, flags: 0x0},
+ 237: {region: 0x165, script: 0x57, flags: 0x0},
+ 238: {region: 0x165, script: 0x57, flags: 0x0},
+ 239: {region: 0xc5, script: 0xcc, flags: 0x0},
+ 240: {region: 0x78, script: 0x57, flags: 0x0},
+ 241: {region: 0x6b, script: 0x1c, flags: 0x0},
+ 242: {region: 0xe7, script: 0x57, flags: 0x0},
+ 243: {region: 0x49, script: 0x17, flags: 0x0},
+ 244: {region: 0x130, script: 0x1f, flags: 0x0},
+ 245: {region: 0x49, script: 0x17, flags: 0x0},
+ 246: {region: 0x49, script: 0x17, flags: 0x0},
+ 247: {region: 0x49, script: 0x17, flags: 0x0},
+ 248: {region: 0x49, script: 0x17, flags: 0x0},
+ 249: {region: 0x10a, script: 0x57, flags: 0x0},
+ 250: {region: 0x5e, script: 0x57, flags: 0x0},
+ 251: {region: 0xe9, script: 0x57, flags: 0x0},
+ 252: {region: 0x49, script: 0x17, flags: 0x0},
+ 253: {region: 0xc4, script: 0x81, flags: 0x0},
+ 254: {region: 0x8, script: 0x2, flags: 0x1},
+ 255: {region: 0x106, script: 0x1f, flags: 0x0},
+ 256: {region: 0x7b, script: 0x57, flags: 0x0},
+ 257: {region: 0x63, script: 0x57, flags: 0x0},
+ 258: {region: 0x165, script: 0x57, flags: 0x0},
+ 259: {region: 0x165, script: 0x57, flags: 0x0},
+ 260: {region: 0x165, script: 0x57, flags: 0x0},
+ 261: {region: 0x165, script: 0x57, flags: 0x0},
+ 262: {region: 0x135, script: 0x57, flags: 0x0},
+ 263: {region: 0x106, script: 0x1f, flags: 0x0},
+ 264: {region: 0xa4, script: 0x57, flags: 0x0},
+ 265: {region: 0x165, script: 0x57, flags: 0x0},
+ 266: {region: 0x165, script: 0x57, flags: 0x0},
+ 267: {region: 0x99, script: 0x5, flags: 0x0},
+ 268: {region: 0x165, script: 0x57, flags: 0x0},
+ 269: {region: 0x60, script: 0x57, flags: 0x0},
+ 270: {region: 0x165, script: 0x57, flags: 0x0},
+ 271: {region: 0x49, script: 0x57, flags: 0x0},
+ 272: {region: 0x165, script: 0x57, flags: 0x0},
+ 273: {region: 0x165, script: 0x57, flags: 0x0},
+ 274: {region: 0x165, script: 0x57, flags: 0x0},
+ 275: {region: 0x165, script: 0x5, flags: 0x0},
+ 276: {region: 0x49, script: 0x57, flags: 0x0},
+ 277: {region: 0x165, script: 0x57, flags: 0x0},
+ 278: {region: 0x165, script: 0x57, flags: 0x0},
+ 279: {region: 0xd4, script: 0x57, flags: 0x0},
+ 280: {region: 0x4f, script: 0x57, flags: 0x0},
+ 281: {region: 0x165, script: 0x57, flags: 0x0},
+ 282: {region: 0x99, script: 0x5, flags: 0x0},
+ 283: {region: 0x165, script: 0x57, flags: 0x0},
+ 284: {region: 0x165, script: 0x57, flags: 0x0},
+ 285: {region: 0x165, script: 0x57, flags: 0x0},
+ 286: {region: 0x165, script: 0x29, flags: 0x0},
+ 287: {region: 0x60, script: 0x57, flags: 0x0},
+ 288: {region: 0xc3, script: 0x57, flags: 0x0},
+ 289: {region: 0xd0, script: 0x57, flags: 0x0},
+ 290: {region: 0x165, script: 0x57, flags: 0x0},
+ 291: {region: 0xdb, script: 0x21, flags: 0x0},
+ 292: {region: 0x52, script: 0x57, flags: 0x0},
+ 293: {region: 0x165, script: 0x57, flags: 0x0},
+ 294: {region: 0x165, script: 0x57, flags: 0x0},
+ 295: {region: 0x165, script: 0x57, flags: 0x0},
+ 296: {region: 0xcd, script: 0xde, flags: 0x0},
+ 297: {region: 0x165, script: 0x57, flags: 0x0},
+ 298: {region: 0x165, script: 0x57, flags: 0x0},
+ 299: {region: 0x114, script: 0x57, flags: 0x0},
+ 300: {region: 0x37, script: 0x57, flags: 0x0},
+ 301: {region: 0x43, script: 0xe0, flags: 0x0},
+ 302: {region: 0x165, script: 0x57, flags: 0x0},
+ 303: {region: 0xa4, script: 0x57, flags: 0x0},
+ 304: {region: 0x80, script: 0x57, flags: 0x0},
+ 305: {region: 0xd6, script: 0x57, flags: 0x0},
+ 306: {region: 0x9e, script: 0x57, flags: 0x0},
+ 307: {region: 0x6b, script: 0x27, flags: 0x0},
+ 308: {region: 0x165, script: 0x57, flags: 0x0},
+ 309: {region: 0xc4, script: 0x48, flags: 0x0},
+ 310: {region: 0x87, script: 0x31, flags: 0x0},
+ 311: {region: 0x165, script: 0x57, flags: 0x0},
+ 312: {region: 0x165, script: 0x57, flags: 0x0},
+ 313: {region: 0xa, script: 0x2, flags: 0x1},
+ 314: {region: 0x165, script: 0x57, flags: 0x0},
+ 315: {region: 0x165, script: 0x57, flags: 0x0},
+ 316: {region: 0x1, script: 0x57, flags: 0x0},
+ 317: {region: 0x165, script: 0x57, flags: 0x0},
+ 318: {region: 0x6e, script: 0x57, flags: 0x0},
+ 319: {region: 0x135, script: 0x57, flags: 0x0},
+ 320: {region: 0x6a, script: 0x57, flags: 0x0},
+ 321: {region: 0x165, script: 0x57, flags: 0x0},
+ 322: {region: 0x9e, script: 0x43, flags: 0x0},
+ 323: {region: 0x165, script: 0x57, flags: 0x0},
+ 324: {region: 0x165, script: 0x57, flags: 0x0},
+ 325: {region: 0x6e, script: 0x57, flags: 0x0},
+ 326: {region: 0x52, script: 0x57, flags: 0x0},
+ 327: {region: 0x6e, script: 0x57, flags: 0x0},
+ 328: {region: 0x9c, script: 0x5, flags: 0x0},
+ 329: {region: 0x165, script: 0x57, flags: 0x0},
+ 330: {region: 0x165, script: 0x57, flags: 0x0},
+ 331: {region: 0x165, script: 0x57, flags: 0x0},
+ 332: {region: 0x165, script: 0x57, flags: 0x0},
+ 333: {region: 0x86, script: 0x57, flags: 0x0},
+ 334: {region: 0xc, script: 0x2, flags: 0x1},
+ 335: {region: 0x165, script: 0x57, flags: 0x0},
+ 336: {region: 0xc3, script: 0x57, flags: 0x0},
+ 337: {region: 0x72, script: 0x57, flags: 0x0},
+ 338: {region: 0x10b, script: 0x5, flags: 0x0},
+ 339: {region: 0xe7, script: 0x57, flags: 0x0},
+ 340: {region: 0x10c, script: 0x57, flags: 0x0},
+ 341: {region: 0x73, script: 0x57, flags: 0x0},
+ 342: {region: 0x165, script: 0x57, flags: 0x0},
+ 343: {region: 0x165, script: 0x57, flags: 0x0},
+ 344: {region: 0x76, script: 0x57, flags: 0x0},
+ 345: {region: 0x165, script: 0x57, flags: 0x0},
+ 346: {region: 0x3b, script: 0x57, flags: 0x0},
+ 347: {region: 0x165, script: 0x57, flags: 0x0},
+ 348: {region: 0x165, script: 0x57, flags: 0x0},
+ 349: {region: 0x165, script: 0x57, flags: 0x0},
+ 350: {region: 0x78, script: 0x57, flags: 0x0},
+ 351: {region: 0x135, script: 0x57, flags: 0x0},
+ 352: {region: 0x78, script: 0x57, flags: 0x0},
+ 353: {region: 0x60, script: 0x57, flags: 0x0},
+ 354: {region: 0x60, script: 0x57, flags: 0x0},
+ 355: {region: 0x52, script: 0x5, flags: 0x0},
+ 356: {region: 0x140, script: 0x57, flags: 0x0},
+ 357: {region: 0x165, script: 0x57, flags: 0x0},
+ 358: {region: 0x84, script: 0x57, flags: 0x0},
+ 359: {region: 0x165, script: 0x57, flags: 0x0},
+ 360: {region: 0xd4, script: 0x57, flags: 0x0},
+ 361: {region: 0x9e, script: 0x57, flags: 0x0},
+ 362: {region: 0xd6, script: 0x57, flags: 0x0},
+ 363: {region: 0x165, script: 0x57, flags: 0x0},
+ 364: {region: 0x10b, script: 0x57, flags: 0x0},
+ 365: {region: 0xd9, script: 0x57, flags: 0x0},
+ 366: {region: 0x96, script: 0x57, flags: 0x0},
+ 367: {region: 0x80, script: 0x57, flags: 0x0},
+ 368: {region: 0x165, script: 0x57, flags: 0x0},
+ 369: {region: 0xbc, script: 0x57, flags: 0x0},
+ 370: {region: 0x165, script: 0x57, flags: 0x0},
+ 371: {region: 0x165, script: 0x57, flags: 0x0},
+ 372: {region: 0x165, script: 0x57, flags: 0x0},
+ 373: {region: 0x53, script: 0x38, flags: 0x0},
+ 374: {region: 0x165, script: 0x57, flags: 0x0},
+ 375: {region: 0x95, script: 0x57, flags: 0x0},
+ 376: {region: 0x165, script: 0x57, flags: 0x0},
+ 377: {region: 0x165, script: 0x57, flags: 0x0},
+ 378: {region: 0x99, script: 0x21, flags: 0x0},
+ 379: {region: 0x165, script: 0x57, flags: 0x0},
+ 380: {region: 0x9c, script: 0x5, flags: 0x0},
+ 381: {region: 0x7e, script: 0x57, flags: 0x0},
+ 382: {region: 0x7b, script: 0x57, flags: 0x0},
+ 383: {region: 0x165, script: 0x57, flags: 0x0},
+ 384: {region: 0x165, script: 0x57, flags: 0x0},
+ 385: {region: 0x165, script: 0x57, flags: 0x0},
+ 386: {region: 0x165, script: 0x57, flags: 0x0},
+ 387: {region: 0x165, script: 0x57, flags: 0x0},
+ 388: {region: 0x165, script: 0x57, flags: 0x0},
+ 389: {region: 0x6f, script: 0x29, flags: 0x0},
+ 390: {region: 0x165, script: 0x57, flags: 0x0},
+ 391: {region: 0xdb, script: 0x21, flags: 0x0},
+ 392: {region: 0x165, script: 0x57, flags: 0x0},
+ 393: {region: 0xa7, script: 0x57, flags: 0x0},
+ 394: {region: 0x165, script: 0x57, flags: 0x0},
+ 395: {region: 0xe8, script: 0x5, flags: 0x0},
+ 396: {region: 0x165, script: 0x57, flags: 0x0},
+ 397: {region: 0xe8, script: 0x5, flags: 0x0},
+ 398: {region: 0x165, script: 0x57, flags: 0x0},
+ 399: {region: 0x165, script: 0x57, flags: 0x0},
+ 400: {region: 0x6e, script: 0x57, flags: 0x0},
+ 401: {region: 0x9c, script: 0x5, flags: 0x0},
+ 402: {region: 0x165, script: 0x57, flags: 0x0},
+ 403: {region: 0x165, script: 0x29, flags: 0x0},
+ 404: {region: 0xf1, script: 0x57, flags: 0x0},
+ 405: {region: 0x165, script: 0x57, flags: 0x0},
+ 406: {region: 0x165, script: 0x57, flags: 0x0},
+ 407: {region: 0x165, script: 0x57, flags: 0x0},
+ 408: {region: 0x165, script: 0x29, flags: 0x0},
+ 409: {region: 0x165, script: 0x57, flags: 0x0},
+ 410: {region: 0x99, script: 0x21, flags: 0x0},
+ 411: {region: 0x99, script: 0xda, flags: 0x0},
+ 412: {region: 0x95, script: 0x57, flags: 0x0},
+ 413: {region: 0xd9, script: 0x57, flags: 0x0},
+ 414: {region: 0x130, script: 0x2f, flags: 0x0},
+ 415: {region: 0x165, script: 0x57, flags: 0x0},
+ 416: {region: 0xe, script: 0x2, flags: 0x1},
+ 417: {region: 0x99, script: 0xe, flags: 0x0},
+ 418: {region: 0x165, script: 0x57, flags: 0x0},
+ 419: {region: 0x4e, script: 0x57, flags: 0x0},
+ 420: {region: 0x99, script: 0x32, flags: 0x0},
+ 421: {region: 0x41, script: 0x57, flags: 0x0},
+ 422: {region: 0x54, script: 0x57, flags: 0x0},
+ 423: {region: 0x165, script: 0x57, flags: 0x0},
+ 424: {region: 0x80, script: 0x57, flags: 0x0},
+ 425: {region: 0x165, script: 0x57, flags: 0x0},
+ 426: {region: 0x165, script: 0x57, flags: 0x0},
+ 427: {region: 0xa4, script: 0x57, flags: 0x0},
+ 428: {region: 0x98, script: 0x57, flags: 0x0},
+ 429: {region: 0x165, script: 0x57, flags: 0x0},
+ 430: {region: 0xdb, script: 0x21, flags: 0x0},
+ 431: {region: 0x165, script: 0x57, flags: 0x0},
+ 432: {region: 0x165, script: 0x5, flags: 0x0},
+ 433: {region: 0x49, script: 0x57, flags: 0x0},
+ 434: {region: 0x165, script: 0x5, flags: 0x0},
+ 435: {region: 0x165, script: 0x57, flags: 0x0},
+ 436: {region: 0x10, script: 0x3, flags: 0x1},
+ 437: {region: 0x165, script: 0x57, flags: 0x0},
+ 438: {region: 0x53, script: 0x38, flags: 0x0},
+ 439: {region: 0x165, script: 0x57, flags: 0x0},
+ 440: {region: 0x135, script: 0x57, flags: 0x0},
+ 441: {region: 0x24, script: 0x5, flags: 0x0},
+ 442: {region: 0x165, script: 0x57, flags: 0x0},
+ 443: {region: 0x165, script: 0x29, flags: 0x0},
+ 444: {region: 0x97, script: 0x3b, flags: 0x0},
+ 445: {region: 0x165, script: 0x57, flags: 0x0},
+ 446: {region: 0x99, script: 0x21, flags: 0x0},
+ 447: {region: 0x165, script: 0x57, flags: 0x0},
+ 448: {region: 0x73, script: 0x57, flags: 0x0},
+ 449: {region: 0x165, script: 0x57, flags: 0x0},
+ 450: {region: 0x165, script: 0x57, flags: 0x0},
+ 451: {region: 0xe7, script: 0x57, flags: 0x0},
+ 452: {region: 0x165, script: 0x57, flags: 0x0},
+ 453: {region: 0x12b, script: 0x3d, flags: 0x0},
+ 454: {region: 0x53, script: 0x89, flags: 0x0},
+ 455: {region: 0x165, script: 0x57, flags: 0x0},
+ 456: {region: 0xe8, script: 0x5, flags: 0x0},
+ 457: {region: 0x99, script: 0x21, flags: 0x0},
+ 458: {region: 0xaf, script: 0x3e, flags: 0x0},
+ 459: {region: 0xe7, script: 0x57, flags: 0x0},
+ 460: {region: 0xe8, script: 0x5, flags: 0x0},
+ 461: {region: 0xe6, script: 0x57, flags: 0x0},
+ 462: {region: 0x99, script: 0x21, flags: 0x0},
+ 463: {region: 0x99, script: 0x21, flags: 0x0},
+ 464: {region: 0x165, script: 0x57, flags: 0x0},
+ 465: {region: 0x90, script: 0x57, flags: 0x0},
+ 466: {region: 0x60, script: 0x57, flags: 0x0},
+ 467: {region: 0x53, script: 0x38, flags: 0x0},
+ 468: {region: 0x91, script: 0x57, flags: 0x0},
+ 469: {region: 0x92, script: 0x57, flags: 0x0},
+ 470: {region: 0x165, script: 0x57, flags: 0x0},
+ 471: {region: 0x28, script: 0x8, flags: 0x0},
+ 472: {region: 0xd2, script: 0x57, flags: 0x0},
+ 473: {region: 0x78, script: 0x57, flags: 0x0},
+ 474: {region: 0x165, script: 0x57, flags: 0x0},
+ 475: {region: 0x165, script: 0x57, flags: 0x0},
+ 476: {region: 0xd0, script: 0x57, flags: 0x0},
+ 477: {region: 0xd6, script: 0x57, flags: 0x0},
+ 478: {region: 0x165, script: 0x57, flags: 0x0},
+ 479: {region: 0x165, script: 0x57, flags: 0x0},
+ 480: {region: 0x165, script: 0x57, flags: 0x0},
+ 481: {region: 0x95, script: 0x57, flags: 0x0},
+ 482: {region: 0x165, script: 0x57, flags: 0x0},
+ 483: {region: 0x165, script: 0x57, flags: 0x0},
+ 484: {region: 0x165, script: 0x57, flags: 0x0},
+ 486: {region: 0x122, script: 0x57, flags: 0x0},
+ 487: {region: 0xd6, script: 0x57, flags: 0x0},
+ 488: {region: 0x165, script: 0x57, flags: 0x0},
+ 489: {region: 0x165, script: 0x57, flags: 0x0},
+ 490: {region: 0x53, script: 0xea, flags: 0x0},
+ 491: {region: 0x165, script: 0x57, flags: 0x0},
+ 492: {region: 0x135, script: 0x57, flags: 0x0},
+ 493: {region: 0x165, script: 0x57, flags: 0x0},
+ 494: {region: 0x49, script: 0x57, flags: 0x0},
+ 495: {region: 0x165, script: 0x57, flags: 0x0},
+ 496: {region: 0x165, script: 0x57, flags: 0x0},
+ 497: {region: 0xe7, script: 0x57, flags: 0x0},
+ 498: {region: 0x165, script: 0x57, flags: 0x0},
+ 499: {region: 0x95, script: 0x57, flags: 0x0},
+ 500: {region: 0x106, script: 0x1f, flags: 0x0},
+ 501: {region: 0x1, script: 0x57, flags: 0x0},
+ 502: {region: 0x165, script: 0x57, flags: 0x0},
+ 503: {region: 0x165, script: 0x57, flags: 0x0},
+ 504: {region: 0x9d, script: 0x57, flags: 0x0},
+ 505: {region: 0x9e, script: 0x57, flags: 0x0},
+ 506: {region: 0x49, script: 0x17, flags: 0x0},
+ 507: {region: 0x97, script: 0x3b, flags: 0x0},
+ 508: {region: 0x165, script: 0x57, flags: 0x0},
+ 509: {region: 0x165, script: 0x57, flags: 0x0},
+ 510: {region: 0x106, script: 0x57, flags: 0x0},
+ 511: {region: 0x165, script: 0x57, flags: 0x0},
+ 512: {region: 0xa2, script: 0x46, flags: 0x0},
+ 513: {region: 0x165, script: 0x57, flags: 0x0},
+ 514: {region: 0xa0, script: 0x57, flags: 0x0},
+ 515: {region: 0x1, script: 0x57, flags: 0x0},
+ 516: {region: 0x165, script: 0x57, flags: 0x0},
+ 517: {region: 0x165, script: 0x57, flags: 0x0},
+ 518: {region: 0x165, script: 0x57, flags: 0x0},
+ 519: {region: 0x52, script: 0x57, flags: 0x0},
+ 520: {region: 0x130, script: 0x3b, flags: 0x0},
+ 521: {region: 0x165, script: 0x57, flags: 0x0},
+ 522: {region: 0x12f, script: 0x57, flags: 0x0},
+ 523: {region: 0xdb, script: 0x21, flags: 0x0},
+ 524: {region: 0x165, script: 0x57, flags: 0x0},
+ 525: {region: 0x63, script: 0x57, flags: 0x0},
+ 526: {region: 0x95, script: 0x57, flags: 0x0},
+ 527: {region: 0x95, script: 0x57, flags: 0x0},
+ 528: {region: 0x7d, script: 0x2b, flags: 0x0},
+ 529: {region: 0x137, script: 0x1f, flags: 0x0},
+ 530: {region: 0x67, script: 0x57, flags: 0x0},
+ 531: {region: 0xc4, script: 0x57, flags: 0x0},
+ 532: {region: 0x165, script: 0x57, flags: 0x0},
+ 533: {region: 0x165, script: 0x57, flags: 0x0},
+ 534: {region: 0xd6, script: 0x57, flags: 0x0},
+ 535: {region: 0xa4, script: 0x57, flags: 0x0},
+ 536: {region: 0xc3, script: 0x57, flags: 0x0},
+ 537: {region: 0x106, script: 0x1f, flags: 0x0},
+ 538: {region: 0x165, script: 0x57, flags: 0x0},
+ 539: {region: 0x165, script: 0x57, flags: 0x0},
+ 540: {region: 0x165, script: 0x57, flags: 0x0},
+ 541: {region: 0x165, script: 0x57, flags: 0x0},
+ 542: {region: 0xd4, script: 0x5, flags: 0x0},
+ 543: {region: 0xd6, script: 0x57, flags: 0x0},
+ 544: {region: 0x164, script: 0x57, flags: 0x0},
+ 545: {region: 0x165, script: 0x57, flags: 0x0},
+ 546: {region: 0x165, script: 0x57, flags: 0x0},
+ 547: {region: 0x12f, script: 0x57, flags: 0x0},
+ 548: {region: 0x122, script: 0x5, flags: 0x0},
+ 549: {region: 0x165, script: 0x57, flags: 0x0},
+ 550: {region: 0x123, script: 0xdf, flags: 0x0},
+ 551: {region: 0x5a, script: 0x57, flags: 0x0},
+ 552: {region: 0x52, script: 0x57, flags: 0x0},
+ 553: {region: 0x165, script: 0x57, flags: 0x0},
+ 554: {region: 0x4f, script: 0x57, flags: 0x0},
+ 555: {region: 0x99, script: 0x21, flags: 0x0},
+ 556: {region: 0x99, script: 0x21, flags: 0x0},
+ 557: {region: 0x4b, script: 0x57, flags: 0x0},
+ 558: {region: 0x95, script: 0x57, flags: 0x0},
+ 559: {region: 0x165, script: 0x57, flags: 0x0},
+ 560: {region: 0x41, script: 0x57, flags: 0x0},
+ 561: {region: 0x99, script: 0x57, flags: 0x0},
+ 562: {region: 0x53, script: 0xd6, flags: 0x0},
+ 563: {region: 0x99, script: 0x21, flags: 0x0},
+ 564: {region: 0xc3, script: 0x57, flags: 0x0},
+ 565: {region: 0x165, script: 0x57, flags: 0x0},
+ 566: {region: 0x99, script: 0x72, flags: 0x0},
+ 567: {region: 0xe8, script: 0x5, flags: 0x0},
+ 568: {region: 0x165, script: 0x57, flags: 0x0},
+ 569: {region: 0xa4, script: 0x57, flags: 0x0},
+ 570: {region: 0x165, script: 0x57, flags: 0x0},
+ 571: {region: 0x12b, script: 0x57, flags: 0x0},
+ 572: {region: 0x165, script: 0x57, flags: 0x0},
+ 573: {region: 0xd2, script: 0x57, flags: 0x0},
+ 574: {region: 0x165, script: 0x57, flags: 0x0},
+ 575: {region: 0xaf, script: 0x54, flags: 0x0},
+ 576: {region: 0x165, script: 0x57, flags: 0x0},
+ 577: {region: 0x165, script: 0x57, flags: 0x0},
+ 578: {region: 0x13, script: 0x6, flags: 0x1},
+ 579: {region: 0x165, script: 0x57, flags: 0x0},
+ 580: {region: 0x52, script: 0x57, flags: 0x0},
+ 581: {region: 0x82, script: 0x57, flags: 0x0},
+ 582: {region: 0xa4, script: 0x57, flags: 0x0},
+ 583: {region: 0x165, script: 0x57, flags: 0x0},
+ 584: {region: 0x165, script: 0x57, flags: 0x0},
+ 585: {region: 0x165, script: 0x57, flags: 0x0},
+ 586: {region: 0xa6, script: 0x4b, flags: 0x0},
+ 587: {region: 0x2a, script: 0x57, flags: 0x0},
+ 588: {region: 0x165, script: 0x57, flags: 0x0},
+ 589: {region: 0x165, script: 0x57, flags: 0x0},
+ 590: {region: 0x165, script: 0x57, flags: 0x0},
+ 591: {region: 0x165, script: 0x57, flags: 0x0},
+ 592: {region: 0x165, script: 0x57, flags: 0x0},
+ 593: {region: 0x99, script: 0x4f, flags: 0x0},
+ 594: {region: 0x8b, script: 0x57, flags: 0x0},
+ 595: {region: 0x165, script: 0x57, flags: 0x0},
+ 596: {region: 0xab, script: 0x50, flags: 0x0},
+ 597: {region: 0x106, script: 0x1f, flags: 0x0},
+ 598: {region: 0x99, script: 0x21, flags: 0x0},
+ 599: {region: 0x165, script: 0x57, flags: 0x0},
+ 600: {region: 0x75, script: 0x57, flags: 0x0},
+ 601: {region: 0x165, script: 0x57, flags: 0x0},
+ 602: {region: 0xb4, script: 0x57, flags: 0x0},
+ 603: {region: 0x165, script: 0x57, flags: 0x0},
+ 604: {region: 0x165, script: 0x57, flags: 0x0},
+ 605: {region: 0x165, script: 0x57, flags: 0x0},
+ 606: {region: 0x165, script: 0x57, flags: 0x0},
+ 607: {region: 0x165, script: 0x57, flags: 0x0},
+ 608: {region: 0x165, script: 0x57, flags: 0x0},
+ 609: {region: 0x165, script: 0x57, flags: 0x0},
+ 610: {region: 0x165, script: 0x29, flags: 0x0},
+ 611: {region: 0x165, script: 0x57, flags: 0x0},
+ 612: {region: 0x106, script: 0x1f, flags: 0x0},
+ 613: {region: 0x112, script: 0x57, flags: 0x0},
+ 614: {region: 0xe7, script: 0x57, flags: 0x0},
+ 615: {region: 0x106, script: 0x57, flags: 0x0},
+ 616: {region: 0x165, script: 0x57, flags: 0x0},
+ 617: {region: 0x99, script: 0x21, flags: 0x0},
+ 618: {region: 0x99, script: 0x5, flags: 0x0},
+ 619: {region: 0x12f, script: 0x57, flags: 0x0},
+ 620: {region: 0x165, script: 0x57, flags: 0x0},
+ 621: {region: 0x52, script: 0x57, flags: 0x0},
+ 622: {region: 0x60, script: 0x57, flags: 0x0},
+ 623: {region: 0x165, script: 0x57, flags: 0x0},
+ 624: {region: 0x165, script: 0x57, flags: 0x0},
+ 625: {region: 0x165, script: 0x29, flags: 0x0},
+ 626: {region: 0x165, script: 0x57, flags: 0x0},
+ 627: {region: 0x165, script: 0x57, flags: 0x0},
+ 628: {region: 0x19, script: 0x3, flags: 0x1},
+ 629: {region: 0x165, script: 0x57, flags: 0x0},
+ 630: {region: 0x165, script: 0x57, flags: 0x0},
+ 631: {region: 0x165, script: 0x57, flags: 0x0},
+ 632: {region: 0x165, script: 0x57, flags: 0x0},
+ 633: {region: 0x106, script: 0x1f, flags: 0x0},
+ 634: {region: 0x165, script: 0x57, flags: 0x0},
+ 635: {region: 0x165, script: 0x57, flags: 0x0},
+ 636: {region: 0x165, script: 0x57, flags: 0x0},
+ 637: {region: 0x106, script: 0x1f, flags: 0x0},
+ 638: {region: 0x165, script: 0x57, flags: 0x0},
+ 639: {region: 0x95, script: 0x57, flags: 0x0},
+ 640: {region: 0xe8, script: 0x5, flags: 0x0},
+ 641: {region: 0x7b, script: 0x57, flags: 0x0},
+ 642: {region: 0x165, script: 0x57, flags: 0x0},
+ 643: {region: 0x165, script: 0x57, flags: 0x0},
+ 644: {region: 0x165, script: 0x57, flags: 0x0},
+ 645: {region: 0x165, script: 0x29, flags: 0x0},
+ 646: {region: 0x123, script: 0xdf, flags: 0x0},
+ 647: {region: 0xe8, script: 0x5, flags: 0x0},
+ 648: {region: 0x165, script: 0x57, flags: 0x0},
+ 649: {region: 0x165, script: 0x57, flags: 0x0},
+ 650: {region: 0x1c, script: 0x5, flags: 0x1},
+ 651: {region: 0x165, script: 0x57, flags: 0x0},
+ 652: {region: 0x165, script: 0x57, flags: 0x0},
+ 653: {region: 0x165, script: 0x57, flags: 0x0},
+ 654: {region: 0x138, script: 0x57, flags: 0x0},
+ 655: {region: 0x87, script: 0x5b, flags: 0x0},
+ 656: {region: 0x97, script: 0x3b, flags: 0x0},
+ 657: {region: 0x12f, script: 0x57, flags: 0x0},
+ 658: {region: 0xe8, script: 0x5, flags: 0x0},
+ 659: {region: 0x131, script: 0x57, flags: 0x0},
+ 660: {region: 0x165, script: 0x57, flags: 0x0},
+ 661: {region: 0xb7, script: 0x57, flags: 0x0},
+ 662: {region: 0x106, script: 0x1f, flags: 0x0},
+ 663: {region: 0x165, script: 0x57, flags: 0x0},
+ 664: {region: 0x95, script: 0x57, flags: 0x0},
+ 665: {region: 0x165, script: 0x57, flags: 0x0},
+ 666: {region: 0x53, script: 0xdf, flags: 0x0},
+ 667: {region: 0x165, script: 0x57, flags: 0x0},
+ 668: {region: 0x165, script: 0x57, flags: 0x0},
+ 669: {region: 0x165, script: 0x57, flags: 0x0},
+ 670: {region: 0x165, script: 0x57, flags: 0x0},
+ 671: {region: 0x99, script: 0x59, flags: 0x0},
+ 672: {region: 0x165, script: 0x57, flags: 0x0},
+ 673: {region: 0x165, script: 0x57, flags: 0x0},
+ 674: {region: 0x106, script: 0x1f, flags: 0x0},
+ 675: {region: 0x131, script: 0x57, flags: 0x0},
+ 676: {region: 0x165, script: 0x57, flags: 0x0},
+ 677: {region: 0xd9, script: 0x57, flags: 0x0},
+ 678: {region: 0x165, script: 0x57, flags: 0x0},
+ 679: {region: 0x165, script: 0x57, flags: 0x0},
+ 680: {region: 0x21, script: 0x2, flags: 0x1},
+ 681: {region: 0x165, script: 0x57, flags: 0x0},
+ 682: {region: 0x165, script: 0x57, flags: 0x0},
+ 683: {region: 0x9e, script: 0x57, flags: 0x0},
+ 684: {region: 0x53, script: 0x5d, flags: 0x0},
+ 685: {region: 0x95, script: 0x57, flags: 0x0},
+ 686: {region: 0x9c, script: 0x5, flags: 0x0},
+ 687: {region: 0x135, script: 0x57, flags: 0x0},
+ 688: {region: 0x165, script: 0x57, flags: 0x0},
+ 689: {region: 0x165, script: 0x57, flags: 0x0},
+ 690: {region: 0x99, script: 0xda, flags: 0x0},
+ 691: {region: 0x9e, script: 0x57, flags: 0x0},
+ 692: {region: 0x165, script: 0x57, flags: 0x0},
+ 693: {region: 0x4b, script: 0x57, flags: 0x0},
+ 694: {region: 0x165, script: 0x57, flags: 0x0},
+ 695: {region: 0x165, script: 0x57, flags: 0x0},
+ 696: {region: 0xaf, script: 0x54, flags: 0x0},
+ 697: {region: 0x165, script: 0x57, flags: 0x0},
+ 698: {region: 0x165, script: 0x57, flags: 0x0},
+ 699: {region: 0x4b, script: 0x57, flags: 0x0},
+ 700: {region: 0x165, script: 0x57, flags: 0x0},
+ 701: {region: 0x165, script: 0x57, flags: 0x0},
+ 702: {region: 0x162, script: 0x57, flags: 0x0},
+ 703: {region: 0x9c, script: 0x5, flags: 0x0},
+ 704: {region: 0xb6, script: 0x57, flags: 0x0},
+ 705: {region: 0xb8, script: 0x57, flags: 0x0},
+ 706: {region: 0x4b, script: 0x57, flags: 0x0},
+ 707: {region: 0x4b, script: 0x57, flags: 0x0},
+ 708: {region: 0xa4, script: 0x57, flags: 0x0},
+ 709: {region: 0xa4, script: 0x57, flags: 0x0},
+ 710: {region: 0x9c, script: 0x5, flags: 0x0},
+ 711: {region: 0xb8, script: 0x57, flags: 0x0},
+ 712: {region: 0x123, script: 0xdf, flags: 0x0},
+ 713: {region: 0x53, script: 0x38, flags: 0x0},
+ 714: {region: 0x12b, script: 0x57, flags: 0x0},
+ 715: {region: 0x95, script: 0x57, flags: 0x0},
+ 716: {region: 0x52, script: 0x57, flags: 0x0},
+ 717: {region: 0x99, script: 0x21, flags: 0x0},
+ 718: {region: 0x99, script: 0x21, flags: 0x0},
+ 719: {region: 0x95, script: 0x57, flags: 0x0},
+ 720: {region: 0x23, script: 0x3, flags: 0x1},
+ 721: {region: 0xa4, script: 0x57, flags: 0x0},
+ 722: {region: 0x165, script: 0x57, flags: 0x0},
+ 723: {region: 0xcf, script: 0x57, flags: 0x0},
+ 724: {region: 0x165, script: 0x57, flags: 0x0},
+ 725: {region: 0x165, script: 0x57, flags: 0x0},
+ 726: {region: 0x165, script: 0x57, flags: 0x0},
+ 727: {region: 0x165, script: 0x57, flags: 0x0},
+ 728: {region: 0x165, script: 0x57, flags: 0x0},
+ 729: {region: 0x165, script: 0x57, flags: 0x0},
+ 730: {region: 0x165, script: 0x57, flags: 0x0},
+ 731: {region: 0x165, script: 0x57, flags: 0x0},
+ 732: {region: 0x165, script: 0x57, flags: 0x0},
+ 733: {region: 0x165, script: 0x57, flags: 0x0},
+ 734: {region: 0x165, script: 0x57, flags: 0x0},
+ 735: {region: 0x165, script: 0x5, flags: 0x0},
+ 736: {region: 0x106, script: 0x1f, flags: 0x0},
+ 737: {region: 0xe7, script: 0x57, flags: 0x0},
+ 738: {region: 0x165, script: 0x57, flags: 0x0},
+ 739: {region: 0x95, script: 0x57, flags: 0x0},
+ 740: {region: 0x165, script: 0x29, flags: 0x0},
+ 741: {region: 0x165, script: 0x57, flags: 0x0},
+ 742: {region: 0x165, script: 0x57, flags: 0x0},
+ 743: {region: 0x165, script: 0x57, flags: 0x0},
+ 744: {region: 0x112, script: 0x57, flags: 0x0},
+ 745: {region: 0xa4, script: 0x57, flags: 0x0},
+ 746: {region: 0x165, script: 0x57, flags: 0x0},
+ 747: {region: 0x165, script: 0x57, flags: 0x0},
+ 748: {region: 0x123, script: 0x5, flags: 0x0},
+ 749: {region: 0xcc, script: 0x57, flags: 0x0},
+ 750: {region: 0x165, script: 0x57, flags: 0x0},
+ 751: {region: 0x165, script: 0x57, flags: 0x0},
+ 752: {region: 0x165, script: 0x57, flags: 0x0},
+ 753: {region: 0xbf, script: 0x57, flags: 0x0},
+ 754: {region: 0xd1, script: 0x57, flags: 0x0},
+ 755: {region: 0x165, script: 0x57, flags: 0x0},
+ 756: {region: 0x52, script: 0x57, flags: 0x0},
+ 757: {region: 0xdb, script: 0x21, flags: 0x0},
+ 758: {region: 0x12f, script: 0x57, flags: 0x0},
+ 759: {region: 0xc0, script: 0x57, flags: 0x0},
+ 760: {region: 0x165, script: 0x57, flags: 0x0},
+ 761: {region: 0x165, script: 0x57, flags: 0x0},
+ 762: {region: 0xe0, script: 0x57, flags: 0x0},
+ 763: {region: 0x165, script: 0x57, flags: 0x0},
+ 764: {region: 0x95, script: 0x57, flags: 0x0},
+ 765: {region: 0x9b, script: 0x3a, flags: 0x0},
+ 766: {region: 0x165, script: 0x57, flags: 0x0},
+ 767: {region: 0xc2, script: 0x1f, flags: 0x0},
+ 768: {region: 0x165, script: 0x5, flags: 0x0},
+ 769: {region: 0x165, script: 0x57, flags: 0x0},
+ 770: {region: 0x165, script: 0x57, flags: 0x0},
+ 771: {region: 0x165, script: 0x57, flags: 0x0},
+ 772: {region: 0x99, script: 0x6b, flags: 0x0},
+ 773: {region: 0x165, script: 0x57, flags: 0x0},
+ 774: {region: 0x165, script: 0x57, flags: 0x0},
+ 775: {region: 0x10b, script: 0x57, flags: 0x0},
+ 776: {region: 0x165, script: 0x57, flags: 0x0},
+ 777: {region: 0x165, script: 0x57, flags: 0x0},
+ 778: {region: 0x165, script: 0x57, flags: 0x0},
+ 779: {region: 0x26, script: 0x3, flags: 0x1},
+ 780: {region: 0x165, script: 0x57, flags: 0x0},
+ 781: {region: 0x165, script: 0x57, flags: 0x0},
+ 782: {region: 0x99, script: 0xe, flags: 0x0},
+ 783: {region: 0xc4, script: 0x72, flags: 0x0},
+ 785: {region: 0x165, script: 0x57, flags: 0x0},
+ 786: {region: 0x49, script: 0x57, flags: 0x0},
+ 787: {region: 0x49, script: 0x57, flags: 0x0},
+ 788: {region: 0x37, script: 0x57, flags: 0x0},
+ 789: {region: 0x165, script: 0x57, flags: 0x0},
+ 790: {region: 0x165, script: 0x57, flags: 0x0},
+ 791: {region: 0x165, script: 0x57, flags: 0x0},
+ 792: {region: 0x165, script: 0x57, flags: 0x0},
+ 793: {region: 0x165, script: 0x57, flags: 0x0},
+ 794: {region: 0x165, script: 0x57, flags: 0x0},
+ 795: {region: 0x99, script: 0x21, flags: 0x0},
+ 796: {region: 0xdb, script: 0x21, flags: 0x0},
+ 797: {region: 0x106, script: 0x1f, flags: 0x0},
+ 798: {region: 0x35, script: 0x6f, flags: 0x0},
+ 799: {region: 0x29, script: 0x3, flags: 0x1},
+ 800: {region: 0xcb, script: 0x57, flags: 0x0},
+ 801: {region: 0x165, script: 0x57, flags: 0x0},
+ 802: {region: 0x165, script: 0x57, flags: 0x0},
+ 803: {region: 0x165, script: 0x57, flags: 0x0},
+ 804: {region: 0x99, script: 0x21, flags: 0x0},
+ 805: {region: 0x52, script: 0x57, flags: 0x0},
+ 807: {region: 0x165, script: 0x57, flags: 0x0},
+ 808: {region: 0x135, script: 0x57, flags: 0x0},
+ 809: {region: 0x165, script: 0x57, flags: 0x0},
+ 810: {region: 0x165, script: 0x57, flags: 0x0},
+ 811: {region: 0xe8, script: 0x5, flags: 0x0},
+ 812: {region: 0xc3, script: 0x57, flags: 0x0},
+ 813: {region: 0x99, script: 0x21, flags: 0x0},
+ 814: {region: 0x95, script: 0x57, flags: 0x0},
+ 815: {region: 0x164, script: 0x57, flags: 0x0},
+ 816: {region: 0x165, script: 0x57, flags: 0x0},
+ 817: {region: 0xc4, script: 0x72, flags: 0x0},
+ 818: {region: 0x165, script: 0x57, flags: 0x0},
+ 819: {region: 0x165, script: 0x29, flags: 0x0},
+ 820: {region: 0x106, script: 0x1f, flags: 0x0},
+ 821: {region: 0x165, script: 0x57, flags: 0x0},
+ 822: {region: 0x131, script: 0x57, flags: 0x0},
+ 823: {region: 0x9c, script: 0x63, flags: 0x0},
+ 824: {region: 0x165, script: 0x57, flags: 0x0},
+ 825: {region: 0x165, script: 0x57, flags: 0x0},
+ 826: {region: 0x9c, script: 0x5, flags: 0x0},
+ 827: {region: 0x165, script: 0x57, flags: 0x0},
+ 828: {region: 0x165, script: 0x57, flags: 0x0},
+ 829: {region: 0x165, script: 0x57, flags: 0x0},
+ 830: {region: 0xdd, script: 0x57, flags: 0x0},
+ 831: {region: 0x165, script: 0x57, flags: 0x0},
+ 832: {region: 0x165, script: 0x57, flags: 0x0},
+ 834: {region: 0x165, script: 0x57, flags: 0x0},
+ 835: {region: 0x53, script: 0x38, flags: 0x0},
+ 836: {region: 0x9e, script: 0x57, flags: 0x0},
+ 837: {region: 0xd2, script: 0x57, flags: 0x0},
+ 838: {region: 0x165, script: 0x57, flags: 0x0},
+ 839: {region: 0xda, script: 0x57, flags: 0x0},
+ 840: {region: 0x165, script: 0x57, flags: 0x0},
+ 841: {region: 0x165, script: 0x57, flags: 0x0},
+ 842: {region: 0x165, script: 0x57, flags: 0x0},
+ 843: {region: 0xcf, script: 0x57, flags: 0x0},
+ 844: {region: 0x165, script: 0x57, flags: 0x0},
+ 845: {region: 0x165, script: 0x57, flags: 0x0},
+ 846: {region: 0x164, script: 0x57, flags: 0x0},
+ 847: {region: 0xd1, script: 0x57, flags: 0x0},
+ 848: {region: 0x60, script: 0x57, flags: 0x0},
+ 849: {region: 0xdb, script: 0x21, flags: 0x0},
+ 850: {region: 0x165, script: 0x57, flags: 0x0},
+ 851: {region: 0xdb, script: 0x21, flags: 0x0},
+ 852: {region: 0x165, script: 0x57, flags: 0x0},
+ 853: {region: 0x165, script: 0x57, flags: 0x0},
+ 854: {region: 0xd2, script: 0x57, flags: 0x0},
+ 855: {region: 0x165, script: 0x57, flags: 0x0},
+ 856: {region: 0x165, script: 0x57, flags: 0x0},
+ 857: {region: 0xd1, script: 0x57, flags: 0x0},
+ 858: {region: 0x165, script: 0x57, flags: 0x0},
+ 859: {region: 0xcf, script: 0x57, flags: 0x0},
+ 860: {region: 0xcf, script: 0x57, flags: 0x0},
+ 861: {region: 0x165, script: 0x57, flags: 0x0},
+ 862: {region: 0x165, script: 0x57, flags: 0x0},
+ 863: {region: 0x95, script: 0x57, flags: 0x0},
+ 864: {region: 0x165, script: 0x57, flags: 0x0},
+ 865: {region: 0xdf, script: 0x57, flags: 0x0},
+ 866: {region: 0x165, script: 0x57, flags: 0x0},
+ 867: {region: 0x165, script: 0x57, flags: 0x0},
+ 868: {region: 0x99, script: 0x57, flags: 0x0},
+ 869: {region: 0x165, script: 0x57, flags: 0x0},
+ 870: {region: 0x165, script: 0x57, flags: 0x0},
+ 871: {region: 0xd9, script: 0x57, flags: 0x0},
+ 872: {region: 0x52, script: 0x57, flags: 0x0},
+ 873: {region: 0x165, script: 0x57, flags: 0x0},
+ 874: {region: 0xda, script: 0x57, flags: 0x0},
+ 875: {region: 0x165, script: 0x57, flags: 0x0},
+ 876: {region: 0x52, script: 0x57, flags: 0x0},
+ 877: {region: 0x165, script: 0x57, flags: 0x0},
+ 878: {region: 0x165, script: 0x57, flags: 0x0},
+ 879: {region: 0xda, script: 0x57, flags: 0x0},
+ 880: {region: 0x123, script: 0x53, flags: 0x0},
+ 881: {region: 0x99, script: 0x21, flags: 0x0},
+ 882: {region: 0x10c, script: 0xbf, flags: 0x0},
+ 883: {region: 0x165, script: 0x57, flags: 0x0},
+ 884: {region: 0x165, script: 0x57, flags: 0x0},
+ 885: {region: 0x84, script: 0x78, flags: 0x0},
+ 886: {region: 0x161, script: 0x57, flags: 0x0},
+ 887: {region: 0x165, script: 0x57, flags: 0x0},
+ 888: {region: 0x49, script: 0x17, flags: 0x0},
+ 889: {region: 0x165, script: 0x57, flags: 0x0},
+ 890: {region: 0x161, script: 0x57, flags: 0x0},
+ 891: {region: 0x165, script: 0x57, flags: 0x0},
+ 892: {region: 0x165, script: 0x57, flags: 0x0},
+ 893: {region: 0x165, script: 0x57, flags: 0x0},
+ 894: {region: 0x165, script: 0x57, flags: 0x0},
+ 895: {region: 0x165, script: 0x57, flags: 0x0},
+ 896: {region: 0x117, script: 0x57, flags: 0x0},
+ 897: {region: 0x165, script: 0x57, flags: 0x0},
+ 898: {region: 0x165, script: 0x57, flags: 0x0},
+ 899: {region: 0x135, script: 0x57, flags: 0x0},
+ 900: {region: 0x165, script: 0x57, flags: 0x0},
+ 901: {region: 0x53, script: 0x57, flags: 0x0},
+ 902: {region: 0x165, script: 0x57, flags: 0x0},
+ 903: {region: 0xce, script: 0x57, flags: 0x0},
+ 904: {region: 0x12f, script: 0x57, flags: 0x0},
+ 905: {region: 0x131, script: 0x57, flags: 0x0},
+ 906: {region: 0x80, script: 0x57, flags: 0x0},
+ 907: {region: 0x78, script: 0x57, flags: 0x0},
+ 908: {region: 0x165, script: 0x57, flags: 0x0},
+ 910: {region: 0x165, script: 0x57, flags: 0x0},
+ 911: {region: 0x165, script: 0x57, flags: 0x0},
+ 912: {region: 0x6f, script: 0x57, flags: 0x0},
+ 913: {region: 0x165, script: 0x57, flags: 0x0},
+ 914: {region: 0x165, script: 0x57, flags: 0x0},
+ 915: {region: 0x165, script: 0x57, flags: 0x0},
+ 916: {region: 0x165, script: 0x57, flags: 0x0},
+ 917: {region: 0x99, script: 0x7d, flags: 0x0},
+ 918: {region: 0x165, script: 0x57, flags: 0x0},
+ 919: {region: 0x165, script: 0x5, flags: 0x0},
+ 920: {region: 0x7d, script: 0x1f, flags: 0x0},
+ 921: {region: 0x135, script: 0x7e, flags: 0x0},
+ 922: {region: 0x165, script: 0x5, flags: 0x0},
+ 923: {region: 0xc5, script: 0x7c, flags: 0x0},
+ 924: {region: 0x165, script: 0x57, flags: 0x0},
+ 925: {region: 0x2c, script: 0x3, flags: 0x1},
+ 926: {region: 0xe7, script: 0x57, flags: 0x0},
+ 927: {region: 0x2f, script: 0x2, flags: 0x1},
+ 928: {region: 0xe7, script: 0x57, flags: 0x0},
+ 929: {region: 0x30, script: 0x57, flags: 0x0},
+ 930: {region: 0xf0, script: 0x57, flags: 0x0},
+ 931: {region: 0x165, script: 0x57, flags: 0x0},
+ 932: {region: 0x78, script: 0x57, flags: 0x0},
+ 933: {region: 0xd6, script: 0x57, flags: 0x0},
+ 934: {region: 0x135, script: 0x57, flags: 0x0},
+ 935: {region: 0x49, script: 0x57, flags: 0x0},
+ 936: {region: 0x165, script: 0x57, flags: 0x0},
+ 937: {region: 0x9c, script: 0xe8, flags: 0x0},
+ 938: {region: 0x165, script: 0x57, flags: 0x0},
+ 939: {region: 0x60, script: 0x57, flags: 0x0},
+ 940: {region: 0x165, script: 0x5, flags: 0x0},
+ 941: {region: 0xb0, script: 0x87, flags: 0x0},
+ 943: {region: 0x165, script: 0x57, flags: 0x0},
+ 944: {region: 0x165, script: 0x57, flags: 0x0},
+ 945: {region: 0x99, script: 0x12, flags: 0x0},
+ 946: {region: 0xa4, script: 0x57, flags: 0x0},
+ 947: {region: 0xe9, script: 0x57, flags: 0x0},
+ 948: {region: 0x165, script: 0x57, flags: 0x0},
+ 949: {region: 0x9e, script: 0x57, flags: 0x0},
+ 950: {region: 0x165, script: 0x57, flags: 0x0},
+ 951: {region: 0x165, script: 0x57, flags: 0x0},
+ 952: {region: 0x87, script: 0x31, flags: 0x0},
+ 953: {region: 0x75, script: 0x57, flags: 0x0},
+ 954: {region: 0x165, script: 0x57, flags: 0x0},
+ 955: {region: 0xe8, script: 0x4a, flags: 0x0},
+ 956: {region: 0x9c, script: 0x5, flags: 0x0},
+ 957: {region: 0x1, script: 0x57, flags: 0x0},
+ 958: {region: 0x24, script: 0x5, flags: 0x0},
+ 959: {region: 0x165, script: 0x57, flags: 0x0},
+ 960: {region: 0x41, script: 0x57, flags: 0x0},
+ 961: {region: 0x165, script: 0x57, flags: 0x0},
+ 962: {region: 0x7a, script: 0x57, flags: 0x0},
+ 963: {region: 0x165, script: 0x57, flags: 0x0},
+ 964: {region: 0xe4, script: 0x57, flags: 0x0},
+ 965: {region: 0x89, script: 0x57, flags: 0x0},
+ 966: {region: 0x69, script: 0x57, flags: 0x0},
+ 967: {region: 0x165, script: 0x57, flags: 0x0},
+ 968: {region: 0x99, script: 0x21, flags: 0x0},
+ 969: {region: 0x165, script: 0x57, flags: 0x0},
+ 970: {region: 0x102, script: 0x57, flags: 0x0},
+ 971: {region: 0x95, script: 0x57, flags: 0x0},
+ 972: {region: 0x165, script: 0x57, flags: 0x0},
+ 973: {region: 0x165, script: 0x57, flags: 0x0},
+ 974: {region: 0x9e, script: 0x57, flags: 0x0},
+ 975: {region: 0x165, script: 0x5, flags: 0x0},
+ 976: {region: 0x99, script: 0x57, flags: 0x0},
+ 977: {region: 0x31, script: 0x2, flags: 0x1},
+ 978: {region: 0xdb, script: 0x21, flags: 0x0},
+ 979: {region: 0x35, script: 0xe, flags: 0x0},
+ 980: {region: 0x4e, script: 0x57, flags: 0x0},
+ 981: {region: 0x72, script: 0x57, flags: 0x0},
+ 982: {region: 0x4e, script: 0x57, flags: 0x0},
+ 983: {region: 0x9c, script: 0x5, flags: 0x0},
+ 984: {region: 0x10c, script: 0x57, flags: 0x0},
+ 985: {region: 0x3a, script: 0x57, flags: 0x0},
+ 986: {region: 0x165, script: 0x57, flags: 0x0},
+ 987: {region: 0xd1, script: 0x57, flags: 0x0},
+ 988: {region: 0x104, script: 0x57, flags: 0x0},
+ 989: {region: 0x95, script: 0x57, flags: 0x0},
+ 990: {region: 0x12f, script: 0x57, flags: 0x0},
+ 991: {region: 0x165, script: 0x57, flags: 0x0},
+ 992: {region: 0x165, script: 0x57, flags: 0x0},
+ 993: {region: 0x73, script: 0x57, flags: 0x0},
+ 994: {region: 0x106, script: 0x1f, flags: 0x0},
+ 995: {region: 0x130, script: 0x1f, flags: 0x0},
+ 996: {region: 0x109, script: 0x57, flags: 0x0},
+ 997: {region: 0x107, script: 0x57, flags: 0x0},
+ 998: {region: 0x12f, script: 0x57, flags: 0x0},
+ 999: {region: 0x165, script: 0x57, flags: 0x0},
+ 1000: {region: 0xa2, script: 0x49, flags: 0x0},
+ 1001: {region: 0x99, script: 0x21, flags: 0x0},
+ 1002: {region: 0x80, script: 0x57, flags: 0x0},
+ 1003: {region: 0x106, script: 0x1f, flags: 0x0},
+ 1004: {region: 0xa4, script: 0x57, flags: 0x0},
+ 1005: {region: 0x95, script: 0x57, flags: 0x0},
+ 1006: {region: 0x99, script: 0x57, flags: 0x0},
+ 1007: {region: 0x114, script: 0x57, flags: 0x0},
+ 1008: {region: 0x99, script: 0xc3, flags: 0x0},
+ 1009: {region: 0x165, script: 0x57, flags: 0x0},
+ 1010: {region: 0x165, script: 0x57, flags: 0x0},
+ 1011: {region: 0x12f, script: 0x57, flags: 0x0},
+ 1012: {region: 0x9e, script: 0x57, flags: 0x0},
+ 1013: {region: 0x99, script: 0x21, flags: 0x0},
+ 1014: {region: 0x165, script: 0x5, flags: 0x0},
+ 1015: {region: 0x9e, script: 0x57, flags: 0x0},
+ 1016: {region: 0x7b, script: 0x57, flags: 0x0},
+ 1017: {region: 0x49, script: 0x57, flags: 0x0},
+ 1018: {region: 0x33, script: 0x4, flags: 0x1},
+ 1019: {region: 0x9e, script: 0x57, flags: 0x0},
+ 1020: {region: 0x9c, script: 0x5, flags: 0x0},
+ 1021: {region: 0xda, script: 0x57, flags: 0x0},
+ 1022: {region: 0x4f, script: 0x57, flags: 0x0},
+ 1023: {region: 0xd1, script: 0x57, flags: 0x0},
+ 1024: {region: 0xcf, script: 0x57, flags: 0x0},
+ 1025: {region: 0xc3, script: 0x57, flags: 0x0},
+ 1026: {region: 0x4c, script: 0x57, flags: 0x0},
+ 1027: {region: 0x96, script: 0x7a, flags: 0x0},
+ 1028: {region: 0xb6, script: 0x57, flags: 0x0},
+ 1029: {region: 0x165, script: 0x29, flags: 0x0},
+ 1030: {region: 0x165, script: 0x57, flags: 0x0},
+ 1032: {region: 0xba, script: 0xdc, flags: 0x0},
+ 1033: {region: 0x165, script: 0x57, flags: 0x0},
+ 1034: {region: 0xc4, script: 0x72, flags: 0x0},
+ 1035: {region: 0x165, script: 0x5, flags: 0x0},
+ 1036: {region: 0xb3, script: 0xca, flags: 0x0},
+ 1037: {region: 0x6f, script: 0x57, flags: 0x0},
+ 1038: {region: 0x165, script: 0x57, flags: 0x0},
+ 1039: {region: 0x165, script: 0x57, flags: 0x0},
+ 1040: {region: 0x165, script: 0x57, flags: 0x0},
+ 1041: {region: 0x165, script: 0x57, flags: 0x0},
+ 1042: {region: 0x111, script: 0x57, flags: 0x0},
+ 1043: {region: 0x165, script: 0x57, flags: 0x0},
+ 1044: {region: 0xe8, script: 0x5, flags: 0x0},
+ 1045: {region: 0x165, script: 0x57, flags: 0x0},
+ 1046: {region: 0x10f, script: 0x57, flags: 0x0},
+ 1047: {region: 0x165, script: 0x57, flags: 0x0},
+ 1048: {region: 0xe9, script: 0x57, flags: 0x0},
+ 1049: {region: 0x165, script: 0x57, flags: 0x0},
+ 1050: {region: 0x95, script: 0x57, flags: 0x0},
+ 1051: {region: 0x142, script: 0x57, flags: 0x0},
+ 1052: {region: 0x10c, script: 0x57, flags: 0x0},
+ 1054: {region: 0x10c, script: 0x57, flags: 0x0},
+ 1055: {region: 0x72, script: 0x57, flags: 0x0},
+ 1056: {region: 0x97, script: 0xc0, flags: 0x0},
+ 1057: {region: 0x165, script: 0x57, flags: 0x0},
+ 1058: {region: 0x72, script: 0x57, flags: 0x0},
+ 1059: {region: 0x164, script: 0x57, flags: 0x0},
+ 1060: {region: 0x165, script: 0x57, flags: 0x0},
+ 1061: {region: 0xc3, script: 0x57, flags: 0x0},
+ 1062: {region: 0x165, script: 0x57, flags: 0x0},
+ 1063: {region: 0x165, script: 0x57, flags: 0x0},
+ 1064: {region: 0x165, script: 0x57, flags: 0x0},
+ 1065: {region: 0x115, script: 0x57, flags: 0x0},
+ 1066: {region: 0x165, script: 0x57, flags: 0x0},
+ 1067: {region: 0x165, script: 0x57, flags: 0x0},
+ 1068: {region: 0x123, script: 0xdf, flags: 0x0},
+ 1069: {region: 0x165, script: 0x57, flags: 0x0},
+ 1070: {region: 0x165, script: 0x57, flags: 0x0},
+ 1071: {region: 0x165, script: 0x57, flags: 0x0},
+ 1072: {region: 0x165, script: 0x57, flags: 0x0},
+ 1073: {region: 0x27, script: 0x57, flags: 0x0},
+ 1074: {region: 0x37, script: 0x5, flags: 0x1},
+ 1075: {region: 0x99, script: 0xcb, flags: 0x0},
+ 1076: {region: 0x116, script: 0x57, flags: 0x0},
+ 1077: {region: 0x114, script: 0x57, flags: 0x0},
+ 1078: {region: 0x99, script: 0x21, flags: 0x0},
+ 1079: {region: 0x161, script: 0x57, flags: 0x0},
+ 1080: {region: 0x165, script: 0x57, flags: 0x0},
+ 1081: {region: 0x165, script: 0x57, flags: 0x0},
+ 1082: {region: 0x6d, script: 0x57, flags: 0x0},
+ 1083: {region: 0x161, script: 0x57, flags: 0x0},
+ 1084: {region: 0x165, script: 0x57, flags: 0x0},
+ 1085: {region: 0x60, script: 0x57, flags: 0x0},
+ 1086: {region: 0x95, script: 0x57, flags: 0x0},
+ 1087: {region: 0x165, script: 0x57, flags: 0x0},
+ 1088: {region: 0x165, script: 0x57, flags: 0x0},
+ 1089: {region: 0x12f, script: 0x57, flags: 0x0},
+ 1090: {region: 0x165, script: 0x57, flags: 0x0},
+ 1091: {region: 0x84, script: 0x57, flags: 0x0},
+ 1092: {region: 0x10c, script: 0x57, flags: 0x0},
+ 1093: {region: 0x12f, script: 0x57, flags: 0x0},
+ 1094: {region: 0x15f, script: 0x5, flags: 0x0},
+ 1095: {region: 0x4b, script: 0x57, flags: 0x0},
+ 1096: {region: 0x60, script: 0x57, flags: 0x0},
+ 1097: {region: 0x165, script: 0x57, flags: 0x0},
+ 1098: {region: 0x99, script: 0x21, flags: 0x0},
+ 1099: {region: 0x95, script: 0x57, flags: 0x0},
+ 1100: {region: 0x165, script: 0x57, flags: 0x0},
+ 1101: {region: 0x35, script: 0xe, flags: 0x0},
+ 1102: {region: 0x9b, script: 0xcf, flags: 0x0},
+ 1103: {region: 0xe9, script: 0x57, flags: 0x0},
+ 1104: {region: 0x99, script: 0xd7, flags: 0x0},
+ 1105: {region: 0xdb, script: 0x21, flags: 0x0},
+ 1106: {region: 0x165, script: 0x57, flags: 0x0},
+ 1107: {region: 0x165, script: 0x57, flags: 0x0},
+ 1108: {region: 0x165, script: 0x57, flags: 0x0},
+ 1109: {region: 0x165, script: 0x57, flags: 0x0},
+ 1110: {region: 0x165, script: 0x57, flags: 0x0},
+ 1111: {region: 0x165, script: 0x57, flags: 0x0},
+ 1112: {region: 0x165, script: 0x57, flags: 0x0},
+ 1113: {region: 0x165, script: 0x57, flags: 0x0},
+ 1114: {region: 0xe7, script: 0x57, flags: 0x0},
+ 1115: {region: 0x165, script: 0x57, flags: 0x0},
+ 1116: {region: 0x165, script: 0x57, flags: 0x0},
+ 1117: {region: 0x99, script: 0x4f, flags: 0x0},
+ 1118: {region: 0x53, script: 0xd5, flags: 0x0},
+ 1119: {region: 0xdb, script: 0x21, flags: 0x0},
+ 1120: {region: 0xdb, script: 0x21, flags: 0x0},
+ 1121: {region: 0x99, script: 0xda, flags: 0x0},
+ 1122: {region: 0x165, script: 0x57, flags: 0x0},
+ 1123: {region: 0x112, script: 0x57, flags: 0x0},
+ 1124: {region: 0x131, script: 0x57, flags: 0x0},
+ 1125: {region: 0x126, script: 0x57, flags: 0x0},
+ 1126: {region: 0x165, script: 0x57, flags: 0x0},
+ 1127: {region: 0x3c, script: 0x3, flags: 0x1},
+ 1128: {region: 0x165, script: 0x57, flags: 0x0},
+ 1129: {region: 0x165, script: 0x57, flags: 0x0},
+ 1130: {region: 0x165, script: 0x57, flags: 0x0},
+ 1131: {region: 0x123, script: 0xdf, flags: 0x0},
+ 1132: {region: 0xdb, script: 0x21, flags: 0x0},
+ 1133: {region: 0xdb, script: 0x21, flags: 0x0},
+ 1134: {region: 0xdb, script: 0x21, flags: 0x0},
+ 1135: {region: 0x6f, script: 0x29, flags: 0x0},
+ 1136: {region: 0x165, script: 0x57, flags: 0x0},
+ 1137: {region: 0x6d, script: 0x29, flags: 0x0},
+ 1138: {region: 0x165, script: 0x57, flags: 0x0},
+ 1139: {region: 0x165, script: 0x57, flags: 0x0},
+ 1140: {region: 0x165, script: 0x57, flags: 0x0},
+ 1141: {region: 0xd6, script: 0x57, flags: 0x0},
+ 1142: {region: 0x127, script: 0x57, flags: 0x0},
+ 1143: {region: 0x125, script: 0x57, flags: 0x0},
+ 1144: {region: 0x32, script: 0x57, flags: 0x0},
+ 1145: {region: 0xdb, script: 0x21, flags: 0x0},
+ 1146: {region: 0xe7, script: 0x57, flags: 0x0},
+ 1147: {region: 0x165, script: 0x57, flags: 0x0},
+ 1148: {region: 0x165, script: 0x57, flags: 0x0},
+ 1149: {region: 0x32, script: 0x57, flags: 0x0},
+ 1150: {region: 0xd4, script: 0x57, flags: 0x0},
+ 1151: {region: 0x165, script: 0x57, flags: 0x0},
+ 1152: {region: 0x161, script: 0x57, flags: 0x0},
+ 1153: {region: 0x165, script: 0x57, flags: 0x0},
+ 1154: {region: 0x129, script: 0x57, flags: 0x0},
+ 1155: {region: 0x165, script: 0x57, flags: 0x0},
+ 1156: {region: 0xce, script: 0x57, flags: 0x0},
+ 1157: {region: 0x165, script: 0x57, flags: 0x0},
+ 1158: {region: 0xe6, script: 0x57, flags: 0x0},
+ 1159: {region: 0x165, script: 0x57, flags: 0x0},
+ 1160: {region: 0x165, script: 0x57, flags: 0x0},
+ 1161: {region: 0x165, script: 0x57, flags: 0x0},
+ 1162: {region: 0x12b, script: 0x57, flags: 0x0},
+ 1163: {region: 0x12b, script: 0x57, flags: 0x0},
+ 1164: {region: 0x12e, script: 0x57, flags: 0x0},
+ 1165: {region: 0x165, script: 0x5, flags: 0x0},
+ 1166: {region: 0x161, script: 0x57, flags: 0x0},
+ 1167: {region: 0x87, script: 0x31, flags: 0x0},
+ 1168: {region: 0xdb, script: 0x21, flags: 0x0},
+ 1169: {region: 0xe7, script: 0x57, flags: 0x0},
+ 1170: {region: 0x43, script: 0xe0, flags: 0x0},
+ 1171: {region: 0x165, script: 0x57, flags: 0x0},
+ 1172: {region: 0x106, script: 0x1f, flags: 0x0},
+ 1173: {region: 0x165, script: 0x57, flags: 0x0},
+ 1174: {region: 0x165, script: 0x57, flags: 0x0},
+ 1175: {region: 0x131, script: 0x57, flags: 0x0},
+ 1176: {region: 0x165, script: 0x57, flags: 0x0},
+ 1177: {region: 0x123, script: 0xdf, flags: 0x0},
+ 1178: {region: 0x32, script: 0x57, flags: 0x0},
+ 1179: {region: 0x165, script: 0x57, flags: 0x0},
+ 1180: {region: 0x165, script: 0x57, flags: 0x0},
+ 1181: {region: 0xce, script: 0x57, flags: 0x0},
+ 1182: {region: 0x165, script: 0x57, flags: 0x0},
+ 1183: {region: 0x165, script: 0x57, flags: 0x0},
+ 1184: {region: 0x12d, script: 0x57, flags: 0x0},
+ 1185: {region: 0x165, script: 0x57, flags: 0x0},
+ 1187: {region: 0x165, script: 0x57, flags: 0x0},
+ 1188: {region: 0xd4, script: 0x57, flags: 0x0},
+ 1189: {region: 0x53, script: 0xd8, flags: 0x0},
+ 1190: {region: 0xe5, script: 0x57, flags: 0x0},
+ 1191: {region: 0x165, script: 0x57, flags: 0x0},
+ 1192: {region: 0x106, script: 0x1f, flags: 0x0},
+ 1193: {region: 0xba, script: 0x57, flags: 0x0},
+ 1194: {region: 0x165, script: 0x57, flags: 0x0},
+ 1195: {region: 0x106, script: 0x1f, flags: 0x0},
+ 1196: {region: 0x3f, script: 0x4, flags: 0x1},
+ 1197: {region: 0x11c, script: 0xe2, flags: 0x0},
+ 1198: {region: 0x130, script: 0x1f, flags: 0x0},
+ 1199: {region: 0x75, script: 0x57, flags: 0x0},
+ 1200: {region: 0x2a, script: 0x57, flags: 0x0},
+ 1202: {region: 0x43, script: 0x3, flags: 0x1},
+ 1203: {region: 0x99, script: 0xe, flags: 0x0},
+ 1204: {region: 0xe8, script: 0x5, flags: 0x0},
+ 1205: {region: 0x165, script: 0x57, flags: 0x0},
+ 1206: {region: 0x165, script: 0x57, flags: 0x0},
+ 1207: {region: 0x165, script: 0x57, flags: 0x0},
+ 1208: {region: 0x165, script: 0x57, flags: 0x0},
+ 1209: {region: 0x165, script: 0x57, flags: 0x0},
+ 1210: {region: 0x165, script: 0x57, flags: 0x0},
+ 1211: {region: 0x165, script: 0x57, flags: 0x0},
+ 1212: {region: 0x46, script: 0x4, flags: 0x1},
+ 1213: {region: 0x165, script: 0x57, flags: 0x0},
+ 1214: {region: 0xb4, script: 0xe3, flags: 0x0},
+ 1215: {region: 0x165, script: 0x57, flags: 0x0},
+ 1216: {region: 0x161, script: 0x57, flags: 0x0},
+ 1217: {region: 0x9e, script: 0x57, flags: 0x0},
+ 1218: {region: 0x106, script: 0x57, flags: 0x0},
+ 1219: {region: 0x13e, script: 0x57, flags: 0x0},
+ 1220: {region: 0x11b, script: 0x57, flags: 0x0},
+ 1221: {region: 0x165, script: 0x57, flags: 0x0},
+ 1222: {region: 0x36, script: 0x57, flags: 0x0},
+ 1223: {region: 0x60, script: 0x57, flags: 0x0},
+ 1224: {region: 0xd1, script: 0x57, flags: 0x0},
+ 1225: {region: 0x1, script: 0x57, flags: 0x0},
+ 1226: {region: 0x106, script: 0x57, flags: 0x0},
+ 1227: {region: 0x6a, script: 0x57, flags: 0x0},
+ 1228: {region: 0x12f, script: 0x57, flags: 0x0},
+ 1229: {region: 0x165, script: 0x57, flags: 0x0},
+ 1230: {region: 0x36, script: 0x57, flags: 0x0},
+ 1231: {region: 0x4e, script: 0x57, flags: 0x0},
+ 1232: {region: 0x165, script: 0x57, flags: 0x0},
+ 1233: {region: 0x6f, script: 0x29, flags: 0x0},
+ 1234: {region: 0x165, script: 0x57, flags: 0x0},
+ 1235: {region: 0xe7, script: 0x57, flags: 0x0},
+ 1236: {region: 0x2f, script: 0x57, flags: 0x0},
+ 1237: {region: 0x99, script: 0xda, flags: 0x0},
+ 1238: {region: 0x99, script: 0x21, flags: 0x0},
+ 1239: {region: 0x165, script: 0x57, flags: 0x0},
+ 1240: {region: 0x165, script: 0x57, flags: 0x0},
+ 1241: {region: 0x165, script: 0x57, flags: 0x0},
+ 1242: {region: 0x165, script: 0x57, flags: 0x0},
+ 1243: {region: 0x165, script: 0x57, flags: 0x0},
+ 1244: {region: 0x165, script: 0x57, flags: 0x0},
+ 1245: {region: 0x165, script: 0x57, flags: 0x0},
+ 1246: {region: 0x165, script: 0x57, flags: 0x0},
+ 1247: {region: 0x165, script: 0x57, flags: 0x0},
+ 1248: {region: 0x140, script: 0x57, flags: 0x0},
+ 1249: {region: 0x165, script: 0x57, flags: 0x0},
+ 1250: {region: 0x165, script: 0x57, flags: 0x0},
+ 1251: {region: 0xa8, script: 0x5, flags: 0x0},
+ 1252: {region: 0x165, script: 0x57, flags: 0x0},
+ 1253: {region: 0x114, script: 0x57, flags: 0x0},
+ 1254: {region: 0x165, script: 0x57, flags: 0x0},
+ 1255: {region: 0x165, script: 0x57, flags: 0x0},
+ 1256: {region: 0x165, script: 0x57, flags: 0x0},
+ 1257: {region: 0x165, script: 0x57, flags: 0x0},
+ 1258: {region: 0x99, script: 0x21, flags: 0x0},
+ 1259: {region: 0x53, script: 0x38, flags: 0x0},
+ 1260: {region: 0x165, script: 0x57, flags: 0x0},
+ 1261: {region: 0x165, script: 0x57, flags: 0x0},
+ 1262: {region: 0x41, script: 0x57, flags: 0x0},
+ 1263: {region: 0x165, script: 0x57, flags: 0x0},
+ 1264: {region: 0x12b, script: 0x18, flags: 0x0},
+ 1265: {region: 0x165, script: 0x57, flags: 0x0},
+ 1266: {region: 0x161, script: 0x57, flags: 0x0},
+ 1267: {region: 0x165, script: 0x57, flags: 0x0},
+ 1268: {region: 0x12b, script: 0x5f, flags: 0x0},
+ 1269: {region: 0x12b, script: 0x60, flags: 0x0},
+ 1270: {region: 0x7d, script: 0x2b, flags: 0x0},
+ 1271: {region: 0x53, script: 0x64, flags: 0x0},
+ 1272: {region: 0x10b, script: 0x69, flags: 0x0},
+ 1273: {region: 0x108, script: 0x73, flags: 0x0},
+ 1274: {region: 0x99, script: 0x21, flags: 0x0},
+ 1275: {region: 0x131, script: 0x57, flags: 0x0},
+ 1276: {region: 0x165, script: 0x57, flags: 0x0},
+ 1277: {region: 0x9c, script: 0x8a, flags: 0x0},
+ 1278: {region: 0x165, script: 0x57, flags: 0x0},
+ 1279: {region: 0x15e, script: 0xc2, flags: 0x0},
+ 1280: {region: 0x165, script: 0x57, flags: 0x0},
+ 1281: {region: 0x165, script: 0x57, flags: 0x0},
+ 1282: {region: 0xdb, script: 0x21, flags: 0x0},
+ 1283: {region: 0x165, script: 0x57, flags: 0x0},
+ 1284: {region: 0x165, script: 0x57, flags: 0x0},
+ 1285: {region: 0xd1, script: 0x57, flags: 0x0},
+ 1286: {region: 0x75, script: 0x57, flags: 0x0},
+ 1287: {region: 0x165, script: 0x57, flags: 0x0},
+ 1288: {region: 0x165, script: 0x57, flags: 0x0},
+ 1289: {region: 0x52, script: 0x57, flags: 0x0},
+ 1290: {region: 0x165, script: 0x57, flags: 0x0},
+ 1291: {region: 0x165, script: 0x57, flags: 0x0},
+ 1292: {region: 0x165, script: 0x57, flags: 0x0},
+ 1293: {region: 0x52, script: 0x57, flags: 0x0},
+ 1294: {region: 0x165, script: 0x57, flags: 0x0},
+ 1295: {region: 0x165, script: 0x57, flags: 0x0},
+ 1296: {region: 0x165, script: 0x57, flags: 0x0},
+ 1297: {region: 0x165, script: 0x57, flags: 0x0},
+ 1298: {region: 0x1, script: 0x3b, flags: 0x0},
+ 1299: {region: 0x165, script: 0x57, flags: 0x0},
+ 1300: {region: 0x165, script: 0x57, flags: 0x0},
+ 1301: {region: 0x165, script: 0x57, flags: 0x0},
+ 1302: {region: 0x165, script: 0x57, flags: 0x0},
+ 1303: {region: 0x165, script: 0x57, flags: 0x0},
+ 1304: {region: 0xd6, script: 0x57, flags: 0x0},
+ 1305: {region: 0x165, script: 0x57, flags: 0x0},
+ 1306: {region: 0x165, script: 0x57, flags: 0x0},
+ 1307: {region: 0x165, script: 0x57, flags: 0x0},
+ 1308: {region: 0x41, script: 0x57, flags: 0x0},
+ 1309: {region: 0x165, script: 0x57, flags: 0x0},
+ 1310: {region: 0xcf, script: 0x57, flags: 0x0},
+ 1311: {region: 0x4a, script: 0x3, flags: 0x1},
+ 1312: {region: 0x165, script: 0x57, flags: 0x0},
+ 1313: {region: 0x165, script: 0x57, flags: 0x0},
+ 1314: {region: 0x165, script: 0x57, flags: 0x0},
+ 1315: {region: 0x53, script: 0x57, flags: 0x0},
+ 1316: {region: 0x10b, script: 0x57, flags: 0x0},
+ 1318: {region: 0xa8, script: 0x5, flags: 0x0},
+ 1319: {region: 0xd9, script: 0x57, flags: 0x0},
+ 1320: {region: 0xba, script: 0xdc, flags: 0x0},
+ 1321: {region: 0x4d, script: 0x14, flags: 0x1},
+ 1322: {region: 0x53, script: 0x79, flags: 0x0},
+ 1323: {region: 0x165, script: 0x57, flags: 0x0},
+ 1324: {region: 0x122, script: 0x57, flags: 0x0},
+ 1325: {region: 0xd0, script: 0x57, flags: 0x0},
+ 1326: {region: 0x165, script: 0x57, flags: 0x0},
+ 1327: {region: 0x161, script: 0x57, flags: 0x0},
+ 1329: {region: 0x12b, script: 0x57, flags: 0x0},
+}
+
+// likelyLangList holds lists info associated with likelyLang.
+// Size: 388 bytes, 97 elements
+var likelyLangList = [97]likelyScriptRegion{
+ 0: {region: 0x9c, script: 0x7, flags: 0x0},
+ 1: {region: 0xa1, script: 0x74, flags: 0x2},
+ 2: {region: 0x11c, script: 0x80, flags: 0x2},
+ 3: {region: 0x32, script: 0x57, flags: 0x0},
+ 4: {region: 0x9b, script: 0x5, flags: 0x4},
+ 5: {region: 0x9c, script: 0x5, flags: 0x4},
+ 6: {region: 0x106, script: 0x1f, flags: 0x4},
+ 7: {region: 0x9c, script: 0x5, flags: 0x2},
+ 8: {region: 0x106, script: 0x1f, flags: 0x0},
+ 9: {region: 0x38, script: 0x2c, flags: 0x2},
+ 10: {region: 0x135, script: 0x57, flags: 0x0},
+ 11: {region: 0x7b, script: 0xc5, flags: 0x2},
+ 12: {region: 0x114, script: 0x57, flags: 0x0},
+ 13: {region: 0x84, script: 0x1, flags: 0x2},
+ 14: {region: 0x5d, script: 0x1e, flags: 0x0},
+ 15: {region: 0x87, script: 0x5c, flags: 0x2},
+ 16: {region: 0xd6, script: 0x57, flags: 0x0},
+ 17: {region: 0x52, script: 0x5, flags: 0x4},
+ 18: {region: 0x10b, script: 0x5, flags: 0x4},
+ 19: {region: 0xae, script: 0x1f, flags: 0x0},
+ 20: {region: 0x24, script: 0x5, flags: 0x4},
+ 21: {region: 0x53, script: 0x5, flags: 0x4},
+ 22: {region: 0x9c, script: 0x5, flags: 0x4},
+ 23: {region: 0xc5, script: 0x5, flags: 0x4},
+ 24: {region: 0x53, script: 0x5, flags: 0x2},
+ 25: {region: 0x12b, script: 0x57, flags: 0x0},
+ 26: {region: 0xb0, script: 0x5, flags: 0x4},
+ 27: {region: 0x9b, script: 0x5, flags: 0x2},
+ 28: {region: 0xa5, script: 0x1f, flags: 0x0},
+ 29: {region: 0x53, script: 0x5, flags: 0x4},
+ 30: {region: 0x12b, script: 0x57, flags: 0x4},
+ 31: {region: 0x53, script: 0x5, flags: 0x2},
+ 32: {region: 0x12b, script: 0x57, flags: 0x2},
+ 33: {region: 0xdb, script: 0x21, flags: 0x0},
+ 34: {region: 0x99, script: 0x5a, flags: 0x2},
+ 35: {region: 0x83, script: 0x57, flags: 0x0},
+ 36: {region: 0x84, script: 0x78, flags: 0x4},
+ 37: {region: 0x84, script: 0x78, flags: 0x2},
+ 38: {region: 0xc5, script: 0x1f, flags: 0x0},
+ 39: {region: 0x53, script: 0x6d, flags: 0x4},
+ 40: {region: 0x53, script: 0x6d, flags: 0x2},
+ 41: {region: 0xd0, script: 0x57, flags: 0x0},
+ 42: {region: 0x4a, script: 0x5, flags: 0x4},
+ 43: {region: 0x95, script: 0x5, flags: 0x4},
+ 44: {region: 0x99, script: 0x33, flags: 0x0},
+ 45: {region: 0xe8, script: 0x5, flags: 0x4},
+ 46: {region: 0xe8, script: 0x5, flags: 0x2},
+ 47: {region: 0x9c, script: 0x84, flags: 0x0},
+ 48: {region: 0x53, script: 0x85, flags: 0x2},
+ 49: {region: 0xba, script: 0xdc, flags: 0x0},
+ 50: {region: 0xd9, script: 0x57, flags: 0x4},
+ 51: {region: 0xe8, script: 0x5, flags: 0x0},
+ 52: {region: 0x99, script: 0x21, flags: 0x2},
+ 53: {region: 0x99, script: 0x4c, flags: 0x2},
+ 54: {region: 0x99, script: 0xc9, flags: 0x2},
+ 55: {region: 0x105, script: 0x1f, flags: 0x0},
+ 56: {region: 0xbd, script: 0x57, flags: 0x4},
+ 57: {region: 0x104, script: 0x57, flags: 0x4},
+ 58: {region: 0x106, script: 0x57, flags: 0x4},
+ 59: {region: 0x12b, script: 0x57, flags: 0x4},
+ 60: {region: 0x124, script: 0x1f, flags: 0x0},
+ 61: {region: 0xe8, script: 0x5, flags: 0x4},
+ 62: {region: 0xe8, script: 0x5, flags: 0x2},
+ 63: {region: 0x53, script: 0x5, flags: 0x0},
+ 64: {region: 0xae, script: 0x1f, flags: 0x4},
+ 65: {region: 0xc5, script: 0x1f, flags: 0x4},
+ 66: {region: 0xae, script: 0x1f, flags: 0x2},
+ 67: {region: 0x99, script: 0xe, flags: 0x0},
+ 68: {region: 0xdb, script: 0x21, flags: 0x4},
+ 69: {region: 0xdb, script: 0x21, flags: 0x2},
+ 70: {region: 0x137, script: 0x57, flags: 0x0},
+ 71: {region: 0x24, script: 0x5, flags: 0x4},
+ 72: {region: 0x53, script: 0x1f, flags: 0x4},
+ 73: {region: 0x24, script: 0x5, flags: 0x2},
+ 74: {region: 0x8d, script: 0x39, flags: 0x0},
+ 75: {region: 0x53, script: 0x38, flags: 0x4},
+ 76: {region: 0x53, script: 0x38, flags: 0x2},
+ 77: {region: 0x53, script: 0x38, flags: 0x0},
+ 78: {region: 0x2f, script: 0x39, flags: 0x4},
+ 79: {region: 0x3e, script: 0x39, flags: 0x4},
+ 80: {region: 0x7b, script: 0x39, flags: 0x4},
+ 81: {region: 0x7e, script: 0x39, flags: 0x4},
+ 82: {region: 0x8d, script: 0x39, flags: 0x4},
+ 83: {region: 0x95, script: 0x39, flags: 0x4},
+ 84: {region: 0xc6, script: 0x39, flags: 0x4},
+ 85: {region: 0xd0, script: 0x39, flags: 0x4},
+ 86: {region: 0xe2, script: 0x39, flags: 0x4},
+ 87: {region: 0xe5, script: 0x39, flags: 0x4},
+ 88: {region: 0xe7, script: 0x39, flags: 0x4},
+ 89: {region: 0x116, script: 0x39, flags: 0x4},
+ 90: {region: 0x123, script: 0x39, flags: 0x4},
+ 91: {region: 0x12e, script: 0x39, flags: 0x4},
+ 92: {region: 0x135, script: 0x39, flags: 0x4},
+ 93: {region: 0x13e, script: 0x39, flags: 0x4},
+ 94: {region: 0x12e, script: 0x11, flags: 0x2},
+ 95: {region: 0x12e, script: 0x34, flags: 0x2},
+ 96: {region: 0x12e, script: 0x39, flags: 0x2},
+}
+
+type likelyLangScript struct {
+ lang uint16
+ script uint8
+ flags uint8
+}
+
+// likelyRegion is a lookup table, indexed by regionID, for the most likely
+// languages and scripts given incomplete information. If more entries exist
+// for a given regionID, lang and script are the index and size respectively
+// of the list in likelyRegionList.
+// TODO: exclude containers and user-definable regions from the list.
+// Size: 1432 bytes, 358 elements
+var likelyRegion = [358]likelyLangScript{
+ 34: {lang: 0xd7, script: 0x57, flags: 0x0},
+ 35: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 36: {lang: 0x0, script: 0x2, flags: 0x1},
+ 39: {lang: 0x2, script: 0x2, flags: 0x1},
+ 40: {lang: 0x4, script: 0x2, flags: 0x1},
+ 42: {lang: 0x3c0, script: 0x57, flags: 0x0},
+ 43: {lang: 0x0, script: 0x57, flags: 0x0},
+ 44: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 45: {lang: 0x41b, script: 0x57, flags: 0x0},
+ 46: {lang: 0x10d, script: 0x57, flags: 0x0},
+ 48: {lang: 0x367, script: 0x57, flags: 0x0},
+ 49: {lang: 0x444, script: 0x57, flags: 0x0},
+ 50: {lang: 0x58, script: 0x57, flags: 0x0},
+ 51: {lang: 0x6, script: 0x2, flags: 0x1},
+ 53: {lang: 0xa5, script: 0xe, flags: 0x0},
+ 54: {lang: 0x367, script: 0x57, flags: 0x0},
+ 55: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 56: {lang: 0x7e, script: 0x1f, flags: 0x0},
+ 57: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 58: {lang: 0x3d9, script: 0x57, flags: 0x0},
+ 59: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 60: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 62: {lang: 0x31f, script: 0x57, flags: 0x0},
+ 63: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 64: {lang: 0x3a1, script: 0x57, flags: 0x0},
+ 65: {lang: 0x3c0, script: 0x57, flags: 0x0},
+ 67: {lang: 0x8, script: 0x2, flags: 0x1},
+ 69: {lang: 0x0, script: 0x57, flags: 0x0},
+ 71: {lang: 0x71, script: 0x1f, flags: 0x0},
+ 73: {lang: 0x512, script: 0x3b, flags: 0x2},
+ 74: {lang: 0x31f, script: 0x5, flags: 0x2},
+ 75: {lang: 0x445, script: 0x57, flags: 0x0},
+ 76: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 77: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 78: {lang: 0x10d, script: 0x57, flags: 0x0},
+ 79: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 81: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 82: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 83: {lang: 0xa, script: 0x4, flags: 0x1},
+ 84: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 85: {lang: 0x0, script: 0x57, flags: 0x0},
+ 86: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 89: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 90: {lang: 0x3c0, script: 0x57, flags: 0x0},
+ 91: {lang: 0x3a1, script: 0x57, flags: 0x0},
+ 93: {lang: 0xe, script: 0x2, flags: 0x1},
+ 94: {lang: 0xfa, script: 0x57, flags: 0x0},
+ 96: {lang: 0x10d, script: 0x57, flags: 0x0},
+ 98: {lang: 0x1, script: 0x57, flags: 0x0},
+ 99: {lang: 0x101, script: 0x57, flags: 0x0},
+ 101: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 103: {lang: 0x10, script: 0x2, flags: 0x1},
+ 104: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 105: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 106: {lang: 0x140, script: 0x57, flags: 0x0},
+ 107: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 108: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 109: {lang: 0x46f, script: 0x29, flags: 0x0},
+ 110: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 111: {lang: 0x12, script: 0x2, flags: 0x1},
+ 113: {lang: 0x10d, script: 0x57, flags: 0x0},
+ 114: {lang: 0x151, script: 0x57, flags: 0x0},
+ 115: {lang: 0x1c0, script: 0x21, flags: 0x2},
+ 118: {lang: 0x158, script: 0x57, flags: 0x0},
+ 120: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 122: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 123: {lang: 0x14, script: 0x2, flags: 0x1},
+ 125: {lang: 0x16, script: 0x3, flags: 0x1},
+ 126: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 128: {lang: 0x21, script: 0x57, flags: 0x0},
+ 130: {lang: 0x245, script: 0x57, flags: 0x0},
+ 132: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 133: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 134: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 135: {lang: 0x19, script: 0x2, flags: 0x1},
+ 136: {lang: 0x0, script: 0x57, flags: 0x0},
+ 137: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 139: {lang: 0x3c0, script: 0x57, flags: 0x0},
+ 141: {lang: 0x529, script: 0x39, flags: 0x0},
+ 142: {lang: 0x0, script: 0x57, flags: 0x0},
+ 143: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 144: {lang: 0x1d1, script: 0x57, flags: 0x0},
+ 145: {lang: 0x1d4, script: 0x57, flags: 0x0},
+ 146: {lang: 0x1d5, script: 0x57, flags: 0x0},
+ 148: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 149: {lang: 0x1b, script: 0x2, flags: 0x1},
+ 151: {lang: 0x1bc, script: 0x3b, flags: 0x0},
+ 153: {lang: 0x1d, script: 0x3, flags: 0x1},
+ 155: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 156: {lang: 0x20, script: 0x2, flags: 0x1},
+ 157: {lang: 0x1f8, script: 0x57, flags: 0x0},
+ 158: {lang: 0x1f9, script: 0x57, flags: 0x0},
+ 161: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 162: {lang: 0x200, script: 0x46, flags: 0x0},
+ 164: {lang: 0x445, script: 0x57, flags: 0x0},
+ 165: {lang: 0x28a, script: 0x1f, flags: 0x0},
+ 166: {lang: 0x22, script: 0x3, flags: 0x1},
+ 168: {lang: 0x25, script: 0x2, flags: 0x1},
+ 170: {lang: 0x254, script: 0x50, flags: 0x0},
+ 171: {lang: 0x254, script: 0x50, flags: 0x0},
+ 172: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 174: {lang: 0x3e2, script: 0x1f, flags: 0x0},
+ 175: {lang: 0x27, script: 0x2, flags: 0x1},
+ 176: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 178: {lang: 0x10d, script: 0x57, flags: 0x0},
+ 179: {lang: 0x40c, script: 0xca, flags: 0x0},
+ 181: {lang: 0x43b, script: 0x57, flags: 0x0},
+ 182: {lang: 0x2c0, script: 0x57, flags: 0x0},
+ 183: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 184: {lang: 0x2c7, script: 0x57, flags: 0x0},
+ 185: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 186: {lang: 0x29, script: 0x2, flags: 0x1},
+ 187: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 188: {lang: 0x2b, script: 0x2, flags: 0x1},
+ 189: {lang: 0x432, script: 0x57, flags: 0x0},
+ 190: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 191: {lang: 0x2f1, script: 0x57, flags: 0x0},
+ 194: {lang: 0x2d, script: 0x2, flags: 0x1},
+ 195: {lang: 0xa0, script: 0x57, flags: 0x0},
+ 196: {lang: 0x2f, script: 0x2, flags: 0x1},
+ 197: {lang: 0x31, script: 0x2, flags: 0x1},
+ 198: {lang: 0x33, script: 0x2, flags: 0x1},
+ 200: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 201: {lang: 0x35, script: 0x2, flags: 0x1},
+ 203: {lang: 0x320, script: 0x57, flags: 0x0},
+ 204: {lang: 0x37, script: 0x3, flags: 0x1},
+ 205: {lang: 0x128, script: 0xde, flags: 0x0},
+ 207: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 208: {lang: 0x31f, script: 0x57, flags: 0x0},
+ 209: {lang: 0x3c0, script: 0x57, flags: 0x0},
+ 210: {lang: 0x16, script: 0x57, flags: 0x0},
+ 211: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 212: {lang: 0x1b4, script: 0x57, flags: 0x0},
+ 214: {lang: 0x1b4, script: 0x5, flags: 0x2},
+ 216: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 217: {lang: 0x367, script: 0x57, flags: 0x0},
+ 218: {lang: 0x347, script: 0x57, flags: 0x0},
+ 219: {lang: 0x351, script: 0x21, flags: 0x0},
+ 225: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 226: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 228: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 229: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 230: {lang: 0x486, script: 0x57, flags: 0x0},
+ 231: {lang: 0x153, script: 0x57, flags: 0x0},
+ 232: {lang: 0x3a, script: 0x3, flags: 0x1},
+ 233: {lang: 0x3b3, script: 0x57, flags: 0x0},
+ 234: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 236: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 237: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 238: {lang: 0x3c0, script: 0x57, flags: 0x0},
+ 240: {lang: 0x3a2, script: 0x57, flags: 0x0},
+ 241: {lang: 0x194, script: 0x57, flags: 0x0},
+ 243: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 258: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 260: {lang: 0x3d, script: 0x2, flags: 0x1},
+ 261: {lang: 0x432, script: 0x1f, flags: 0x0},
+ 262: {lang: 0x3f, script: 0x2, flags: 0x1},
+ 263: {lang: 0x3e5, script: 0x57, flags: 0x0},
+ 264: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 266: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 267: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 268: {lang: 0x41, script: 0x2, flags: 0x1},
+ 271: {lang: 0x416, script: 0x57, flags: 0x0},
+ 272: {lang: 0x347, script: 0x57, flags: 0x0},
+ 273: {lang: 0x43, script: 0x2, flags: 0x1},
+ 275: {lang: 0x1f9, script: 0x57, flags: 0x0},
+ 276: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 277: {lang: 0x429, script: 0x57, flags: 0x0},
+ 278: {lang: 0x367, script: 0x57, flags: 0x0},
+ 280: {lang: 0x3c0, script: 0x57, flags: 0x0},
+ 282: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 284: {lang: 0x45, script: 0x2, flags: 0x1},
+ 288: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 289: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 290: {lang: 0x47, script: 0x2, flags: 0x1},
+ 291: {lang: 0x49, script: 0x3, flags: 0x1},
+ 292: {lang: 0x4c, script: 0x2, flags: 0x1},
+ 293: {lang: 0x477, script: 0x57, flags: 0x0},
+ 294: {lang: 0x3c0, script: 0x57, flags: 0x0},
+ 295: {lang: 0x476, script: 0x57, flags: 0x0},
+ 296: {lang: 0x4e, script: 0x2, flags: 0x1},
+ 297: {lang: 0x482, script: 0x57, flags: 0x0},
+ 299: {lang: 0x50, script: 0x4, flags: 0x1},
+ 301: {lang: 0x4a0, script: 0x57, flags: 0x0},
+ 302: {lang: 0x54, script: 0x2, flags: 0x1},
+ 303: {lang: 0x445, script: 0x57, flags: 0x0},
+ 304: {lang: 0x56, script: 0x3, flags: 0x1},
+ 305: {lang: 0x445, script: 0x57, flags: 0x0},
+ 309: {lang: 0x512, script: 0x3b, flags: 0x2},
+ 310: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 311: {lang: 0x4bc, script: 0x57, flags: 0x0},
+ 312: {lang: 0x1f9, script: 0x57, flags: 0x0},
+ 315: {lang: 0x13e, script: 0x57, flags: 0x0},
+ 318: {lang: 0x4c3, script: 0x57, flags: 0x0},
+ 319: {lang: 0x8a, script: 0x57, flags: 0x0},
+ 320: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 322: {lang: 0x41b, script: 0x57, flags: 0x0},
+ 333: {lang: 0x59, script: 0x2, flags: 0x1},
+ 350: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 351: {lang: 0x5b, script: 0x2, flags: 0x1},
+ 356: {lang: 0x423, script: 0x57, flags: 0x0},
+}
+
+// likelyRegionList holds lists info associated with likelyRegion.
+// Size: 372 bytes, 93 elements
+var likelyRegionList = [93]likelyLangScript{
+ 0: {lang: 0x148, script: 0x5, flags: 0x0},
+ 1: {lang: 0x476, script: 0x57, flags: 0x0},
+ 2: {lang: 0x431, script: 0x57, flags: 0x0},
+ 3: {lang: 0x2ff, script: 0x1f, flags: 0x0},
+ 4: {lang: 0x1d7, script: 0x8, flags: 0x0},
+ 5: {lang: 0x274, script: 0x57, flags: 0x0},
+ 6: {lang: 0xb7, script: 0x57, flags: 0x0},
+ 7: {lang: 0x432, script: 0x1f, flags: 0x0},
+ 8: {lang: 0x12d, script: 0xe0, flags: 0x0},
+ 9: {lang: 0x351, script: 0x21, flags: 0x0},
+ 10: {lang: 0x529, script: 0x38, flags: 0x0},
+ 11: {lang: 0x4ac, script: 0x5, flags: 0x0},
+ 12: {lang: 0x523, script: 0x57, flags: 0x0},
+ 13: {lang: 0x29a, script: 0xdf, flags: 0x0},
+ 14: {lang: 0x136, script: 0x31, flags: 0x0},
+ 15: {lang: 0x48a, script: 0x57, flags: 0x0},
+ 16: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 17: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 18: {lang: 0x27, script: 0x29, flags: 0x0},
+ 19: {lang: 0x139, script: 0x57, flags: 0x0},
+ 20: {lang: 0x26a, script: 0x5, flags: 0x2},
+ 21: {lang: 0x512, script: 0x3b, flags: 0x2},
+ 22: {lang: 0x210, script: 0x2b, flags: 0x0},
+ 23: {lang: 0x5, script: 0x1f, flags: 0x0},
+ 24: {lang: 0x274, script: 0x57, flags: 0x0},
+ 25: {lang: 0x136, script: 0x31, flags: 0x0},
+ 26: {lang: 0x2ff, script: 0x1f, flags: 0x0},
+ 27: {lang: 0x1e1, script: 0x57, flags: 0x0},
+ 28: {lang: 0x31f, script: 0x5, flags: 0x0},
+ 29: {lang: 0x1be, script: 0x21, flags: 0x0},
+ 30: {lang: 0x4b4, script: 0x5, flags: 0x0},
+ 31: {lang: 0x236, script: 0x72, flags: 0x0},
+ 32: {lang: 0x148, script: 0x5, flags: 0x0},
+ 33: {lang: 0x476, script: 0x57, flags: 0x0},
+ 34: {lang: 0x24a, script: 0x4b, flags: 0x0},
+ 35: {lang: 0xe6, script: 0x5, flags: 0x0},
+ 36: {lang: 0x226, script: 0xdf, flags: 0x0},
+ 37: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 38: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 39: {lang: 0x2b8, script: 0x54, flags: 0x0},
+ 40: {lang: 0x226, script: 0xdf, flags: 0x0},
+ 41: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 42: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 43: {lang: 0x3dc, script: 0x57, flags: 0x0},
+ 44: {lang: 0x4ae, script: 0x1f, flags: 0x0},
+ 45: {lang: 0x2ff, script: 0x1f, flags: 0x0},
+ 46: {lang: 0x431, script: 0x57, flags: 0x0},
+ 47: {lang: 0x331, script: 0x72, flags: 0x0},
+ 48: {lang: 0x213, script: 0x57, flags: 0x0},
+ 49: {lang: 0x30b, script: 0x1f, flags: 0x0},
+ 50: {lang: 0x242, script: 0x5, flags: 0x0},
+ 51: {lang: 0x529, script: 0x39, flags: 0x0},
+ 52: {lang: 0x3c0, script: 0x57, flags: 0x0},
+ 53: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 54: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 55: {lang: 0x2ed, script: 0x57, flags: 0x0},
+ 56: {lang: 0x4b4, script: 0x5, flags: 0x0},
+ 57: {lang: 0x88, script: 0x21, flags: 0x0},
+ 58: {lang: 0x4b4, script: 0x5, flags: 0x0},
+ 59: {lang: 0x4b4, script: 0x5, flags: 0x0},
+ 60: {lang: 0xbe, script: 0x21, flags: 0x0},
+ 61: {lang: 0x3dc, script: 0x57, flags: 0x0},
+ 62: {lang: 0x7e, script: 0x1f, flags: 0x0},
+ 63: {lang: 0x3e2, script: 0x1f, flags: 0x0},
+ 64: {lang: 0x267, script: 0x57, flags: 0x0},
+ 65: {lang: 0x444, script: 0x57, flags: 0x0},
+ 66: {lang: 0x512, script: 0x3b, flags: 0x0},
+ 67: {lang: 0x412, script: 0x57, flags: 0x0},
+ 68: {lang: 0x4ae, script: 0x1f, flags: 0x0},
+ 69: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 70: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 71: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 72: {lang: 0x35, script: 0x5, flags: 0x0},
+ 73: {lang: 0x46b, script: 0xdf, flags: 0x0},
+ 74: {lang: 0x2ec, script: 0x5, flags: 0x0},
+ 75: {lang: 0x30f, script: 0x72, flags: 0x0},
+ 76: {lang: 0x467, script: 0x1f, flags: 0x0},
+ 77: {lang: 0x148, script: 0x5, flags: 0x0},
+ 78: {lang: 0x3a, script: 0x5, flags: 0x0},
+ 79: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 80: {lang: 0x48a, script: 0x57, flags: 0x0},
+ 81: {lang: 0x58, script: 0x5, flags: 0x0},
+ 82: {lang: 0x219, script: 0x1f, flags: 0x0},
+ 83: {lang: 0x81, script: 0x31, flags: 0x0},
+ 84: {lang: 0x529, script: 0x39, flags: 0x0},
+ 85: {lang: 0x48c, script: 0x57, flags: 0x0},
+ 86: {lang: 0x4ae, script: 0x1f, flags: 0x0},
+ 87: {lang: 0x512, script: 0x3b, flags: 0x0},
+ 88: {lang: 0x3b3, script: 0x57, flags: 0x0},
+ 89: {lang: 0x431, script: 0x57, flags: 0x0},
+ 90: {lang: 0x432, script: 0x1f, flags: 0x0},
+ 91: {lang: 0x15e, script: 0x57, flags: 0x0},
+ 92: {lang: 0x446, script: 0x5, flags: 0x0},
+}
+
+type likelyTag struct {
+ lang uint16
+ region uint16
+ script uint8
+}
+
+// Size: 198 bytes, 33 elements
+var likelyRegionGroup = [33]likelyTag{
+ 1: {lang: 0x139, region: 0xd6, script: 0x57},
+ 2: {lang: 0x139, region: 0x135, script: 0x57},
+ 3: {lang: 0x3c0, region: 0x41, script: 0x57},
+ 4: {lang: 0x139, region: 0x2f, script: 0x57},
+ 5: {lang: 0x139, region: 0xd6, script: 0x57},
+ 6: {lang: 0x13e, region: 0xcf, script: 0x57},
+ 7: {lang: 0x445, region: 0x12f, script: 0x57},
+ 8: {lang: 0x3a, region: 0x6b, script: 0x5},
+ 9: {lang: 0x445, region: 0x4b, script: 0x57},
+ 10: {lang: 0x139, region: 0x161, script: 0x57},
+ 11: {lang: 0x139, region: 0x135, script: 0x57},
+ 12: {lang: 0x139, region: 0x135, script: 0x57},
+ 13: {lang: 0x13e, region: 0x59, script: 0x57},
+ 14: {lang: 0x529, region: 0x53, script: 0x38},
+ 15: {lang: 0x1be, region: 0x99, script: 0x21},
+ 16: {lang: 0x1e1, region: 0x95, script: 0x57},
+ 17: {lang: 0x1f9, region: 0x9e, script: 0x57},
+ 18: {lang: 0x139, region: 0x2f, script: 0x57},
+ 19: {lang: 0x139, region: 0xe6, script: 0x57},
+ 20: {lang: 0x139, region: 0x8a, script: 0x57},
+ 21: {lang: 0x41b, region: 0x142, script: 0x57},
+ 22: {lang: 0x529, region: 0x53, script: 0x38},
+ 23: {lang: 0x4bc, region: 0x137, script: 0x57},
+ 24: {lang: 0x3a, region: 0x108, script: 0x5},
+ 25: {lang: 0x3e2, region: 0x106, script: 0x1f},
+ 26: {lang: 0x3e2, region: 0x106, script: 0x1f},
+ 27: {lang: 0x139, region: 0x7b, script: 0x57},
+ 28: {lang: 0x10d, region: 0x60, script: 0x57},
+ 29: {lang: 0x139, region: 0xd6, script: 0x57},
+ 30: {lang: 0x13e, region: 0x1f, script: 0x57},
+ 31: {lang: 0x139, region: 0x9a, script: 0x57},
+ 32: {lang: 0x139, region: 0x7b, script: 0x57},
+}
+
+// Size: 358 bytes, 358 elements
+var regionToGroups = [358]uint8{
+ // Entry 0 - 3F
+ 0x00, 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x04,
+ 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x04, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x00,
+ 0x00, 0x04, 0x00, 0x00, 0x04, 0x01, 0x00, 0x00,
+ 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x00, 0x04,
+ // Entry 40 - 7F
+ 0x04, 0x04, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x04, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x04, 0x00, 0x00, 0x04, 0x00, 0x04, 0x00,
+ 0x00, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x04, 0x04, 0x00, 0x08,
+ 0x00, 0x04, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x04, 0x00,
+ // Entry 80 - BF
+ 0x00, 0x00, 0x04, 0x00, 0x00, 0x04, 0x00, 0x00,
+ 0x00, 0x04, 0x01, 0x00, 0x04, 0x02, 0x00, 0x04,
+ 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x04, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00,
+ 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x08, 0x08, 0x00, 0x00, 0x00, 0x04, 0x00,
+ // Entry C0 - FF
+ 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x01,
+ 0x04, 0x08, 0x04, 0x00, 0x00, 0x00, 0x00, 0x04,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x04, 0x00, 0x04, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x04, 0x00, 0x05, 0x00, 0x00, 0x00,
+ 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ // Entry 100 - 13F
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00,
+ 0x00, 0x00, 0x04, 0x04, 0x00, 0x00, 0x00, 0x04,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x08, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x01, 0x00, 0x05, 0x04, 0x00,
+ 0x00, 0x04, 0x00, 0x04, 0x04, 0x05, 0x00, 0x00,
+ // Entry 140 - 17F
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+}
+
+// Size: 18 bytes, 3 elements
+var paradigmLocales = [3][3]uint16{
+ 0: [3]uint16{0x139, 0x0, 0x7b},
+ 1: [3]uint16{0x13e, 0x0, 0x1f},
+ 2: [3]uint16{0x3c0, 0x41, 0xee},
+}
+
+type mutualIntelligibility struct {
+ want uint16
+ have uint16
+ distance uint8
+ oneway bool
+}
+
+type scriptIntelligibility struct {
+ wantLang uint16
+ haveLang uint16
+ wantScript uint8
+ haveScript uint8
+ distance uint8
+}
+
+type regionIntelligibility struct {
+ lang uint16
+ script uint8
+ group uint8
+ distance uint8
+}
+
+// matchLang holds pairs of langIDs of base languages that are typically
+// mutually intelligible. Each pair is associated with a confidence and
+// whether the intelligibility goes one or both ways.
+// Size: 678 bytes, 113 elements
+var matchLang = [113]mutualIntelligibility{
+ 0: {want: 0x1d1, have: 0xb7, distance: 0x4, oneway: false},
+ 1: {want: 0x407, have: 0xb7, distance: 0x4, oneway: false},
+ 2: {want: 0x407, have: 0x1d1, distance: 0x4, oneway: false},
+ 3: {want: 0x407, have: 0x432, distance: 0x4, oneway: false},
+ 4: {want: 0x43a, have: 0x1, distance: 0x4, oneway: false},
+ 5: {want: 0x1a3, have: 0x10d, distance: 0x4, oneway: true},
+ 6: {want: 0x295, have: 0x10d, distance: 0x4, oneway: true},
+ 7: {want: 0x101, have: 0x36f, distance: 0x8, oneway: false},
+ 8: {want: 0x101, have: 0x347, distance: 0x8, oneway: false},
+ 9: {want: 0x5, have: 0x3e2, distance: 0xa, oneway: true},
+ 10: {want: 0xd, have: 0x139, distance: 0xa, oneway: true},
+ 11: {want: 0x16, have: 0x367, distance: 0xa, oneway: true},
+ 12: {want: 0x21, have: 0x139, distance: 0xa, oneway: true},
+ 13: {want: 0x56, have: 0x13e, distance: 0xa, oneway: true},
+ 14: {want: 0x58, have: 0x3e2, distance: 0xa, oneway: true},
+ 15: {want: 0x71, have: 0x3e2, distance: 0xa, oneway: true},
+ 16: {want: 0x75, have: 0x139, distance: 0xa, oneway: true},
+ 17: {want: 0x82, have: 0x1be, distance: 0xa, oneway: true},
+ 18: {want: 0xa5, have: 0x139, distance: 0xa, oneway: true},
+ 19: {want: 0xb2, have: 0x15e, distance: 0xa, oneway: true},
+ 20: {want: 0xdd, have: 0x153, distance: 0xa, oneway: true},
+ 21: {want: 0xe5, have: 0x139, distance: 0xa, oneway: true},
+ 22: {want: 0xe9, have: 0x3a, distance: 0xa, oneway: true},
+ 23: {want: 0xf0, have: 0x15e, distance: 0xa, oneway: true},
+ 24: {want: 0xf9, have: 0x15e, distance: 0xa, oneway: true},
+ 25: {want: 0x100, have: 0x139, distance: 0xa, oneway: true},
+ 26: {want: 0x130, have: 0x139, distance: 0xa, oneway: true},
+ 27: {want: 0x13c, have: 0x139, distance: 0xa, oneway: true},
+ 28: {want: 0x140, have: 0x151, distance: 0xa, oneway: true},
+ 29: {want: 0x145, have: 0x13e, distance: 0xa, oneway: true},
+ 30: {want: 0x158, have: 0x101, distance: 0xa, oneway: true},
+ 31: {want: 0x16d, have: 0x367, distance: 0xa, oneway: true},
+ 32: {want: 0x16e, have: 0x139, distance: 0xa, oneway: true},
+ 33: {want: 0x16f, have: 0x139, distance: 0xa, oneway: true},
+ 34: {want: 0x17e, have: 0x139, distance: 0xa, oneway: true},
+ 35: {want: 0x190, have: 0x13e, distance: 0xa, oneway: true},
+ 36: {want: 0x194, have: 0x13e, distance: 0xa, oneway: true},
+ 37: {want: 0x1a4, have: 0x1be, distance: 0xa, oneway: true},
+ 38: {want: 0x1b4, have: 0x139, distance: 0xa, oneway: true},
+ 39: {want: 0x1b8, have: 0x139, distance: 0xa, oneway: true},
+ 40: {want: 0x1d4, have: 0x15e, distance: 0xa, oneway: true},
+ 41: {want: 0x1d7, have: 0x3e2, distance: 0xa, oneway: true},
+ 42: {want: 0x1d9, have: 0x139, distance: 0xa, oneway: true},
+ 43: {want: 0x1e7, have: 0x139, distance: 0xa, oneway: true},
+ 44: {want: 0x1f8, have: 0x139, distance: 0xa, oneway: true},
+ 45: {want: 0x20e, have: 0x1e1, distance: 0xa, oneway: true},
+ 46: {want: 0x210, have: 0x139, distance: 0xa, oneway: true},
+ 47: {want: 0x22d, have: 0x15e, distance: 0xa, oneway: true},
+ 48: {want: 0x242, have: 0x3e2, distance: 0xa, oneway: true},
+ 49: {want: 0x24a, have: 0x139, distance: 0xa, oneway: true},
+ 50: {want: 0x251, have: 0x139, distance: 0xa, oneway: true},
+ 51: {want: 0x265, have: 0x139, distance: 0xa, oneway: true},
+ 52: {want: 0x274, have: 0x48a, distance: 0xa, oneway: true},
+ 53: {want: 0x28a, have: 0x3e2, distance: 0xa, oneway: true},
+ 54: {want: 0x28e, have: 0x1f9, distance: 0xa, oneway: true},
+ 55: {want: 0x2a3, have: 0x139, distance: 0xa, oneway: true},
+ 56: {want: 0x2b5, have: 0x15e, distance: 0xa, oneway: true},
+ 57: {want: 0x2b8, have: 0x139, distance: 0xa, oneway: true},
+ 58: {want: 0x2be, have: 0x139, distance: 0xa, oneway: true},
+ 59: {want: 0x2c3, have: 0x15e, distance: 0xa, oneway: true},
+ 60: {want: 0x2ed, have: 0x139, distance: 0xa, oneway: true},
+ 61: {want: 0x2f1, have: 0x15e, distance: 0xa, oneway: true},
+ 62: {want: 0x2fa, have: 0x139, distance: 0xa, oneway: true},
+ 63: {want: 0x2ff, have: 0x7e, distance: 0xa, oneway: true},
+ 64: {want: 0x304, have: 0x139, distance: 0xa, oneway: true},
+ 65: {want: 0x30b, have: 0x3e2, distance: 0xa, oneway: true},
+ 66: {want: 0x31b, have: 0x1be, distance: 0xa, oneway: true},
+ 67: {want: 0x31f, have: 0x1e1, distance: 0xa, oneway: true},
+ 68: {want: 0x320, have: 0x139, distance: 0xa, oneway: true},
+ 69: {want: 0x331, have: 0x139, distance: 0xa, oneway: true},
+ 70: {want: 0x351, have: 0x139, distance: 0xa, oneway: true},
+ 71: {want: 0x36a, have: 0x347, distance: 0xa, oneway: false},
+ 72: {want: 0x36a, have: 0x36f, distance: 0xa, oneway: true},
+ 73: {want: 0x37a, have: 0x139, distance: 0xa, oneway: true},
+ 74: {want: 0x387, have: 0x139, distance: 0xa, oneway: true},
+ 75: {want: 0x389, have: 0x139, distance: 0xa, oneway: true},
+ 76: {want: 0x38b, have: 0x15e, distance: 0xa, oneway: true},
+ 77: {want: 0x390, have: 0x139, distance: 0xa, oneway: true},
+ 78: {want: 0x395, have: 0x139, distance: 0xa, oneway: true},
+ 79: {want: 0x39d, have: 0x139, distance: 0xa, oneway: true},
+ 80: {want: 0x3a5, have: 0x139, distance: 0xa, oneway: true},
+ 81: {want: 0x3be, have: 0x139, distance: 0xa, oneway: true},
+ 82: {want: 0x3c4, have: 0x13e, distance: 0xa, oneway: true},
+ 83: {want: 0x3d4, have: 0x10d, distance: 0xa, oneway: true},
+ 84: {want: 0x3d9, have: 0x139, distance: 0xa, oneway: true},
+ 85: {want: 0x3e5, have: 0x15e, distance: 0xa, oneway: true},
+ 86: {want: 0x3e9, have: 0x1be, distance: 0xa, oneway: true},
+ 87: {want: 0x3fa, have: 0x139, distance: 0xa, oneway: true},
+ 88: {want: 0x40c, have: 0x139, distance: 0xa, oneway: true},
+ 89: {want: 0x423, have: 0x139, distance: 0xa, oneway: true},
+ 90: {want: 0x429, have: 0x139, distance: 0xa, oneway: true},
+ 91: {want: 0x431, have: 0x139, distance: 0xa, oneway: true},
+ 92: {want: 0x43b, have: 0x139, distance: 0xa, oneway: true},
+ 93: {want: 0x43e, have: 0x1e1, distance: 0xa, oneway: true},
+ 94: {want: 0x445, have: 0x139, distance: 0xa, oneway: true},
+ 95: {want: 0x450, have: 0x139, distance: 0xa, oneway: true},
+ 96: {want: 0x461, have: 0x139, distance: 0xa, oneway: true},
+ 97: {want: 0x467, have: 0x3e2, distance: 0xa, oneway: true},
+ 98: {want: 0x46f, have: 0x139, distance: 0xa, oneway: true},
+ 99: {want: 0x476, have: 0x3e2, distance: 0xa, oneway: true},
+ 100: {want: 0x3883, have: 0x139, distance: 0xa, oneway: true},
+ 101: {want: 0x480, have: 0x139, distance: 0xa, oneway: true},
+ 102: {want: 0x482, have: 0x139, distance: 0xa, oneway: true},
+ 103: {want: 0x494, have: 0x3e2, distance: 0xa, oneway: true},
+ 104: {want: 0x49d, have: 0x139, distance: 0xa, oneway: true},
+ 105: {want: 0x4ac, have: 0x529, distance: 0xa, oneway: true},
+ 106: {want: 0x4b4, have: 0x139, distance: 0xa, oneway: true},
+ 107: {want: 0x4bc, have: 0x3e2, distance: 0xa, oneway: true},
+ 108: {want: 0x4e5, have: 0x15e, distance: 0xa, oneway: true},
+ 109: {want: 0x4f2, have: 0x139, distance: 0xa, oneway: true},
+ 110: {want: 0x512, have: 0x139, distance: 0xa, oneway: true},
+ 111: {want: 0x518, have: 0x139, distance: 0xa, oneway: true},
+ 112: {want: 0x52f, have: 0x139, distance: 0xa, oneway: true},
+}
+
+// matchScript holds pairs of scriptIDs where readers of one script
+// can typically also read the other. Each is associated with a confidence.
+// Size: 208 bytes, 26 elements
+var matchScript = [26]scriptIntelligibility{
+ 0: {wantLang: 0x432, haveLang: 0x432, wantScript: 0x57, haveScript: 0x1f, distance: 0x5},
+ 1: {wantLang: 0x432, haveLang: 0x432, wantScript: 0x1f, haveScript: 0x57, distance: 0x5},
+ 2: {wantLang: 0x58, haveLang: 0x3e2, wantScript: 0x57, haveScript: 0x1f, distance: 0xa},
+ 3: {wantLang: 0xa5, haveLang: 0x139, wantScript: 0xe, haveScript: 0x57, distance: 0xa},
+ 4: {wantLang: 0x1d7, haveLang: 0x3e2, wantScript: 0x8, haveScript: 0x1f, distance: 0xa},
+ 5: {wantLang: 0x210, haveLang: 0x139, wantScript: 0x2b, haveScript: 0x57, distance: 0xa},
+ 6: {wantLang: 0x24a, haveLang: 0x139, wantScript: 0x4b, haveScript: 0x57, distance: 0xa},
+ 7: {wantLang: 0x251, haveLang: 0x139, wantScript: 0x4f, haveScript: 0x57, distance: 0xa},
+ 8: {wantLang: 0x2b8, haveLang: 0x139, wantScript: 0x54, haveScript: 0x57, distance: 0xa},
+ 9: {wantLang: 0x304, haveLang: 0x139, wantScript: 0x6b, haveScript: 0x57, distance: 0xa},
+ 10: {wantLang: 0x331, haveLang: 0x139, wantScript: 0x72, haveScript: 0x57, distance: 0xa},
+ 11: {wantLang: 0x351, haveLang: 0x139, wantScript: 0x21, haveScript: 0x57, distance: 0xa},
+ 12: {wantLang: 0x395, haveLang: 0x139, wantScript: 0x7d, haveScript: 0x57, distance: 0xa},
+ 13: {wantLang: 0x39d, haveLang: 0x139, wantScript: 0x33, haveScript: 0x57, distance: 0xa},
+ 14: {wantLang: 0x3be, haveLang: 0x139, wantScript: 0x5, haveScript: 0x57, distance: 0xa},
+ 15: {wantLang: 0x3fa, haveLang: 0x139, wantScript: 0x5, haveScript: 0x57, distance: 0xa},
+ 16: {wantLang: 0x40c, haveLang: 0x139, wantScript: 0xca, haveScript: 0x57, distance: 0xa},
+ 17: {wantLang: 0x450, haveLang: 0x139, wantScript: 0xd7, haveScript: 0x57, distance: 0xa},
+ 18: {wantLang: 0x461, haveLang: 0x139, wantScript: 0xda, haveScript: 0x57, distance: 0xa},
+ 19: {wantLang: 0x46f, haveLang: 0x139, wantScript: 0x29, haveScript: 0x57, distance: 0xa},
+ 20: {wantLang: 0x476, haveLang: 0x3e2, wantScript: 0x57, haveScript: 0x1f, distance: 0xa},
+ 21: {wantLang: 0x4b4, haveLang: 0x139, wantScript: 0x5, haveScript: 0x57, distance: 0xa},
+ 22: {wantLang: 0x4bc, haveLang: 0x3e2, wantScript: 0x57, haveScript: 0x1f, distance: 0xa},
+ 23: {wantLang: 0x512, haveLang: 0x139, wantScript: 0x3b, haveScript: 0x57, distance: 0xa},
+ 24: {wantLang: 0x529, haveLang: 0x529, wantScript: 0x38, haveScript: 0x39, distance: 0xf},
+ 25: {wantLang: 0x529, haveLang: 0x529, wantScript: 0x39, haveScript: 0x38, distance: 0x13},
+}
+
+// Size: 90 bytes, 15 elements
+var matchRegion = [15]regionIntelligibility{
+ 0: {lang: 0x3a, script: 0x0, group: 0x4, distance: 0x4},
+ 1: {lang: 0x3a, script: 0x0, group: 0x84, distance: 0x4},
+ 2: {lang: 0x139, script: 0x0, group: 0x1, distance: 0x4},
+ 3: {lang: 0x139, script: 0x0, group: 0x81, distance: 0x4},
+ 4: {lang: 0x13e, script: 0x0, group: 0x3, distance: 0x4},
+ 5: {lang: 0x13e, script: 0x0, group: 0x83, distance: 0x4},
+ 6: {lang: 0x3c0, script: 0x0, group: 0x3, distance: 0x4},
+ 7: {lang: 0x3c0, script: 0x0, group: 0x83, distance: 0x4},
+ 8: {lang: 0x529, script: 0x39, group: 0x2, distance: 0x4},
+ 9: {lang: 0x529, script: 0x39, group: 0x82, distance: 0x4},
+ 10: {lang: 0x3a, script: 0x0, group: 0x80, distance: 0x5},
+ 11: {lang: 0x139, script: 0x0, group: 0x80, distance: 0x5},
+ 12: {lang: 0x13e, script: 0x0, group: 0x80, distance: 0x5},
+ 13: {lang: 0x3c0, script: 0x0, group: 0x80, distance: 0x5},
+ 14: {lang: 0x529, script: 0x39, group: 0x80, distance: 0x5},
+}
+
+// Size: 264 bytes, 33 elements
+var regionContainment = [33]uint64{
+ // Entry 0 - 1F
+ 0x00000001ffffffff, 0x00000000200007a2, 0x0000000000003044, 0x0000000000000008,
+ 0x00000000803c0010, 0x0000000000000020, 0x0000000000000040, 0x0000000000000080,
+ 0x0000000000000100, 0x0000000000000200, 0x0000000000000400, 0x000000004000384c,
+ 0x0000000000001000, 0x0000000000002000, 0x0000000000004000, 0x0000000000008000,
+ 0x0000000000010000, 0x0000000000020000, 0x0000000000040000, 0x0000000000080000,
+ 0x0000000000100000, 0x0000000000200000, 0x0000000001c1c000, 0x0000000000800000,
+ 0x0000000001000000, 0x000000001e020000, 0x0000000004000000, 0x0000000008000000,
+ 0x0000000010000000, 0x00000000200006a0, 0x0000000040002048, 0x0000000080000000,
+ // Entry 20 - 3F
+ 0x0000000100000000,
+}
+
+// regionInclusion maps region identifiers to sets of regions in regionInclusionBits,
+// where each set holds all groupings that are directly connected in a region
+// containment graph.
+// Size: 358 bytes, 358 elements
+var regionInclusion = [358]uint8{
+ // Entry 0 - 3F
+ 0x00, 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06,
+ 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e,
+ 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16,
+ 0x17, 0x18, 0x19, 0x1a, 0x1b, 0x1c, 0x1d, 0x1e,
+ 0x21, 0x22, 0x23, 0x24, 0x25, 0x26, 0x26, 0x23,
+ 0x24, 0x26, 0x27, 0x22, 0x28, 0x29, 0x2a, 0x2b,
+ 0x26, 0x2c, 0x24, 0x23, 0x26, 0x25, 0x2a, 0x2d,
+ 0x2e, 0x24, 0x2f, 0x2d, 0x26, 0x30, 0x31, 0x28,
+ // Entry 40 - 7F
+ 0x26, 0x28, 0x26, 0x25, 0x31, 0x22, 0x32, 0x33,
+ 0x34, 0x30, 0x22, 0x27, 0x27, 0x27, 0x35, 0x2d,
+ 0x29, 0x28, 0x27, 0x36, 0x28, 0x22, 0x34, 0x23,
+ 0x21, 0x26, 0x2d, 0x26, 0x22, 0x37, 0x2e, 0x35,
+ 0x2a, 0x22, 0x2f, 0x38, 0x26, 0x26, 0x21, 0x39,
+ 0x39, 0x28, 0x38, 0x39, 0x39, 0x2f, 0x3a, 0x2f,
+ 0x20, 0x21, 0x38, 0x3b, 0x28, 0x3c, 0x2c, 0x21,
+ 0x2a, 0x35, 0x27, 0x38, 0x26, 0x24, 0x28, 0x2c,
+ // Entry 80 - BF
+ 0x2d, 0x23, 0x30, 0x2d, 0x2d, 0x26, 0x27, 0x3a,
+ 0x22, 0x34, 0x3c, 0x2d, 0x28, 0x36, 0x22, 0x34,
+ 0x3a, 0x26, 0x2e, 0x21, 0x39, 0x31, 0x38, 0x24,
+ 0x2c, 0x25, 0x22, 0x24, 0x25, 0x2c, 0x3a, 0x2c,
+ 0x26, 0x24, 0x36, 0x21, 0x2f, 0x3d, 0x31, 0x3c,
+ 0x2f, 0x26, 0x36, 0x36, 0x24, 0x26, 0x3d, 0x31,
+ 0x24, 0x26, 0x35, 0x25, 0x2d, 0x32, 0x38, 0x2a,
+ 0x38, 0x39, 0x39, 0x35, 0x33, 0x23, 0x26, 0x2f,
+ // Entry C0 - FF
+ 0x3c, 0x21, 0x23, 0x2d, 0x31, 0x36, 0x36, 0x3c,
+ 0x26, 0x2d, 0x26, 0x3a, 0x2f, 0x25, 0x2f, 0x34,
+ 0x31, 0x2f, 0x32, 0x3b, 0x2d, 0x2b, 0x2d, 0x21,
+ 0x34, 0x2a, 0x2c, 0x25, 0x21, 0x3c, 0x24, 0x29,
+ 0x2b, 0x24, 0x34, 0x21, 0x28, 0x29, 0x3b, 0x31,
+ 0x25, 0x2e, 0x30, 0x29, 0x26, 0x24, 0x3a, 0x21,
+ 0x3c, 0x28, 0x21, 0x24, 0x21, 0x21, 0x1f, 0x21,
+ 0x21, 0x21, 0x21, 0x21, 0x21, 0x21, 0x21, 0x21,
+ // Entry 100 - 13F
+ 0x21, 0x21, 0x2f, 0x21, 0x2e, 0x23, 0x33, 0x2f,
+ 0x24, 0x3b, 0x2f, 0x39, 0x38, 0x31, 0x2d, 0x3a,
+ 0x2c, 0x2e, 0x2d, 0x23, 0x2d, 0x2f, 0x28, 0x2f,
+ 0x27, 0x33, 0x34, 0x26, 0x24, 0x32, 0x22, 0x26,
+ 0x27, 0x22, 0x2d, 0x31, 0x3d, 0x29, 0x31, 0x3d,
+ 0x39, 0x29, 0x31, 0x24, 0x26, 0x29, 0x36, 0x2f,
+ 0x33, 0x2f, 0x21, 0x22, 0x21, 0x30, 0x28, 0x3d,
+ 0x23, 0x26, 0x21, 0x28, 0x26, 0x26, 0x31, 0x3b,
+ // Entry 140 - 17F
+ 0x29, 0x21, 0x29, 0x21, 0x21, 0x21, 0x21, 0x21,
+ 0x21, 0x21, 0x21, 0x21, 0x21, 0x23, 0x21, 0x21,
+ 0x21, 0x21, 0x21, 0x21, 0x21, 0x21, 0x21, 0x21,
+ 0x21, 0x21, 0x21, 0x21, 0x21, 0x24, 0x24, 0x2f,
+ 0x23, 0x32, 0x2f, 0x27, 0x2f, 0x21,
+}
+
+// regionInclusionBits is an array of bit vectors where every vector represents
+// a set of region groupings. These sets are used to compute the distance
+// between two regions for the purpose of language matching.
+// Size: 584 bytes, 73 elements
+var regionInclusionBits = [73]uint64{
+ // Entry 0 - 1F
+ 0x0000000102400813, 0x00000000200007a3, 0x0000000000003844, 0x0000000040000808,
+ 0x00000000803c0011, 0x0000000020000022, 0x0000000040000844, 0x0000000020000082,
+ 0x0000000000000102, 0x0000000020000202, 0x0000000020000402, 0x000000004000384d,
+ 0x0000000000001804, 0x0000000040002804, 0x0000000000404000, 0x0000000000408000,
+ 0x0000000000410000, 0x0000000002020000, 0x0000000000040010, 0x0000000000080010,
+ 0x0000000000100010, 0x0000000000200010, 0x0000000001c1c001, 0x0000000000c00000,
+ 0x0000000001400000, 0x000000001e020001, 0x0000000006000000, 0x000000000a000000,
+ 0x0000000012000000, 0x00000000200006a2, 0x0000000040002848, 0x0000000080000010,
+ // Entry 20 - 3F
+ 0x0000000100000001, 0x0000000000000001, 0x0000000080000000, 0x0000000000020000,
+ 0x0000000001000000, 0x0000000000008000, 0x0000000000002000, 0x0000000000000200,
+ 0x0000000000000008, 0x0000000000200000, 0x0000000110000000, 0x0000000000040000,
+ 0x0000000008000000, 0x0000000000000020, 0x0000000104000000, 0x0000000000000080,
+ 0x0000000000001000, 0x0000000000010000, 0x0000000000000400, 0x0000000004000000,
+ 0x0000000000000040, 0x0000000010000000, 0x0000000000004000, 0x0000000101000000,
+ 0x0000000108000000, 0x0000000000000100, 0x0000000100020000, 0x0000000000080000,
+ 0x0000000000100000, 0x0000000000800000, 0x00000001ffffffff, 0x0000000122400fb3,
+ // Entry 40 - 5F
+ 0x00000001827c0813, 0x000000014240385f, 0x0000000103c1c813, 0x000000011e420813,
+ 0x0000000112000001, 0x0000000106000001, 0x0000000101400001, 0x000000010a000001,
+ 0x0000000102020001,
+}
+
+// regionInclusionNext marks, for each entry in regionInclusionBits, the set of
+// all groups that are reachable from the groups set in the respective entry.
+// Size: 73 bytes, 73 elements
+var regionInclusionNext = [73]uint8{
+ // Entry 0 - 3F
+ 0x3e, 0x3f, 0x0b, 0x0b, 0x40, 0x01, 0x0b, 0x01,
+ 0x01, 0x01, 0x01, 0x41, 0x0b, 0x0b, 0x16, 0x16,
+ 0x16, 0x19, 0x04, 0x04, 0x04, 0x04, 0x42, 0x16,
+ 0x16, 0x43, 0x19, 0x19, 0x19, 0x01, 0x0b, 0x04,
+ 0x00, 0x00, 0x1f, 0x11, 0x18, 0x0f, 0x0d, 0x09,
+ 0x03, 0x15, 0x44, 0x12, 0x1b, 0x05, 0x45, 0x07,
+ 0x0c, 0x10, 0x0a, 0x1a, 0x06, 0x1c, 0x0e, 0x46,
+ 0x47, 0x08, 0x48, 0x13, 0x14, 0x17, 0x3e, 0x3e,
+ // Entry 40 - 7F
+ 0x3e, 0x3e, 0x3e, 0x3e, 0x43, 0x43, 0x42, 0x43,
+ 0x43,
+}
+
+type parentRel struct {
+ lang uint16
+ script uint8
+ maxScript uint8
+ toRegion uint16
+ fromRegion []uint16
+}
+
+// Size: 414 bytes, 5 elements
+var parents = [5]parentRel{
+ 0: {lang: 0x139, script: 0x0, maxScript: 0x57, toRegion: 0x1, fromRegion: []uint16{0x1a, 0x25, 0x26, 0x2f, 0x34, 0x36, 0x3d, 0x42, 0x46, 0x48, 0x49, 0x4a, 0x50, 0x52, 0x5c, 0x5d, 0x61, 0x64, 0x6d, 0x73, 0x74, 0x75, 0x7b, 0x7c, 0x7f, 0x80, 0x81, 0x83, 0x8c, 0x8d, 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9f, 0xa0, 0xa4, 0xa7, 0xa9, 0xad, 0xb1, 0xb4, 0xb5, 0xbf, 0xc6, 0xca, 0xcb, 0xcc, 0xce, 0xd0, 0xd2, 0xd5, 0xd6, 0xdd, 0xdf, 0xe0, 0xe6, 0xe7, 0xe8, 0xeb, 0xf0, 0x107, 0x109, 0x10a, 0x10b, 0x10d, 0x10e, 0x112, 0x117, 0x11b, 0x11d, 0x11f, 0x125, 0x129, 0x12c, 0x12d, 0x12f, 0x131, 0x139, 0x13c, 0x13f, 0x142, 0x161, 0x162, 0x164}},
+ 1: {lang: 0x139, script: 0x0, maxScript: 0x57, toRegion: 0x1a, fromRegion: []uint16{0x2e, 0x4e, 0x60, 0x63, 0x72, 0xd9, 0x10c, 0x10f}},
+ 2: {lang: 0x13e, script: 0x0, maxScript: 0x57, toRegion: 0x1f, fromRegion: []uint16{0x2c, 0x3f, 0x41, 0x48, 0x51, 0x54, 0x56, 0x59, 0x65, 0x69, 0x89, 0x8f, 0xcf, 0xd8, 0xe2, 0xe4, 0xec, 0xf1, 0x11a, 0x135, 0x136, 0x13b}},
+ 3: {lang: 0x3c0, script: 0x0, maxScript: 0x57, toRegion: 0xee, fromRegion: []uint16{0x2a, 0x4e, 0x5a, 0x86, 0x8b, 0xb7, 0xc6, 0xd1, 0x118, 0x126}},
+ 4: {lang: 0x529, script: 0x39, maxScript: 0x39, toRegion: 0x8d, fromRegion: []uint16{0xc6}},
+}
+
+// Total table size 27238 bytes (26KiB); checksum: C9BBE4D5
diff --git a/language/internal/tags.go b/language/internal/tags.go
new file mode 100644
index 0000000..de30155
--- /dev/null
+++ b/language/internal/tags.go
@@ -0,0 +1,143 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package language
+
+// TODO: Various sets of commonly use tags and regions.
+
+// MustParse is like Parse, but panics if the given BCP 47 tag cannot be parsed.
+// It simplifies safe initialization of Tag values.
+func MustParse(s string) Tag {
+ t, err := Parse(s)
+ if err != nil {
+ panic(err)
+ }
+ return t
+}
+
+// MustParse is like Parse, but panics if the given BCP 47 tag cannot be parsed.
+// It simplifies safe initialization of Tag values.
+func (c CanonType) MustParse(s string) Tag {
+ t, err := c.Parse(s)
+ if err != nil {
+ panic(err)
+ }
+ return t
+}
+
+// MustParseBase is like ParseBase, but panics if the given base cannot be parsed.
+// It simplifies safe initialization of Base values.
+func MustParseBase(s string) Base {
+ b, err := ParseBase(s)
+ if err != nil {
+ panic(err)
+ }
+ return b
+}
+
+// MustParseScript is like ParseScript, but panics if the given script cannot be
+// parsed. It simplifies safe initialization of Script values.
+func MustParseScript(s string) Script {
+ scr, err := ParseScript(s)
+ if err != nil {
+ panic(err)
+ }
+ return scr
+}
+
+// MustParseRegion is like ParseRegion, but panics if the given region cannot be
+// parsed. It simplifies safe initialization of Region values.
+func MustParseRegion(s string) Region {
+ r, err := ParseRegion(s)
+ if err != nil {
+ panic(err)
+ }
+ return r
+}
+
+var (
+ und = Tag{}
+
+ Und Tag = Tag{}
+
+ Afrikaans Tag = Tag{lang: _af} // af
+ Amharic Tag = Tag{lang: _am} // am
+ Arabic Tag = Tag{lang: _ar} // ar
+ ModernStandardArabic Tag = Tag{lang: _ar, region: _001} // ar-001
+ Azerbaijani Tag = Tag{lang: _az} // az
+ Bulgarian Tag = Tag{lang: _bg} // bg
+ Bengali Tag = Tag{lang: _bn} // bn
+ Catalan Tag = Tag{lang: _ca} // ca
+ Czech Tag = Tag{lang: _cs} // cs
+ Danish Tag = Tag{lang: _da} // da
+ German Tag = Tag{lang: _de} // de
+ Greek Tag = Tag{lang: _el} // el
+ English Tag = Tag{lang: _en} // en
+ AmericanEnglish Tag = Tag{lang: _en, region: _US} // en-US
+ BritishEnglish Tag = Tag{lang: _en, region: _GB} // en-GB
+ Spanish Tag = Tag{lang: _es} // es
+ EuropeanSpanish Tag = Tag{lang: _es, region: _ES} // es-ES
+ LatinAmericanSpanish Tag = Tag{lang: _es, region: _419} // es-419
+ Estonian Tag = Tag{lang: _et} // et
+ Persian Tag = Tag{lang: _fa} // fa
+ Finnish Tag = Tag{lang: _fi} // fi
+ Filipino Tag = Tag{lang: _fil} // fil
+ French Tag = Tag{lang: _fr} // fr
+ CanadianFrench Tag = Tag{lang: _fr, region: _CA} // fr-CA
+ Gujarati Tag = Tag{lang: _gu} // gu
+ Hebrew Tag = Tag{lang: _he} // he
+ Hindi Tag = Tag{lang: _hi} // hi
+ Croatian Tag = Tag{lang: _hr} // hr
+ Hungarian Tag = Tag{lang: _hu} // hu
+ Armenian Tag = Tag{lang: _hy} // hy
+ Indonesian Tag = Tag{lang: _id} // id
+ Icelandic Tag = Tag{lang: _is} // is
+ Italian Tag = Tag{lang: _it} // it
+ Japanese Tag = Tag{lang: _ja} // ja
+ Georgian Tag = Tag{lang: _ka} // ka
+ Kazakh Tag = Tag{lang: _kk} // kk
+ Khmer Tag = Tag{lang: _km} // km
+ Kannada Tag = Tag{lang: _kn} // kn
+ Korean Tag = Tag{lang: _ko} // ko
+ Kirghiz Tag = Tag{lang: _ky} // ky
+ Lao Tag = Tag{lang: _lo} // lo
+ Lithuanian Tag = Tag{lang: _lt} // lt
+ Latvian Tag = Tag{lang: _lv} // lv
+ Macedonian Tag = Tag{lang: _mk} // mk
+ Malayalam Tag = Tag{lang: _ml} // ml
+ Mongolian Tag = Tag{lang: _mn} // mn
+ Marathi Tag = Tag{lang: _mr} // mr
+ Malay Tag = Tag{lang: _ms} // ms
+ Burmese Tag = Tag{lang: _my} // my
+ Nepali Tag = Tag{lang: _ne} // ne
+ Dutch Tag = Tag{lang: _nl} // nl
+ Norwegian Tag = Tag{lang: _no} // no
+ Punjabi Tag = Tag{lang: _pa} // pa
+ Polish Tag = Tag{lang: _pl} // pl
+ Portuguese Tag = Tag{lang: _pt} // pt
+ BrazilianPortuguese Tag = Tag{lang: _pt, region: _BR} // pt-BR
+ EuropeanPortuguese Tag = Tag{lang: _pt, region: _PT} // pt-PT
+ Romanian Tag = Tag{lang: _ro} // ro
+ Russian Tag = Tag{lang: _ru} // ru
+ Sinhala Tag = Tag{lang: _si} // si
+ Slovak Tag = Tag{lang: _sk} // sk
+ Slovenian Tag = Tag{lang: _sl} // sl
+ Albanian Tag = Tag{lang: _sq} // sq
+ Serbian Tag = Tag{lang: _sr} // sr
+ SerbianLatin Tag = Tag{lang: _sr, script: _Latn} // sr-Latn
+ Swedish Tag = Tag{lang: _sv} // sv
+ Swahili Tag = Tag{lang: _sw} // sw
+ Tamil Tag = Tag{lang: _ta} // ta
+ Telugu Tag = Tag{lang: _te} // te
+ Thai Tag = Tag{lang: _th} // th
+ Turkish Tag = Tag{lang: _tr} // tr
+ Ukrainian Tag = Tag{lang: _uk} // uk
+ Urdu Tag = Tag{lang: _ur} // ur
+ Uzbek Tag = Tag{lang: _uz} // uz
+ Vietnamese Tag = Tag{lang: _vi} // vi
+ Chinese Tag = Tag{lang: _zh} // zh
+ SimplifiedChinese Tag = Tag{lang: _zh, script: _Hans} // zh-Hans
+ TraditionalChinese Tag = Tag{lang: _zh, script: _Hant} // zh-Hant
+ Zulu Tag = Tag{lang: _zu} // zu
+)
