language: implement paradigm locales
This allows deleting parent distance.
This also fully implements allowing the user to
order dialects non-contiguously.
Change-Id: I09a8d21a6c6e18edc24db136b280ba960eb72370
Reviewed-on: https://go-review.googlesource.com/55911
Run-TryBot: Marcel van Lohuizen <mpvl@golang.org>
Reviewed-by: Nigel Tao <nigeltao@golang.org>
diff --git a/language/gen.go b/language/gen.go
index 50f772d..7c260e5 100644
--- a/language/gen.go
+++ b/language/gen.go
@@ -1417,6 +1417,22 @@
}
b.writeSlice("regionToGroups", regionToGroups)
+ // maps language id to in- and out-of-group region.
+ paradigmLocales := [][3]uint16{}
+ locales := strings.Split(lm[0].ParadigmLocales[0].Locales, " ")
+ for i := 0; i < len(locales); i += 2 {
+ x := [3]uint16{}
+ for j := 0; j < 2; j++ {
+ pc := strings.SplitN(locales[i+j], "-", 2)
+ x[0] = b.langIndex(pc[0])
+ if len(pc) == 2 {
+ x[1+j] = uint16(b.region.index(pc[1]))
+ }
+ }
+ paradigmLocales = append(paradigmLocales, x)
+ }
+ b.writeSlice("paradigmLocales", paradigmLocales)
+
b.writeType(mutualIntelligibility{})
b.writeType(scriptIntelligibility{})
b.writeType(regionIntelligibility{})
diff --git a/language/match.go b/language/match.go
index 2f14436..b7779ed 100644
--- a/language/match.go
+++ b/language/match.go
@@ -559,9 +559,6 @@
}
}
- // TODO: include alt script.
- // - don't replace regions, but allow regions to be made more specific.
-
// update is used to add indexes in the map for equivalent languages.
// update will only add entries to original indexes, thus not computing any
// transitive relations.
@@ -687,16 +684,18 @@
// bestMatch accumulates the best match so far.
type bestMatch struct {
- have *haveTag
- want Tag
- conf Confidence
- pinLanguage bool
+ have *haveTag
+ want Tag
+ conf Confidence
+ pinnedRegion regionID
+ pinLanguage bool
+ sameRegionGroup bool
// Cached results from applying tie-breaking rules.
origLang bool
origReg bool
+ paradigmReg bool
regGroupDist uint8
origScript bool
- parentDist uint8 // 255 if have is not an ancestor of want tag.
}
// update updates the existing best match if the new pair is considered to be a
@@ -723,12 +722,20 @@
if m.pinLanguage && tag.lang != m.want.lang {
return
}
- if c == Exact && have.tag.script == tag.script {
+ // Pin the region group if we are comparing tags for the same language.
+ if tag.lang == m.want.lang && m.sameRegionGroup {
+ _, sameGroup := regionGroupDist(m.pinnedRegion, have.maxRegion, have.maxScript, m.want.lang)
+ if !sameGroup {
+ return
+ }
+ }
+ if c == Exact && have.maxScript == maxScript {
+ // If there is another language and then another entry of this language,
+ // don't pin anything, otherwise pin the language.
m.pinLanguage = pin
}
if have.tag.equalsRest(tag) {
} else if have.maxScript != maxScript {
- // fmt.Println("FFFFF", maxScript, have.maxScript)
// There is usually very little comprehension between different scripts.
// In a few cases there may still be Low comprehension. This possibility
// is pre-computed and stored in have.altScript.
@@ -737,9 +744,8 @@
}
c = Low
} else if have.maxRegion != maxRegion {
- // There is usually a small difference between languages across regions.
- // We use the region distance (below) to disambiguate between equal matches.
if High < c {
+ // There is usually a small difference between languages across regions.
c = High
}
}
@@ -766,7 +772,16 @@
beaten = true
}
- regGroupDist := regionGroupDist(have.maxRegion, maxRegion, maxScript, tag.lang)
+ // We prefer if the pre-maximized region was specified and identical.
+ origReg := have.tag.region == tag.region && tag.region != 0
+ if !beaten && m.origReg != origReg {
+ if m.origReg {
+ return
+ }
+ beaten = true
+ }
+
+ regGroupDist, sameGroup := regionGroupDist(have.maxRegion, maxRegion, maxScript, tag.lang)
if !beaten && m.regGroupDist != regGroupDist {
if regGroupDist > m.regGroupDist {
return
@@ -774,10 +789,9 @@
beaten = true
}
- // We prefer if the pre-maximized region was specified and identical.
- origReg := have.tag.region == tag.region && tag.region != 0
- if !beaten && m.origReg != origReg {
- if m.origReg {
+ paradigmReg := isParadigmLocale(tag.lang, have.maxRegion)
+ if !beaten && m.paradigmReg != paradigmReg {
+ if !paradigmReg {
return
}
beaten = true
@@ -792,48 +806,35 @@
beaten = true
}
- // TODO: remove parent distance once primary locales are implemented.
- parentDist := parentDistance(have.tag.region, tag)
- if !beaten && m.parentDist != parentDist {
- if parentDist > m.parentDist {
- return
- }
- beaten = true
- }
-
// Update m to the newly found best match.
if beaten {
m.have = have
m.want = tag
m.conf = c
+ m.pinnedRegion = maxRegion
+ m.sameRegionGroup = sameGroup
m.origLang = origLang
m.origReg = origReg
+ m.paradigmReg = paradigmReg
m.origScript = origScript
m.regGroupDist = regGroupDist
- m.parentDist = parentDist
}
}
-// parentDistance returns the number of times Parent must be called before the
-// regions match. It is assumed that it has already been checked that lang and
-// script are identical. If haveRegion does not occur in the ancestor chain of
-// tag, it returns 255.
-func parentDistance(haveRegion regionID, tag Tag) uint8 {
- p := tag.Parent()
- d := uint8(1)
- for haveRegion != p.region {
- if p.region == 0 {
- return 255
+func isParadigmLocale(lang langID, r regionID) bool {
+ for _, e := range paradigmLocales {
+ if langID(e[0]) == lang && (r == regionID(e[1]) || r == regionID(e[2])) {
+ return true
}
- p = p.Parent()
- d++
}
- return d
+ return false
}
// regionGroupDist computes the distance between two regions based on their
// CLDR grouping.
-func regionGroupDist(a, b regionID, script scriptID, lang langID) uint8 {
+func regionGroupDist(a, b regionID, script scriptID, lang langID) (dist uint8, same bool) {
+ const defaultDistance = 4
+
aGroup := uint(regionToGroups[a]) << 1
bGroup := uint(regionToGroups[b]) << 1
for _, ri := range matchRegion {
@@ -841,17 +842,16 @@
group := uint(1 << (ri.group &^ 0x80))
if 0x80&ri.group == 0 {
if aGroup&bGroup&group != 0 { // Both regions are in the group.
- return ri.distance
+ return ri.distance, ri.distance == defaultDistance
}
} else {
if (aGroup|bGroup)&group == 0 { // Both regions are not in the group.
- return ri.distance
+ return ri.distance, ri.distance == defaultDistance
}
}
}
}
- const defaultDistance = 4
- return defaultDistance
+ return defaultDistance, true
}
func (t Tag) variants() string {
@@ -898,4 +898,14 @@
notEquivalent = append(notEquivalent, langID(lm.from))
}
}
+ // Maximize undefined regions of paradigm locales.
+ for i, v := range paradigmLocales {
+ max, _ := addTags(Tag{lang: langID(v[0])})
+ if v[1] == 0 {
+ paradigmLocales[i][1] = uint16(max.region)
+ }
+ if v[2] == 0 {
+ paradigmLocales[i][2] = uint16(max.region)
+ }
+ }
}
diff --git a/language/match_test.go b/language/match_test.go
index ca77115..f1b9010 100644
--- a/language/match_test.go
+++ b/language/match_test.go
@@ -64,15 +64,6 @@
// combined: got "en-GB-u-ca-buddhist-nu-arab"; want "en-GB-fonipa-t-m0-iso-i0-pinyin-u-ca-buddhist-nu-arab"
"und,en-GB-u-sd-gbsct/en-fonipa-u-nu-Arab-ca-buddhist-t-m0-iso-i0-pinyin": true,
- // Go prefers exact matches over less exact preferred ones.
- // Preferring desired ones might be better.
- // NOTE: allow users to distinguish languages is a good solution.
- // the remaining cases are due to preferred locale rules.
- "pt-PT,pt-BR,es,es-419/pt-US,pt-PT": true, // match: got "pt-PT"; want "pt-BR"
- "pt-PT,pt,es,es-419/pt-US,pt-PT,pt": true, // match: got "pt-PT"; want "pt"
- // TODO: implement prefer primary locales.
- "und,en,en-GU,en-IN,en-GB/en-ZA": true, // match: got "en-IN"; want "en-GB"
-
// Inconsistencies with Mark Davis' implementation where it is not clear
// which is better.
@@ -259,6 +250,12 @@
}{
{"zh-TW", "zh-HK", 5},
{"zh-MO", "zh-HK", 4},
+ {"es-ES", "es-AR", 5},
+ {"es-ES", "es", 4},
+ {"es-419", "es-MX", 4},
+ {"es-AR", "es-MX", 4},
+ {"es-ES", "es-MX", 5},
+ {"es-PT", "es-MX", 5},
}
for _, tc := range testCases {
a := MustParse(tc.a)
@@ -270,33 +267,27 @@
t.Errorf("scripts differ: %q vs %q", aScript, bScript)
continue
}
- d := regionGroupDist(a.region, b.region, aScript.scriptID, a.lang)
+ d, _ := regionGroupDist(a.region, b.region, aScript.scriptID, a.lang)
if d != tc.distance {
t.Errorf("got %q; want %q", d, tc.distance)
}
}
}
-func TestParentDistance(t *testing.T) {
- tests := []struct {
- parent string
- tag string
- d uint8
- }{
- {"en-001", "en-AU", 1},
- {"pt-PT", "pt-AO", 1},
- {"pt", "pt-AO", 2},
- {"en-AU", "en-GB", 255},
- {"en-NL", "en-AU", 255},
- // Note that pt-BR and en-US are not automatically minimized.
- {"pt-BR", "pt-AO", 255},
- {"en-US", "en-AU", 255},
+func TestIsParadigmLocale(t *testing.T) {
+ testCases := map[string]bool{
+ "en-US": true,
+ "en-GB": true,
+ "en-VI": false,
+ "es-GB": false,
+ "es-ES": true,
+ "es-419": true,
}
- for _, tt := range tests {
- r := Raw.MustParse(tt.parent).region
- tag := Raw.MustParse(tt.tag)
- if d := parentDistance(r, tag); d != tt.d {
- t.Errorf("d(%s, %s) was %d; want %d", r, tag, d, tt.d)
+ for str, want := range testCases {
+ tag := Make(str)
+ got := isParadigmLocale(tag.lang, tag.region)
+ if got != want {
+ t.Errorf("isPL(%q) = %v; want %v", str, got, want)
}
}
}
diff --git a/language/tables.go b/language/tables.go
index a5e59d3..ec17f97 100644
--- a/language/tables.go
+++ b/language/tables.go
@@ -3335,6 +3335,13 @@
0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
}
+// Size: 18 bytes, 3 elements
+var paradigmLocales = [3][3]uint16{
+ 0: [3]uint16{0x138, 0x0, 0x7b},
+ 1: [3]uint16{0x13d, 0x0, 0x1f},
+ 2: [3]uint16{0x3be, 0x41, 0xee},
+}
+
type mutualIntelligibility struct {
want uint16
have uint16
@@ -3665,4 +3672,4 @@
4: {lang: 0x527, script: 0x38, maxScript: 0x38, toRegion: 0x8d, fromRegion: []uint16{0xc6}},
}
-// Total table size 27157 bytes (26KiB); checksum: F21EE307
+// Total table size 27175 bytes (26KiB); checksum: 569649CD
