| // Copyright 2009 The Go Authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style |
| // license that can be found in the LICENSE file. |
| |
| package unicode_test |
| |
| import ( |
| "flag" |
| "fmt" |
| "runtime" |
| "sort" |
| "strings" |
| "testing" |
| . "unicode" |
| ) |
| |
| var upperTest = []rune{ |
| 0x41, |
| 0xc0, |
| 0xd8, |
| 0x100, |
| 0x139, |
| 0x14a, |
| 0x178, |
| 0x181, |
| 0x376, |
| 0x3cf, |
| 0x13bd, |
| 0x1f2a, |
| 0x2102, |
| 0x2c00, |
| 0x2c10, |
| 0x2c20, |
| 0xa650, |
| 0xa722, |
| 0xff3a, |
| 0x10400, |
| 0x1d400, |
| 0x1d7ca, |
| } |
| |
| var notupperTest = []rune{ |
| 0x40, |
| 0x5b, |
| 0x61, |
| 0x185, |
| 0x1b0, |
| 0x377, |
| 0x387, |
| 0x2150, |
| 0xab7d, |
| 0xffff, |
| 0x10000, |
| } |
| |
| var letterTest = []rune{ |
| 0x41, |
| 0x61, |
| 0xaa, |
| 0xba, |
| 0xc8, |
| 0xdb, |
| 0xf9, |
| 0x2ec, |
| 0x535, |
| 0x620, |
| 0x6e6, |
| 0x93d, |
| 0xa15, |
| 0xb99, |
| 0xdc0, |
| 0xedd, |
| 0x1000, |
| 0x1200, |
| 0x1312, |
| 0x1401, |
| 0x2c00, |
| 0xa800, |
| 0xf900, |
| 0xfa30, |
| 0xffda, |
| 0xffdc, |
| 0x10000, |
| 0x10300, |
| 0x10400, |
| 0x20000, |
| 0x2f800, |
| 0x2fa1d, |
| } |
| |
| var notletterTest = []rune{ |
| 0x20, |
| 0x35, |
| 0x375, |
| 0x619, |
| 0x700, |
| 0x1885, |
| 0xfffe, |
| 0x1ffff, |
| 0x10ffff, |
| } |
| |
| // Contains all the special cased Latin-1 chars. |
| var spaceTest = []rune{ |
| 0x09, |
| 0x0a, |
| 0x0b, |
| 0x0c, |
| 0x0d, |
| 0x20, |
| 0x85, |
| 0xA0, |
| 0x2000, |
| 0x3000, |
| } |
| |
| type caseT struct { |
| cas int |
| in, out rune |
| } |
| |
| var caseTest = []caseT{ |
| // errors |
| {-1, '\n', 0xFFFD}, |
| {UpperCase, -1, -1}, |
| {UpperCase, 1 << 30, 1 << 30}, |
| |
| // ASCII (special-cased so test carefully) |
| {UpperCase, '\n', '\n'}, |
| {UpperCase, 'a', 'A'}, |
| {UpperCase, 'A', 'A'}, |
| {UpperCase, '7', '7'}, |
| {LowerCase, '\n', '\n'}, |
| {LowerCase, 'a', 'a'}, |
| {LowerCase, 'A', 'a'}, |
| {LowerCase, '7', '7'}, |
| {TitleCase, '\n', '\n'}, |
| {TitleCase, 'a', 'A'}, |
| {TitleCase, 'A', 'A'}, |
| {TitleCase, '7', '7'}, |
| |
| // Latin-1: easy to read the tests! |
| {UpperCase, 0x80, 0x80}, |
| {UpperCase, 'Å', 'Å'}, |
| {UpperCase, 'å', 'Å'}, |
| {LowerCase, 0x80, 0x80}, |
| {LowerCase, 'Å', 'å'}, |
| {LowerCase, 'å', 'å'}, |
| {TitleCase, 0x80, 0x80}, |
| {TitleCase, 'Å', 'Å'}, |
| {TitleCase, 'å', 'Å'}, |
| |
| // 0131;LATIN SMALL LETTER DOTLESS I;Ll;0;L;;;;;N;;;0049;;0049 |
| {UpperCase, 0x0131, 'I'}, |
| {LowerCase, 0x0131, 0x0131}, |
| {TitleCase, 0x0131, 'I'}, |
| |
| // 0133;LATIN SMALL LIGATURE IJ;Ll;0;L;<compat> 0069 006A;;;;N;LATIN SMALL LETTER I J;;0132;;0132 |
| {UpperCase, 0x0133, 0x0132}, |
| {LowerCase, 0x0133, 0x0133}, |
| {TitleCase, 0x0133, 0x0132}, |
| |
| // 212A;KELVIN SIGN;Lu;0;L;004B;;;;N;DEGREES KELVIN;;;006B; |
| {UpperCase, 0x212A, 0x212A}, |
| {LowerCase, 0x212A, 'k'}, |
| {TitleCase, 0x212A, 0x212A}, |
| |
| // From an UpperLower sequence |
| // A640;CYRILLIC CAPITAL LETTER ZEMLYA;Lu;0;L;;;;;N;;;;A641; |
| {UpperCase, 0xA640, 0xA640}, |
| {LowerCase, 0xA640, 0xA641}, |
| {TitleCase, 0xA640, 0xA640}, |
| // A641;CYRILLIC SMALL LETTER ZEMLYA;Ll;0;L;;;;;N;;;A640;;A640 |
| {UpperCase, 0xA641, 0xA640}, |
| {LowerCase, 0xA641, 0xA641}, |
| {TitleCase, 0xA641, 0xA640}, |
| // A64E;CYRILLIC CAPITAL LETTER NEUTRAL YER;Lu;0;L;;;;;N;;;;A64F; |
| {UpperCase, 0xA64E, 0xA64E}, |
| {LowerCase, 0xA64E, 0xA64F}, |
| {TitleCase, 0xA64E, 0xA64E}, |
| // A65F;CYRILLIC SMALL LETTER YN;Ll;0;L;;;;;N;;;A65E;;A65E |
| {UpperCase, 0xA65F, 0xA65E}, |
| {LowerCase, 0xA65F, 0xA65F}, |
| {TitleCase, 0xA65F, 0xA65E}, |
| |
| // From another UpperLower sequence |
| // 0139;LATIN CAPITAL LETTER L WITH ACUTE;Lu;0;L;004C 0301;;;;N;LATIN CAPITAL LETTER L ACUTE;;;013A; |
| {UpperCase, 0x0139, 0x0139}, |
| {LowerCase, 0x0139, 0x013A}, |
| {TitleCase, 0x0139, 0x0139}, |
| // 013F;LATIN CAPITAL LETTER L WITH MIDDLE DOT;Lu;0;L;<compat> 004C 00B7;;;;N;;;;0140; |
| {UpperCase, 0x013f, 0x013f}, |
| {LowerCase, 0x013f, 0x0140}, |
| {TitleCase, 0x013f, 0x013f}, |
| // 0148;LATIN SMALL LETTER N WITH CARON;Ll;0;L;006E 030C;;;;N;LATIN SMALL LETTER N HACEK;;0147;;0147 |
| {UpperCase, 0x0148, 0x0147}, |
| {LowerCase, 0x0148, 0x0148}, |
| {TitleCase, 0x0148, 0x0147}, |
| |
| // Lowercase lower than uppercase. |
| // AB78;CHEROKEE SMALL LETTER GE;Ll;0;L;;;;;N;;;13A8;;13A8 |
| {UpperCase, 0xab78, 0x13a8}, |
| {LowerCase, 0xab78, 0xab78}, |
| {TitleCase, 0xab78, 0x13a8}, |
| {UpperCase, 0x13a8, 0x13a8}, |
| {LowerCase, 0x13a8, 0xab78}, |
| {TitleCase, 0x13a8, 0x13a8}, |
| |
| // Last block in the 5.1.0 table |
| // 10400;DESERET CAPITAL LETTER LONG I;Lu;0;L;;;;;N;;;;10428; |
| {UpperCase, 0x10400, 0x10400}, |
| {LowerCase, 0x10400, 0x10428}, |
| {TitleCase, 0x10400, 0x10400}, |
| // 10427;DESERET CAPITAL LETTER EW;Lu;0;L;;;;;N;;;;1044F; |
| {UpperCase, 0x10427, 0x10427}, |
| {LowerCase, 0x10427, 0x1044F}, |
| {TitleCase, 0x10427, 0x10427}, |
| // 10428;DESERET SMALL LETTER LONG I;Ll;0;L;;;;;N;;;10400;;10400 |
| {UpperCase, 0x10428, 0x10400}, |
| {LowerCase, 0x10428, 0x10428}, |
| {TitleCase, 0x10428, 0x10400}, |
| // 1044F;DESERET SMALL LETTER EW;Ll;0;L;;;;;N;;;10427;;10427 |
| {UpperCase, 0x1044F, 0x10427}, |
| {LowerCase, 0x1044F, 0x1044F}, |
| {TitleCase, 0x1044F, 0x10427}, |
| |
| // First one not in the 5.1.0 table |
| // 10450;SHAVIAN LETTER PEEP;Lo;0;L;;;;;N;;;;; |
| {UpperCase, 0x10450, 0x10450}, |
| {LowerCase, 0x10450, 0x10450}, |
| {TitleCase, 0x10450, 0x10450}, |
| |
| // Non-letters with case. |
| {LowerCase, 0x2161, 0x2171}, |
| {UpperCase, 0x0345, 0x0399}, |
| } |
| |
| func TestIsLetter(t *testing.T) { |
| for _, r := range upperTest { |
| if !IsLetter(r) { |
| t.Errorf("IsLetter(U+%04X) = false, want true", r) |
| } |
| } |
| for _, r := range letterTest { |
| if !IsLetter(r) { |
| t.Errorf("IsLetter(U+%04X) = false, want true", r) |
| } |
| } |
| for _, r := range notletterTest { |
| if IsLetter(r) { |
| t.Errorf("IsLetter(U+%04X) = true, want false", r) |
| } |
| } |
| } |
| |
| func TestIsUpper(t *testing.T) { |
| for _, r := range upperTest { |
| if !IsUpper(r) { |
| t.Errorf("IsUpper(U+%04X) = false, want true", r) |
| } |
| } |
| for _, r := range notupperTest { |
| if IsUpper(r) { |
| t.Errorf("IsUpper(U+%04X) = true, want false", r) |
| } |
| } |
| for _, r := range notletterTest { |
| if IsUpper(r) { |
| t.Errorf("IsUpper(U+%04X) = true, want false", r) |
| } |
| } |
| } |
| |
| func caseString(c int) string { |
| switch c { |
| case UpperCase: |
| return "UpperCase" |
| case LowerCase: |
| return "LowerCase" |
| case TitleCase: |
| return "TitleCase" |
| } |
| return "ErrorCase" |
| } |
| |
| func TestTo(t *testing.T) { |
| for _, c := range caseTest { |
| r := To(c.cas, c.in) |
| if c.out != r { |
| t.Errorf("To(U+%04X, %s) = U+%04X want U+%04X", c.in, caseString(c.cas), r, c.out) |
| } |
| } |
| } |
| |
| func TestToUpperCase(t *testing.T) { |
| for _, c := range caseTest { |
| if c.cas != UpperCase { |
| continue |
| } |
| r := ToUpper(c.in) |
| if c.out != r { |
| t.Errorf("ToUpper(U+%04X) = U+%04X want U+%04X", c.in, r, c.out) |
| } |
| } |
| } |
| |
| func TestToLowerCase(t *testing.T) { |
| for _, c := range caseTest { |
| if c.cas != LowerCase { |
| continue |
| } |
| r := ToLower(c.in) |
| if c.out != r { |
| t.Errorf("ToLower(U+%04X) = U+%04X want U+%04X", c.in, r, c.out) |
| } |
| } |
| } |
| |
| func TestToTitleCase(t *testing.T) { |
| for _, c := range caseTest { |
| if c.cas != TitleCase { |
| continue |
| } |
| r := ToTitle(c.in) |
| if c.out != r { |
| t.Errorf("ToTitle(U+%04X) = U+%04X want U+%04X", c.in, r, c.out) |
| } |
| } |
| } |
| |
| func TestIsSpace(t *testing.T) { |
| for _, c := range spaceTest { |
| if !IsSpace(c) { |
| t.Errorf("IsSpace(U+%04X) = false; want true", c) |
| } |
| } |
| for _, c := range letterTest { |
| if IsSpace(c) { |
| t.Errorf("IsSpace(U+%04X) = true; want false", c) |
| } |
| } |
| } |
| |
| // Check that the optimizations for IsLetter etc. agree with the tables. |
| // We only need to check the Latin-1 range. |
| func TestLetterOptimizations(t *testing.T) { |
| for i := rune(0); i <= MaxLatin1; i++ { |
| if Is(Letter, i) != IsLetter(i) { |
| t.Errorf("IsLetter(U+%04X) disagrees with Is(Letter)", i) |
| } |
| if Is(Upper, i) != IsUpper(i) { |
| t.Errorf("IsUpper(U+%04X) disagrees with Is(Upper)", i) |
| } |
| if Is(Lower, i) != IsLower(i) { |
| t.Errorf("IsLower(U+%04X) disagrees with Is(Lower)", i) |
| } |
| if Is(Title, i) != IsTitle(i) { |
| t.Errorf("IsTitle(U+%04X) disagrees with Is(Title)", i) |
| } |
| if Is(White_Space, i) != IsSpace(i) { |
| t.Errorf("IsSpace(U+%04X) disagrees with Is(White_Space)", i) |
| } |
| if To(UpperCase, i) != ToUpper(i) { |
| t.Errorf("ToUpper(U+%04X) disagrees with To(Upper)", i) |
| } |
| if To(LowerCase, i) != ToLower(i) { |
| t.Errorf("ToLower(U+%04X) disagrees with To(Lower)", i) |
| } |
| if To(TitleCase, i) != ToTitle(i) { |
| t.Errorf("ToTitle(U+%04X) disagrees with To(Title)", i) |
| } |
| } |
| } |
| |
| func TestTurkishCase(t *testing.T) { |
| lower := []rune("abcçdefgğhıijklmnoöprsştuüvyz") |
| upper := []rune("ABCÇDEFGĞHIİJKLMNOÖPRSŞTUÜVYZ") |
| for i, l := range lower { |
| u := upper[i] |
| if TurkishCase.ToLower(l) != l { |
| t.Errorf("lower(U+%04X) is U+%04X not U+%04X", l, TurkishCase.ToLower(l), l) |
| } |
| if TurkishCase.ToUpper(u) != u { |
| t.Errorf("upper(U+%04X) is U+%04X not U+%04X", u, TurkishCase.ToUpper(u), u) |
| } |
| if TurkishCase.ToUpper(l) != u { |
| t.Errorf("upper(U+%04X) is U+%04X not U+%04X", l, TurkishCase.ToUpper(l), u) |
| } |
| if TurkishCase.ToLower(u) != l { |
| t.Errorf("lower(U+%04X) is U+%04X not U+%04X", u, TurkishCase.ToLower(l), l) |
| } |
| if TurkishCase.ToTitle(u) != u { |
| t.Errorf("title(U+%04X) is U+%04X not U+%04X", u, TurkishCase.ToTitle(u), u) |
| } |
| if TurkishCase.ToTitle(l) != u { |
| t.Errorf("title(U+%04X) is U+%04X not U+%04X", l, TurkishCase.ToTitle(l), u) |
| } |
| } |
| } |
| |
| var simpleFoldTests = []string{ |
| // SimpleFold(x) returns the next equivalent rune > x or wraps |
| // around to smaller values. |
| |
| // Easy cases. |
| "Aa", |
| "δΔ", |
| |
| // ASCII special cases. |
| "KkK", |
| "Ssſ", |
| |
| // Non-ASCII special cases. |
| "ρϱΡ", |
| "ͅΙιι", |
| |
| // Extra special cases: has lower/upper but no case fold. |
| "İ", |
| "ı", |
| |
| // Upper comes before lower (Cherokee). |
| "\u13b0\uab80", |
| } |
| |
| func TestSimpleFold(t *testing.T) { |
| for _, tt := range simpleFoldTests { |
| cycle := []rune(tt) |
| r := cycle[len(cycle)-1] |
| for _, out := range cycle { |
| if r := SimpleFold(r); r != out { |
| t.Errorf("SimpleFold(%#U) = %#U, want %#U", r, r, out) |
| } |
| r = out |
| } |
| } |
| |
| if r := SimpleFold(-42); r != -42 { |
| t.Errorf("SimpleFold(-42) = %v, want -42", r) |
| } |
| } |
| |
| // Running 'go test -calibrate' runs the calibration to find a plausible |
| // cutoff point for linear search of a range list vs. binary search. |
| // We create a fake table and then time how long it takes to do a |
| // sequence of searches within that table, for all possible inputs |
| // relative to the ranges (something before all, in each, between each, after all). |
| // This assumes that all possible runes are equally likely. |
| // In practice most runes are ASCII so this is a conservative estimate |
| // of an effective cutoff value. In practice we could probably set it higher |
| // than what this function recommends. |
| |
| var calibrate = flag.Bool("calibrate", false, "compute crossover for linear vs. binary search") |
| |
| func TestCalibrate(t *testing.T) { |
| if !*calibrate { |
| return |
| } |
| |
| if runtime.GOARCH == "amd64" { |
| fmt.Printf("warning: running calibration on %s\n", runtime.GOARCH) |
| } |
| |
| // Find the point where binary search wins by more than 10%. |
| // The 10% bias gives linear search an edge when they're close, |
| // because on predominantly ASCII inputs linear search is even |
| // better than our benchmarks measure. |
| n := sort.Search(64, func(n int) bool { |
| tab := fakeTable(n) |
| blinear := func(b *testing.B) { |
| tab := tab |
| max := n*5 + 20 |
| for i := 0; i < b.N; i++ { |
| for j := 0; j <= max; j++ { |
| linear(tab, uint16(j)) |
| } |
| } |
| } |
| bbinary := func(b *testing.B) { |
| tab := tab |
| max := n*5 + 20 |
| for i := 0; i < b.N; i++ { |
| for j := 0; j <= max; j++ { |
| binary(tab, uint16(j)) |
| } |
| } |
| } |
| bmlinear := testing.Benchmark(blinear) |
| bmbinary := testing.Benchmark(bbinary) |
| fmt.Printf("n=%d: linear=%d binary=%d\n", n, bmlinear.NsPerOp(), bmbinary.NsPerOp()) |
| return bmlinear.NsPerOp()*100 > bmbinary.NsPerOp()*110 |
| }) |
| fmt.Printf("calibration: linear cutoff = %d\n", n) |
| } |
| |
| func fakeTable(n int) []Range16 { |
| var r16 []Range16 |
| for i := 0; i < n; i++ { |
| r16 = append(r16, Range16{uint16(i*5 + 10), uint16(i*5 + 12), 1}) |
| } |
| return r16 |
| } |
| |
| func linear(ranges []Range16, r uint16) bool { |
| for i := range ranges { |
| range_ := &ranges[i] |
| if r < range_.Lo { |
| return false |
| } |
| if r <= range_.Hi { |
| return (r-range_.Lo)%range_.Stride == 0 |
| } |
| } |
| return false |
| } |
| |
| func binary(ranges []Range16, r uint16) bool { |
| // binary search over ranges |
| lo := 0 |
| hi := len(ranges) |
| for lo < hi { |
| m := int(uint(lo+hi) >> 1) |
| range_ := &ranges[m] |
| if range_.Lo <= r && r <= range_.Hi { |
| return (r-range_.Lo)%range_.Stride == 0 |
| } |
| if r < range_.Lo { |
| hi = m |
| } else { |
| lo = m + 1 |
| } |
| } |
| return false |
| } |
| |
| func TestLatinOffset(t *testing.T) { |
| var maps = []map[string]*RangeTable{ |
| Categories, |
| FoldCategory, |
| FoldScript, |
| Properties, |
| Scripts, |
| } |
| for _, m := range maps { |
| for name, tab := range m { |
| i := 0 |
| for i < len(tab.R16) && tab.R16[i].Hi <= MaxLatin1 { |
| i++ |
| } |
| if tab.LatinOffset != i { |
| t.Errorf("%s: LatinOffset=%d, want %d", name, tab.LatinOffset, i) |
| } |
| } |
| } |
| } |
| |
| func TestSpecialCaseNoMapping(t *testing.T) { |
| // Issue 25636 |
| // no change for rune 'A', zero delta, under upper/lower/title case change. |
| var noChangeForCapitalA = CaseRange{'A', 'A', [MaxCase]rune{0, 0, 0}} |
| got := strings.ToLowerSpecial(SpecialCase([]CaseRange{noChangeForCapitalA}), "ABC") |
| want := "Abc" |
| if got != want { |
| t.Errorf("got %q; want %q", got, want) |
| } |
| } |
| |
| func TestNegativeRune(t *testing.T) { |
| // Issue 43254 |
| // These tests cover negative rune handling by testing values which, |
| // when cast to uint8 or uint16, look like a particular valid rune. |
| // This package has Latin-1-specific optimizations, so we test all of |
| // Latin-1 and representative non-Latin-1 values in the character |
| // categories covered by IsGraphic, etc. |
| nonLatin1 := []uint32{ |
| // Lu: LATIN CAPITAL LETTER A WITH MACRON |
| 0x0100, |
| // Ll: LATIN SMALL LETTER A WITH MACRON |
| 0x0101, |
| // Lt: LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON |
| 0x01C5, |
| // M: COMBINING GRAVE ACCENT |
| 0x0300, |
| // Nd: ARABIC-INDIC DIGIT ZERO |
| 0x0660, |
| // P: GREEK QUESTION MARK |
| 0x037E, |
| // S: MODIFIER LETTER LEFT ARROWHEAD |
| 0x02C2, |
| // Z: OGHAM SPACE MARK |
| 0x1680, |
| } |
| for i := 0; i < MaxLatin1+len(nonLatin1); i++ { |
| base := uint32(i) |
| if i >= MaxLatin1 { |
| base = nonLatin1[i-MaxLatin1] |
| } |
| |
| // Note r is negative, but uint8(r) == uint8(base) and |
| // uint16(r) == uint16(base). |
| r := rune(base - 1<<31) |
| if Is(Letter, r) { |
| t.Errorf("Is(Letter, 0x%x - 1<<31) = true, want false", base) |
| } |
| if IsControl(r) { |
| t.Errorf("IsControl(0x%x - 1<<31) = true, want false", base) |
| } |
| if IsDigit(r) { |
| t.Errorf("IsDigit(0x%x - 1<<31) = true, want false", base) |
| } |
| if IsGraphic(r) { |
| t.Errorf("IsGraphic(0x%x - 1<<31) = true, want false", base) |
| } |
| if IsLetter(r) { |
| t.Errorf("IsLetter(0x%x - 1<<31) = true, want false", base) |
| } |
| if IsLower(r) { |
| t.Errorf("IsLower(0x%x - 1<<31) = true, want false", base) |
| } |
| if IsMark(r) { |
| t.Errorf("IsMark(0x%x - 1<<31) = true, want false", base) |
| } |
| if IsNumber(r) { |
| t.Errorf("IsNumber(0x%x - 1<<31) = true, want false", base) |
| } |
| if IsPrint(r) { |
| t.Errorf("IsPrint(0x%x - 1<<31) = true, want false", base) |
| } |
| if IsPunct(r) { |
| t.Errorf("IsPunct(0x%x - 1<<31) = true, want false", base) |
| } |
| if IsSpace(r) { |
| t.Errorf("IsSpace(0x%x - 1<<31) = true, want false", base) |
| } |
| if IsSymbol(r) { |
| t.Errorf("IsSymbol(0x%x - 1<<31) = true, want false", base) |
| } |
| if IsTitle(r) { |
| t.Errorf("IsTitle(0x%x - 1<<31) = true, want false", base) |
| } |
| if IsUpper(r) { |
| t.Errorf("IsUpper(0x%x - 1<<31) = true, want false", base) |
| } |
| } |
| } |