| # Copyright 2019 Google LLC |
| # |
| # Licensed under the Apache License, Version 2.0 (the "License"); |
| # you may not use this file except in compliance with the License. |
| # You may obtain a copy of the License at |
| # |
| # https://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, software |
| # distributed under the License is distributed on an "AS IS" BASIS, |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| # See the License for the specific language governing permissions and |
| # limitations under the License. |
| |
| """Tests for tokenizer.""" |
| |
| import unittest |
| from compiler.front_end import tokenizer |
| from compiler.util import error |
| from compiler.util import parser_types |
| |
| |
| def _token_symbols(token_list): |
| """Given a list of tokens, returns a list of their symbol names.""" |
| return [token.symbol for token in token_list] |
| |
| |
| class TokenizerTest(unittest.TestCase): |
| """Tests for the tokenizer.tokenize function.""" |
| |
| def test_bad_indent_tab_versus_space(self): |
| # A bad indent is one that doesn't match a previous unmatched indent. |
| tokens, errors = tokenizer.tokenize(" a\n\tb", "file") |
| self.assertFalse(tokens) |
| self.assertEqual([[error.error("file", parser_types.make_location( |
| (2, 1), (2, 2)), "Bad indentation")]], errors) |
| |
| def test_bad_indent_tab_versus_eight_spaces(self): |
| tokens, errors = tokenizer.tokenize(" a\n\tb", "file") |
| self.assertFalse(tokens) |
| self.assertEqual([[error.error("file", parser_types.make_location( |
| (2, 1), (2, 2)), "Bad indentation")]], errors) |
| |
| def test_bad_indent_tab_versus_four_spaces(self): |
| tokens, errors = tokenizer.tokenize(" a\n\tb", "file") |
| self.assertFalse(tokens) |
| self.assertEqual([[error.error("file", parser_types.make_location( |
| (2, 1), (2, 2)), "Bad indentation")]], errors) |
| |
| def test_bad_indent_two_spaces_versus_one_space(self): |
| tokens, errors = tokenizer.tokenize(" a\n b", "file") |
| self.assertFalse(tokens) |
| self.assertEqual([[error.error("file", parser_types.make_location( |
| (2, 1), (2, 2)), "Bad indentation")]], errors) |
| |
| def test_bad_indent_matches_closed_indent(self): |
| tokens, errors = tokenizer.tokenize(" a\nb\n c\n d", "file") |
| self.assertFalse(tokens) |
| self.assertEqual([[error.error("file", parser_types.make_location( |
| (4, 1), (4, 2)), "Bad indentation")]], errors) |
| |
| def test_bad_string_after_string_with_escaped_backslash_at_end(self): |
| tokens, errors = tokenizer.tokenize(r'"\\""', "name") |
| self.assertFalse(tokens) |
| self.assertEqual([[error.error("name", parser_types.make_location( |
| (1, 5), (1, 6)), "Unrecognized token")]], errors) |
| |
| |
| def _make_short_token_match_tests(): |
| """Makes tests for short, simple tokenization cases.""" |
| eol = '"\\n"' |
| cases = { |
| "Cam": ["CamelWord", eol], |
| "Ca9": ["CamelWord", eol], |
| "CanB": ["CamelWord", eol], |
| "CanBee": ["CamelWord", eol], |
| "CBa": ["CamelWord", eol], |
| "cam": ["SnakeWord", eol], |
| "ca9": ["SnakeWord", eol], |
| "can_b": ["SnakeWord", eol], |
| "can_bee": ["SnakeWord", eol], |
| "c_ba": ["SnakeWord", eol], |
| "cba_": ["SnakeWord", eol], |
| "c_b_a_": ["SnakeWord", eol], |
| "CAM": ["ShoutyWord", eol], |
| "CA9": ["ShoutyWord", eol], |
| "CAN_B": ["ShoutyWord", eol], |
| "CAN_BEE": ["ShoutyWord", eol], |
| "C_BA": ["ShoutyWord", eol], |
| "C": ["BadWord", eol], |
| "C1": ["BadWord", eol], |
| "c": ["SnakeWord", eol], |
| "$": ["BadWord", eol], |
| "_": ["BadWord", eol], |
| "_a": ["BadWord", eol], |
| "_A": ["BadWord", eol], |
| "Cb_A": ["BadWord", eol], |
| "aCb": ["BadWord", eol], |
| "a b": ["SnakeWord", "SnakeWord", eol], |
| "a\tb": ["SnakeWord", "SnakeWord", eol], |
| "a \t b ": ["SnakeWord", "SnakeWord", eol], |
| " \t ": [eol], |
| "a #b": ["SnakeWord", "Comment", eol], |
| "a#": ["SnakeWord", "Comment", eol], |
| "# b": ["Comment", eol], |
| " # b": ["Comment", eol], |
| " #": ["Comment", eol], |
| "": [], |
| "\n": [eol], |
| "\na": [eol, "SnakeWord", eol], |
| "a--example": ["SnakeWord", "BadDocumentation", eol], |
| "a ---- example": ["SnakeWord", "BadDocumentation", eol], |
| "a --- example": ["SnakeWord", "BadDocumentation", eol], |
| "a-- example": ["SnakeWord", "Documentation", eol], |
| "a -- -- example": ["SnakeWord", "Documentation", eol], |
| "a -- - example": ["SnakeWord", "Documentation", eol], |
| "--": ["Documentation", eol], |
| "-- ": ["Documentation", eol], |
| "-- ": ["Documentation", eol], |
| "$default": ['"$default"', eol], |
| "$defaultx": ["BadWord", eol], |
| "$def": ["BadWord", eol], |
| "x$default": ["BadWord", eol], |
| "9$default": ["BadWord", eol], |
| "struct": ['"struct"', eol], |
| "external": ['"external"', eol], |
| "bits": ['"bits"', eol], |
| "enum": ['"enum"', eol], |
| "as": ['"as"', eol], |
| "import": ['"import"', eol], |
| "true": ["BooleanConstant", eol], |
| "false": ["BooleanConstant", eol], |
| "truex": ["SnakeWord", eol], |
| "falsex": ["SnakeWord", eol], |
| "structx": ["SnakeWord", eol], |
| "bitsx": ["SnakeWord", eol], |
| "enumx": ["SnakeWord", eol], |
| "0b": ["BadNumber", eol], |
| "0x": ["BadNumber", eol], |
| "0b011101": ["Number", eol], |
| "0b0": ["Number", eol], |
| "0b0111_1111_0000": ["Number", eol], |
| "0b00_000_00": ["BadNumber", eol], |
| "0b0_0_0": ["BadNumber", eol], |
| "0b0111012": ["BadNumber", eol], |
| "0b011101x": ["BadWord", eol], |
| "0b011101b": ["BadNumber", eol], |
| "0B0": ["BadNumber", eol], |
| "0X0": ["BadNumber", eol], |
| "0b_": ["BadNumber", eol], |
| "0x_": ["BadNumber", eol], |
| "0b__": ["BadNumber", eol], |
| "0x__": ["BadNumber", eol], |
| "0b_0000": ["Number", eol], |
| "0b0000_": ["BadNumber", eol], |
| "0b00_____00": ["BadNumber", eol], |
| "0x00_000_00": ["BadNumber", eol], |
| "0x0_0_0": ["BadNumber", eol], |
| "0b____0____": ["BadNumber", eol], |
| "0b00000000000000000000": ["Number", eol], |
| "0b_00000000": ["Number", eol], |
| "0b0000_0000_0000": ["Number", eol], |
| "0b000_0000_0000": ["Number", eol], |
| "0b00_0000_0000": ["Number", eol], |
| "0b0_0000_0000": ["Number", eol], |
| "0b_0000_0000_0000": ["Number", eol], |
| "0b_000_0000_0000": ["Number", eol], |
| "0b_00_0000_0000": ["Number", eol], |
| "0b_0_0000_0000": ["Number", eol], |
| "0b00000000_00000000_00000000": ["Number", eol], |
| "0b0000000_00000000_00000000": ["Number", eol], |
| "0b000000_00000000_00000000": ["Number", eol], |
| "0b00000_00000000_00000000": ["Number", eol], |
| "0b0000_00000000_00000000": ["Number", eol], |
| "0b000_00000000_00000000": ["Number", eol], |
| "0b00_00000000_00000000": ["Number", eol], |
| "0b0_00000000_00000000": ["Number", eol], |
| "0b_00000000_00000000_00000000": ["Number", eol], |
| "0b_0000000_00000000_00000000": ["Number", eol], |
| "0b_000000_00000000_00000000": ["Number", eol], |
| "0b_00000_00000000_00000000": ["Number", eol], |
| "0b_0000_00000000_00000000": ["Number", eol], |
| "0b_000_00000000_00000000": ["Number", eol], |
| "0b_00_00000000_00000000": ["Number", eol], |
| "0b_0_00000000_00000000": ["Number", eol], |
| "0x0": ["Number", eol], |
| "0x00000000000000000000": ["Number", eol], |
| "0x_0000": ["Number", eol], |
| "0x_00000000": ["Number", eol], |
| "0x0000_0000_0000": ["Number", eol], |
| "0x000_0000_0000": ["Number", eol], |
| "0x00_0000_0000": ["Number", eol], |
| "0x0_0000_0000": ["Number", eol], |
| "0x_0000_0000_0000": ["Number", eol], |
| "0x_000_0000_0000": ["Number", eol], |
| "0x_00_0000_0000": ["Number", eol], |
| "0x_0_0000_0000": ["Number", eol], |
| "0x00000000_00000000_00000000": ["Number", eol], |
| "0x0000000_00000000_00000000": ["Number", eol], |
| "0x000000_00000000_00000000": ["Number", eol], |
| "0x00000_00000000_00000000": ["Number", eol], |
| "0x0000_00000000_00000000": ["Number", eol], |
| "0x000_00000000_00000000": ["Number", eol], |
| "0x00_00000000_00000000": ["Number", eol], |
| "0x0_00000000_00000000": ["Number", eol], |
| "0x_00000000_00000000_00000000": ["Number", eol], |
| "0x_0000000_00000000_00000000": ["Number", eol], |
| "0x_000000_00000000_00000000": ["Number", eol], |
| "0x_00000_00000000_00000000": ["Number", eol], |
| "0x_0000_00000000_00000000": ["Number", eol], |
| "0x_000_00000000_00000000": ["Number", eol], |
| "0x_00_00000000_00000000": ["Number", eol], |
| "0x_0_00000000_00000000": ["Number", eol], |
| "0x__00000000_00000000": ["BadNumber", eol], |
| "0x00000000_00000000_0000": ["BadNumber", eol], |
| "0x00000000_0000_0000": ["BadNumber", eol], |
| "0x_00000000000000000000": ["BadNumber", eol], |
| "0b_00000000000000000000": ["BadNumber", eol], |
| "0b00000000_00000000_0000": ["BadNumber", eol], |
| "0b00000000_0000_0000": ["BadNumber", eol], |
| "0x0000_": ["BadNumber", eol], |
| "0x00_____00": ["BadNumber", eol], |
| "0x____0____": ["BadNumber", eol], |
| "EmbossReserved": ["BadWord", eol], |
| "EmbossReservedA": ["BadWord", eol], |
| "EmbossReserved_": ["BadWord", eol], |
| "EMBOSS_RESERVED": ["BadWord", eol], |
| "EMBOSS_RESERVED_": ["BadWord", eol], |
| "EMBOSS_RESERVEDA": ["BadWord", eol], |
| "emboss_reserved": ["BadWord", eol], |
| "emboss_reserved_": ["BadWord", eol], |
| "emboss_reserveda": ["BadWord", eol], |
| "0x0123456789abcdefABCDEF": ["Number", eol], |
| "0": ["Number", eol], |
| "1": ["Number", eol], |
| "1a": ["BadNumber", eol], |
| "1g": ["BadWord", eol], |
| "1234567890": ["Number", eol], |
| "1_234_567_890": ["Number", eol], |
| "234_567_890": ["Number", eol], |
| "34_567_890": ["Number", eol], |
| "4_567_890": ["Number", eol], |
| "1_2_3_4_5_6_7_8_9_0": ["BadNumber", eol], |
| "1234567890_": ["BadNumber", eol], |
| "1__234567890": ["BadNumber", eol], |
| "_1234567890": ["BadWord", eol], |
| "[]": ['"["', '"]"', eol], |
| "()": ['"("', '")"', eol], |
| "..": ['"."', '"."', eol], |
| "...": ['"."', '"."', '"."', eol], |
| "....": ['"."', '"."', '"."', '"."', eol], |
| '"abc"': ["String", eol], |
| '""': ["String", eol], |
| r'"\\"': ["String", eol], |
| r'"\""': ["String", eol], |
| r'"\n"': ["String", eol], |
| r'"\\n"': ["String", eol], |
| r'"\\xyz"': ["String", eol], |
| r'"\\\\"': ["String", eol], |
| } |
| for c in ("[ ] ( ) ? : = + - * . == != < <= > >= && || , $max $present " |
| "$upper_bound $lower_bound $size_in_bits $size_in_bytes " |
| "$max_size_in_bits $max_size_in_bytes $min_size_in_bits " |
| "$min_size_in_bytes " |
| "$default struct bits enum external import as if let").split(): |
| cases[c] = ['"' + c + '"', eol] |
| |
| def make_test_case(case): |
| |
| def test_case(self): |
| tokens, errors = tokenizer.tokenize(case, "name") |
| symbols = _token_symbols(tokens) |
| self.assertFalse(errors) |
| self.assertEqual(symbols, cases[case]) |
| |
| return test_case |
| |
| for c in cases: |
| setattr(TokenizerTest, "testShortTokenMatch{!r}".format(c), |
| make_test_case(c)) |
| |
| |
| def _make_bad_char_tests(): |
| """Makes tests that an error is returned for bad characters.""" |
| |
| def make_test_case(case): |
| |
| def test_case(self): |
| tokens, errors = tokenizer.tokenize(case, "name") |
| self.assertFalse(tokens) |
| self.assertEqual([[error.error("name", parser_types.make_location( |
| (1, 1), (1, 2)), "Unrecognized token")]], errors) |
| |
| return test_case |
| |
| for c in "~`!@%^&\\|;'\"/{}": |
| setattr(TokenizerTest, "testBadChar{!r}".format(c), make_test_case(c)) |
| |
| |
| def _make_bad_string_tests(): |
| """Makes tests that an error is returned for bad strings.""" |
| bad_strings = (r'"\"', '"\\\n"', r'"\\\"', r'"', r'"\q"', r'"\\\q"') |
| |
| def make_test_case(string): |
| |
| def test_case(self): |
| tokens, errors = tokenizer.tokenize(string, "name") |
| self.assertFalse(tokens) |
| self.assertEqual([[error.error("name", parser_types.make_location( |
| (1, 1), (1, 2)), "Unrecognized token")]], errors) |
| |
| return test_case |
| |
| for s in bad_strings: |
| setattr(TokenizerTest, "testBadString{!r}".format(s), make_test_case(s)) |
| |
| |
| def _make_multiline_tests(): |
| """Makes tests for indent/dedent insertion and eol insertion.""" |
| |
| c = "Comment" |
| eol = '"\\n"' |
| sw = "SnakeWord" |
| ind = "Indent" |
| ded = "Dedent" |
| cases = { |
| "a\nb\n": [sw, eol, sw, eol], |
| "a\n\nb\n": [sw, eol, eol, sw, eol], |
| "a\n#foo\nb\n": [sw, eol, c, eol, sw, eol], |
| "a\n #foo\nb\n": [sw, eol, c, eol, sw, eol], |
| "a\n b\n": [sw, eol, ind, sw, eol, ded], |
| "a\n b\n\n": [sw, eol, ind, sw, eol, eol, ded], |
| "a\n b\n c\n": [sw, eol, ind, sw, eol, ind, sw, eol, ded, ded], |
| "a\n b\n c\n": [sw, eol, ind, sw, eol, sw, eol, ded], |
| "a\n b\n\n c\n": [sw, eol, ind, sw, eol, eol, sw, eol, ded], |
| "a\n b\n #\n c\n": [sw, eol, ind, sw, eol, c, eol, sw, eol, ded], |
| "a\n\tb\n #\n\tc\n": [sw, eol, ind, sw, eol, c, eol, sw, eol, ded], |
| " a\n b\n c\n d\n": [ind, sw, eol, ind, sw, eol, ind, sw, eol, ded, |
| ded, sw, eol, ded], |
| } |
| |
| def make_test_case(case): |
| |
| def test_case(self): |
| tokens, errors = tokenizer.tokenize(case, "file") |
| self.assertFalse(errors) |
| self.assertEqual(_token_symbols(tokens), cases[case]) |
| |
| return test_case |
| |
| for c in cases: |
| setattr(TokenizerTest, "testMultiline{!r}".format(c), make_test_case(c)) |
| |
| |
| def _make_offset_tests(): |
| """Makes tests that the tokenizer fills in correct source locations.""" |
| cases = { |
| "a+": ["1:1-1:2", "1:2-1:3", "1:3-1:3"], |
| "a + ": ["1:1-1:2", "1:5-1:6", "1:9-1:9"], |
| "a\n\nb": ["1:1-1:2", "1:2-1:2", "2:1-2:1", "3:1-3:2", "3:2-3:2"], |
| "a\n b": ["1:1-1:2", "1:2-1:2", "2:1-2:3", "2:3-2:4", "2:4-2:4", |
| "3:1-3:1"], |
| "a\n b\nc": ["1:1-1:2", "1:2-1:2", "2:1-2:3", "2:3-2:4", "2:4-2:4", |
| "3:1-3:1", "3:1-3:2", "3:2-3:2"], |
| "a\n b\n c": ["1:1-1:2", "1:2-1:2", "2:1-2:2", "2:2-2:3", "2:3-2:3", |
| "3:2-3:3", "3:3-3:4", "3:4-3:4", "4:1-4:1", "4:1-4:1"], |
| } |
| |
| def make_test_case(case): |
| |
| def test_case(self): |
| self.assertEqual([parser_types.format_location(l.source_location) |
| for l in tokenizer.tokenize(case, "file")[0]], |
| cases[case]) |
| |
| return test_case |
| |
| for c in cases: |
| setattr(TokenizerTest, "testOffset{!r}".format(c), make_test_case(c)) |
| |
| _make_short_token_match_tests() |
| _make_bad_char_tests() |
| _make_bad_string_tests() |
| _make_multiline_tests() |
| _make_offset_tests() |
| |
| if __name__ == "__main__": |
| unittest.main() |
