| from .Classification import classification_by_name |
| from .Node import error # noqa: I201 |
| from .kinds import lowercase_first_word # noqa: I201 |
| |
| |
| class Token(object): |
| """ |
| Represents the specification for a Token in the TokenSyntax file. |
| """ |
| |
| def __init__(self, name, kind, serialization_code, unprefixed_kind=None, |
| text=None, classification='None', is_keyword=False): |
| self.name = name |
| self.kind = kind |
| if unprefixed_kind is None: |
| self.unprefixed_kind = kind |
| else: |
| self.unprefixed_kind = unprefixed_kind |
| self.serialization_code = serialization_code |
| self.text = text or "" |
| self.classification = classification_by_name(classification) |
| self.is_keyword = is_keyword |
| |
| def swift_kind(self): |
| name = lowercase_first_word(self.name) |
| if self.is_keyword: |
| return name + 'Keyword' |
| return name |
| |
| |
| class Keyword(Token): |
| """ |
| Represents a keyword token. |
| """ |
| |
| def __init__(self, name, text, serialization_code, |
| classification='Keyword'): |
| Token.__init__(self, name, 'kw_' + text, serialization_code, |
| unprefixed_kind=text, text=text, |
| classification=classification, is_keyword=True) |
| |
| def macro_name(self): |
| return "KEYWORD" |
| |
| |
| class SwiftKeyword(Keyword): |
| def macro_name(self): |
| return "SWIFT_KEYWORD" |
| |
| |
| class DeclKeyword(SwiftKeyword): |
| def macro_name(self): |
| return "DECL_KEYWORD" |
| |
| |
| class StmtKeyword(SwiftKeyword): |
| def macro_name(self): |
| return "STMT_KEYWORD" |
| |
| |
| class ExprKeyword(SwiftKeyword): |
| def macro_name(self): |
| return "EXPR_KEYWORD" |
| |
| |
| class PatternKeyword(SwiftKeyword): |
| def macro_name(self): |
| return "PAT_KEYWORD" |
| |
| |
| class SilKeyword(Keyword): |
| def macro_name(self): |
| return "SIL_KEYWORD" |
| |
| |
| class PoundKeyword(Token): |
| def __init__(self, name, kind, text, serialization_code, |
| classification='Keyword'): |
| Token.__init__(self, name, 'pound_' + kind, serialization_code, |
| unprefixed_kind=kind, text=text, |
| classification=classification, is_keyword=True) |
| |
| def macro_name(self): |
| return "POUND_KEYWORD" |
| |
| |
| class PoundObjectLiteral(PoundKeyword): |
| def __init__(self, name, kind, text, serialization_code, description, |
| protocol, classification='ObjectLiteral'): |
| PoundKeyword.__init__(self, name, kind, text, serialization_code, |
| classification) |
| self.description = description |
| self.protocol = protocol |
| |
| def macro_name(self): |
| return "POUND_OBJECT_LITERAL" |
| |
| |
| class PoundConfig(PoundKeyword): |
| def macro_name(self): |
| return "POUND_CONFIG" |
| |
| |
| class PoundDirectiveKeyword(PoundKeyword): |
| def __init__(self, name, kind, text, serialization_code, |
| classification='PoundDirectiveKeyword'): |
| PoundKeyword.__init__(self, name, kind, text, serialization_code, |
| classification) |
| |
| def macro_name(self): |
| return "POUND_DIRECTIVE_KEYWORD" |
| |
| |
| class PoundConditionalDirectiveKeyword(PoundDirectiveKeyword): |
| def __init__(self, name, kind, text, serialization_code, |
| classification='PoundDirectiveKeyword'): |
| PoundKeyword.__init__(self, name, kind, text, serialization_code, |
| classification) |
| |
| def macro_name(self): |
| return "POUND_COND_DIRECTIVE_KEYWORD" |
| |
| |
| class Punctuator(Token): |
| def macro_name(self): |
| return "PUNCTUATOR" |
| |
| |
| class Literal(Token): |
| def macro_name(self): |
| return "LITERAL" |
| |
| |
| class Misc(Token): |
| def macro_name(self): |
| return "MISC" |
| |
| |
| SYNTAX_TOKENS = [ |
| # Keywords that start decls |
| DeclKeyword('Associatedtype', 'associatedtype', serialization_code=1), |
| DeclKeyword('Class', 'class', serialization_code=2), |
| DeclKeyword('Deinit', 'deinit', serialization_code=3), |
| DeclKeyword('Enum', 'enum', serialization_code=4), |
| DeclKeyword('Extension', 'extension', serialization_code=5), |
| DeclKeyword('Func', 'func', serialization_code=6), |
| DeclKeyword('Import', 'import', serialization_code=7), |
| DeclKeyword('Init', 'init', serialization_code=8), |
| DeclKeyword('Inout', 'inout', serialization_code=9), |
| DeclKeyword('Let', 'let', serialization_code=10), |
| DeclKeyword('Operator', 'operator', serialization_code=11), |
| DeclKeyword('Precedencegroup', 'precedencegroup', serialization_code=12), |
| DeclKeyword('Protocol', 'protocol', serialization_code=13), |
| DeclKeyword('Struct', 'struct', serialization_code=14), |
| DeclKeyword('Subscript', 'subscript', serialization_code=15), |
| DeclKeyword('Typealias', 'typealias', serialization_code=16), |
| DeclKeyword('Var', 'var', serialization_code=17), |
| |
| DeclKeyword('Fileprivate', 'fileprivate', serialization_code=18), |
| DeclKeyword('Internal', 'internal', serialization_code=19), |
| DeclKeyword('Private', 'private', serialization_code=20), |
| DeclKeyword('Public', 'public', serialization_code=21), |
| DeclKeyword('Static', 'static', serialization_code=22), |
| |
| # Statement keywords |
| StmtKeyword('Defer', 'defer', serialization_code=23), |
| StmtKeyword('If', 'if', serialization_code=24), |
| StmtKeyword('Guard', 'guard', serialization_code=25), |
| StmtKeyword('Do', 'do', serialization_code=26), |
| StmtKeyword('Repeat', 'repeat', serialization_code=27), |
| StmtKeyword('Else', 'else', serialization_code=28), |
| StmtKeyword('For', 'for', serialization_code=29), |
| StmtKeyword('In', 'in', serialization_code=30), |
| StmtKeyword('While', 'while', serialization_code=31), |
| StmtKeyword('Return', 'return', serialization_code=32), |
| StmtKeyword('Break', 'break', serialization_code=33), |
| StmtKeyword('Continue', 'continue', serialization_code=34), |
| StmtKeyword('Fallthrough', 'fallthrough', serialization_code=35), |
| StmtKeyword('Switch', 'switch', serialization_code=36), |
| StmtKeyword('Case', 'case', serialization_code=37), |
| StmtKeyword('Default', 'default', serialization_code=38), |
| StmtKeyword('Where', 'where', serialization_code=39), |
| StmtKeyword('Catch', 'catch', serialization_code=40), |
| StmtKeyword('Throw', 'throw', serialization_code=50), |
| |
| # Expression keywords |
| ExprKeyword('As', 'as', serialization_code=41), |
| ExprKeyword('Any', 'Any', serialization_code=42), |
| ExprKeyword('False', 'false', serialization_code=43), |
| ExprKeyword('Is', 'is', serialization_code=44), |
| ExprKeyword('Nil', 'nil', serialization_code=45), |
| ExprKeyword('Rethrows', 'rethrows', serialization_code=46), |
| ExprKeyword('Super', 'super', serialization_code=47), |
| ExprKeyword('Self', 'self', serialization_code=48), |
| ExprKeyword('CapitalSelf', 'Self', serialization_code=49), |
| ExprKeyword('True', 'true', serialization_code=51), |
| ExprKeyword('Try', 'try', serialization_code=52), |
| ExprKeyword('Throws', 'throws', serialization_code=53), |
| |
| Keyword('__FILE__', '__FILE__', serialization_code=54), |
| Keyword('__LINE__', '__LINE__', serialization_code=55), |
| Keyword('__COLUMN__', '__COLUMN__', serialization_code=56), |
| Keyword('__FUNCTION__', '__FUNCTION__', serialization_code=57), |
| Keyword('__DSO_HANDLE__', '__DSO_HANDLE__', serialization_code=58), |
| |
| # Pattern keywords |
| PatternKeyword('Wildcard', '_', serialization_code=59), |
| |
| # Punctuators |
| Punctuator('LeftParen', 'l_paren', text='(', serialization_code=88), |
| Punctuator('RightParen', 'r_paren', text=')', serialization_code=89), |
| Punctuator('LeftBrace', 'l_brace', text='{', serialization_code=90), |
| Punctuator('RightBrace', 'r_brace', text='}', serialization_code=91), |
| Punctuator('LeftSquareBracket', 'l_square', text='[', |
| serialization_code=92), |
| Punctuator('RightSquareBracket', 'r_square', text=']', |
| serialization_code=93), |
| Punctuator('LeftAngle', 'l_angle', text='<', serialization_code=94), |
| Punctuator('RightAngle', 'r_angle', text='>', serialization_code=95), |
| |
| Punctuator('Period', 'period', text='.', serialization_code=85), |
| Punctuator('PrefixPeriod', 'period_prefix', text='.', |
| serialization_code=87), |
| Punctuator('Comma', 'comma', text=',', serialization_code=84), |
| Punctuator('Ellipsis', 'ellipsis', text='...', serialization_code=118), |
| Punctuator('Colon', 'colon', text=':', serialization_code=82), |
| Punctuator('Semicolon', 'semi', text=';', serialization_code=83), |
| Punctuator('Equal', 'equal', text='=', serialization_code=86), |
| Punctuator('AtSign', 'at_sign', text='@', classification='Attribute', |
| serialization_code=80), |
| Punctuator('Pound', 'pound', text='#', serialization_code=81), |
| |
| Punctuator('PrefixAmpersand', 'amp_prefix', text='&', |
| serialization_code=96), |
| Punctuator('Arrow', 'arrow', text='->', serialization_code=78), |
| |
| |
| Punctuator('Backtick', 'backtick', text='`', serialization_code=79), |
| |
| Punctuator('Backslash', 'backslash', text='\\\\', serialization_code=100), |
| |
| Punctuator('ExclamationMark', 'exclaim_postfix', text='!', |
| serialization_code=99), |
| |
| Punctuator('PostfixQuestionMark', 'question_postfix', text='?', |
| serialization_code=97), |
| Punctuator('InfixQuestionMark', 'question_infix', text='?', |
| serialization_code=98), |
| |
| Punctuator('StringQuote', 'string_quote', text='\\\"', |
| classification='StringLiteral', serialization_code=102), |
| Punctuator('SingleQuote', 'single_quote', text='\\\'', |
| classification='StringLiteral', serialization_code=120), |
| Punctuator('MultilineStringQuote', 'multiline_string_quote', |
| text='\\\"\\\"\\\"', classification='StringLiteral', |
| serialization_code=103), |
| |
| # Keywords prefixed with a '#'. |
| |
| PoundKeyword('PoundKeyPath', 'keyPath', text='#keyPath', |
| serialization_code=74), |
| PoundKeyword('PoundLine', 'line', text='#line', |
| serialization_code=69), |
| PoundKeyword('PoundSelector', 'selector', text='#selector', |
| serialization_code=73), |
| PoundKeyword('PoundFile', 'file', text='#file', |
| serialization_code=68), |
| PoundKeyword('PoundFileID', 'fileID', text='#fileID', |
| serialization_code=122), |
| PoundKeyword('PoundFilePath', 'filePath', text='#filePath', |
| serialization_code=121), |
| PoundKeyword('PoundColumn', 'column', text='#column', |
| serialization_code=70), |
| PoundKeyword('PoundFunction', 'function', text='#function', |
| serialization_code=72), |
| PoundKeyword('PoundDsohandle', 'dsohandle', text='#dsohandle', |
| serialization_code=71), |
| PoundKeyword('PoundAssert', 'assert', text='#assert', |
| serialization_code=117), |
| |
| PoundDirectiveKeyword('PoundSourceLocation', 'sourceLocation', |
| text='#sourceLocation', serialization_code=65), |
| PoundDirectiveKeyword('PoundWarning', 'warning', text='#warning', |
| serialization_code=66), |
| PoundDirectiveKeyword('PoundError', 'error', text='#error', |
| serialization_code=67), |
| |
| PoundConditionalDirectiveKeyword('PoundIf', 'if', text='#if', |
| serialization_code=64), |
| PoundConditionalDirectiveKeyword('PoundElse', 'else', text='#else', |
| serialization_code=62), |
| PoundConditionalDirectiveKeyword('PoundElseif', 'elseif', |
| text='#elseif', serialization_code=63), |
| PoundConditionalDirectiveKeyword('PoundEndif', 'endif', |
| text='#endif', serialization_code=61), |
| |
| PoundConfig('PoundAvailable', 'available', text='#available', |
| serialization_code=60), |
| |
| PoundObjectLiteral('PoundFileLiteral', 'fileLiteral', |
| text='#fileLiteral', serialization_code=76, |
| description='file reference', |
| protocol='ExpressibleByFileReferenceLiteral'), |
| PoundObjectLiteral('PoundImageLiteral', 'imageLiteral', |
| text='#imageLiteral', serialization_code=77, |
| description='image', |
| protocol='ExpressibleByImageLiteral'), |
| PoundObjectLiteral('PoundColorLiteral', 'colorLiteral', |
| text='#colorLiteral', serialization_code=75, |
| description='color', |
| protocol='ExpressibleByColorLiteral'), |
| |
| Literal('IntegerLiteral', 'integer_literal', |
| classification='IntegerLiteral', serialization_code=111), |
| Literal('FloatingLiteral', 'floating_literal', |
| classification='FloatingLiteral', serialization_code=112), |
| Literal('StringLiteral', 'string_literal', |
| classification='StringLiteral', serialization_code=113), |
| |
| Misc('Unknown', 'unknown', serialization_code=115), |
| Misc('Identifier', 'identifier', classification='Identifier', |
| serialization_code=105), |
| Misc('UnspacedBinaryOperator', 'oper_binary_unspaced', |
| serialization_code=107), |
| Misc('SpacedBinaryOperator', 'oper_binary_spaced', serialization_code=108), |
| Misc('PostfixOperator', 'oper_postfix', serialization_code=110), |
| Misc('PrefixOperator', 'oper_prefix', serialization_code=109), |
| Misc('DollarIdentifier', 'dollarident', classification='DollarIdentifier', |
| serialization_code=106), |
| |
| Misc('ContextualKeyword', 'contextual_keyword', classification='Keyword', |
| serialization_code=114), |
| Misc('RawStringDelimiter', 'raw_string_delimiter', serialization_code=119), |
| Misc('StringSegment', 'string_segment', classification='StringLiteral', |
| serialization_code=104), |
| Misc('StringInterpolationAnchor', 'string_interpolation_anchor', |
| text=')', classification='StringInterpolationAnchor', |
| serialization_code=101), |
| Misc('Yield', 'kw_yield', serialization_code=116, text='yield'), |
| ] |
| |
| SYNTAX_TOKEN_MAP = {token.name + 'Token': token for token in SYNTAX_TOKENS} |
| |
| |
| def verify_no_duplicate_serialization_codes(tokens): |
| used_codes = set() |
| for token in tokens: |
| if token.serialization_code in used_codes: |
| error("Serialization code %d used twice for tokens" % |
| token.serialization_code) |
| used_codes.add(token.serialization_code) |
| |
| |
| verify_no_duplicate_serialization_codes(SYNTAX_TOKENS) |
