| //===--- Token.h - Token interface ------------------------------*- C++ -*-===// |
| // |
| // This source file is part of the Swift.org open source project |
| // |
| // Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors |
| // Licensed under Apache License v2.0 with Runtime Library Exception |
| // |
| // See https://swift.org/LICENSE.txt for license information |
| // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors |
| // |
| //===----------------------------------------------------------------------===// |
| // |
| // This file defines the Token interface. |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #ifndef SWIFT_TOKEN_H |
| #define SWIFT_TOKEN_H |
| |
| #include "swift/Basic/SourceLoc.h" |
| #include "swift/Basic/LLVM.h" |
| #include "swift/Syntax/TokenKinds.h" |
| #include "swift/Config.h" |
| #include "llvm/ADT/StringRef.h" |
| |
| namespace swift { |
| |
| /// Token - This structure provides full information about a lexed token. |
| /// It is not intended to be space efficient, it is intended to return as much |
| /// information as possible about each returned token. This is expected to be |
| /// compressed into a smaller form if memory footprint is important. |
| /// |
| class Token { |
| /// Kind - The actual flavor of token this is. |
| /// |
| tok Kind; |
| |
| /// \brief Whether this token is the first token on the line. |
| unsigned AtStartOfLine : 1; |
| |
| /// \brief The length of the comment that precedes the token. |
| /// |
| /// Hopefully 128 Mib is enough. |
| unsigned CommentLength : 27; |
| |
| /// \brief Whether this token is an escaped `identifier` token. |
| unsigned EscapedIdentifier : 1; |
| |
| /// Modifiers for string literals |
| unsigned MultilineString : 1; |
| |
| /// Text - The actual string covered by the token in the source buffer. |
| StringRef Text; |
| |
| StringRef trimComment() const { |
| assert(hasComment() && "Has no comment to trim."); |
| StringRef Raw(Text.begin() - CommentLength, CommentLength); |
| return Raw.trim(); |
| } |
| |
| public: |
| Token() : Kind(tok::NUM_TOKENS), AtStartOfLine(false), CommentLength(0), |
| EscapedIdentifier(false) {} |
| |
| Token(tok Kind, StringRef Text, unsigned CommentLength) |
| : Kind(Kind), AtStartOfLine(false), CommentLength(CommentLength), |
| EscapedIdentifier(false), MultilineString(false), |
| Text(Text) {} |
| |
| Token(tok Kind, StringRef Text): Token(Kind, Text, 0) {}; |
| |
| tok getKind() const { return Kind; } |
| void setKind(tok K) { Kind = K; } |
| void clearCommentLength() { CommentLength = 0; } |
| |
| /// is/isNot - Predicates to check if this token is a specific kind, as in |
| /// "if (Tok.is(tok::l_brace)) {...}". |
| bool is(tok K) const { return Kind == K; } |
| bool isNot(tok K) const { return Kind != K; } |
| |
| // Predicates to check to see if the token is any of a list of tokens. |
| |
| bool isAny(tok K1) const { |
| return is(K1); |
| } |
| template <typename ...T> |
| bool isAny(tok K1, tok K2, T... K) const { |
| if (is(K1)) |
| return true; |
| return isAny(K2, K...); |
| } |
| |
| // Predicates to check to see if the token is not the same as any of a list. |
| template <typename ...T> |
| bool isNot(tok K1, T... K) const { return !isAny(K1, K...); } |
| |
| bool isBinaryOperator() const { |
| return Kind == tok::oper_binary_spaced || Kind == tok::oper_binary_unspaced; |
| } |
| |
| bool isAnyOperator() const { |
| return isBinaryOperator() || Kind == tok::oper_postfix || |
| Kind == tok::oper_prefix; |
| } |
| bool isNotAnyOperator() const { |
| return !isAnyOperator(); |
| } |
| |
| bool isEllipsis() const { |
| return isAnyOperator() && Text == "..."; |
| } |
| bool isNotEllipsis() const { |
| return !isEllipsis(); |
| } |
| |
| /// \brief Determine whether this token occurred at the start of a line. |
| bool isAtStartOfLine() const { return AtStartOfLine; } |
| |
| /// \brief Set whether this token occurred at the start of a line. |
| void setAtStartOfLine(bool value) { AtStartOfLine = value; } |
| |
| /// \brief True if this token is an escaped identifier token. |
| bool isEscapedIdentifier() const { return EscapedIdentifier; } |
| /// \brief Set whether this token is an escaped identifier token. |
| void setEscapedIdentifier(bool value) { |
| assert((!value || Kind == tok::identifier) && |
| "only identifiers can be escaped identifiers"); |
| EscapedIdentifier = value; |
| } |
| |
| bool isContextualKeyword(StringRef ContextKW) const { |
| return is(tok::identifier) && !isEscapedIdentifier() && |
| Text == ContextKW; |
| } |
| |
| /// Return true if this is a contextual keyword that could be the start of a |
| /// decl. |
| bool isContextualDeclKeyword() const { |
| if (isNot(tok::identifier) || isEscapedIdentifier() || Text.empty()) |
| return false; |
| |
| switch (Text[0]) { |
| case 'c': |
| return Text == "convenience"; |
| case 'd': |
| return Text == "dynamic"; |
| case 'f': |
| return Text == "final"; |
| case 'i': |
| return Text == "indirect" || Text == "infix"; |
| case 'l': |
| return Text == "lazy"; |
| case 'm': |
| return Text == "mutating"; |
| case 'n': |
| return Text == "nonmutating"; |
| case 'o': |
| return Text == "open" || Text == "override" || Text == "optional"; |
| case 'p': |
| return Text == "prefix" || Text == "postfix"; |
| case 'r': |
| return Text == "required"; |
| case 'u': |
| return Text == "unowned"; |
| case 'w': |
| return Text == "weak"; |
| default: |
| return false; |
| } |
| } |
| |
| bool isContextualPunctuator(StringRef ContextPunc) const { |
| return isAnyOperator() && Text == ContextPunc; |
| } |
| |
| /// Determine whether the token can be an argument label. |
| /// |
| /// This covers all identifiers and keywords except those keywords |
| /// used |
| bool canBeArgumentLabel() const { |
| // Identifiers, escaped identifiers, and '_' can be argument labels. |
| if (is(tok::identifier) || isEscapedIdentifier() || is(tok::kw__)) { |
| // ... except for '__shared' and '__owned'. |
| if (getRawText().equals("__shared") || |
| getRawText().equals("__owned")) |
| return false; |
| |
| return true; |
| } |
| |
| // 'let', 'var', and 'inout' cannot be argument labels. |
| if (isAny(tok::kw_let, tok::kw_var, tok::kw_inout)) |
| return false; |
| |
| // All other keywords can be argument labels. |
| return isKeyword(); |
| } |
| |
| /// True if the token is an identifier or '_'. |
| bool isIdentifierOrUnderscore() const { |
| return isAny(tok::identifier, tok::kw__); |
| } |
| |
| /// True if the token is an l_paren token that does not start a new line. |
| bool isFollowingLParen() const { |
| return !isAtStartOfLine() && Kind == tok::l_paren; |
| } |
| |
| /// True if the token is an l_square token that does not start a new line. |
| bool isFollowingLSquare() const { |
| return !isAtStartOfLine() && Kind == tok::l_square; |
| } |
| |
| /// True if the token is any keyword. |
| bool isKeyword() const { |
| switch (Kind) { |
| #define KEYWORD(X) case tok::kw_##X: return true; |
| #include "swift/Syntax/TokenKinds.def" |
| default: return false; |
| } |
| } |
| |
| /// True if the token is any literal. |
| bool isLiteral() const { |
| switch(Kind) { |
| case tok::integer_literal: |
| case tok::floating_literal: |
| case tok::string_literal: |
| return true; |
| default: |
| return false; |
| } |
| } |
| |
| bool isPunctuation() const { |
| switch (Kind) { |
| #define PUNCTUATOR(Name, Str) case tok::Name: return true; |
| #include "swift/Syntax/TokenKinds.def" |
| default: return false; |
| } |
| } |
| |
| /// getLoc - Return a source location identifier for the specified |
| /// offset in the current file. |
| SourceLoc getLoc() const { |
| return SourceLoc(llvm::SMLoc::getFromPointer(Text.begin())); |
| } |
| |
| unsigned getLength() const { return Text.size(); } |
| |
| CharSourceRange getRange() const { |
| return CharSourceRange(getLoc(), getLength()); |
| } |
| |
| bool hasComment() const { |
| return CommentLength != 0; |
| } |
| |
| CharSourceRange getCommentRange() const { |
| if (CommentLength == 0) |
| return CharSourceRange(SourceLoc(llvm::SMLoc::getFromPointer(Text.begin())), |
| 0); |
| auto TrimedComment = trimComment(); |
| return CharSourceRange( |
| SourceLoc(llvm::SMLoc::getFromPointer(TrimedComment.begin())), |
| TrimedComment.size()); |
| } |
| |
| SourceLoc getCommentStart() const { |
| if (CommentLength == 0) return SourceLoc(); |
| return SourceLoc(llvm::SMLoc::getFromPointer(trimComment().begin())); |
| } |
| |
| StringRef getRawText() const { |
| return Text; |
| } |
| |
| StringRef getText() const { |
| if (EscapedIdentifier) { |
| // Strip off the backticks on either side. |
| assert(Text.front() == '`' && Text.back() == '`'); |
| return Text.slice(1, Text.size() - 1); |
| } |
| return Text; |
| } |
| |
| void setText(StringRef T) { Text = T; } |
| |
| /// \brief Set the token to the specified kind and source range. |
| void setToken(tok K, StringRef T, unsigned CommentLength = 0, |
| bool MultilineString = false) { |
| Kind = K; |
| Text = T; |
| this->CommentLength = CommentLength; |
| EscapedIdentifier = false; |
| this->MultilineString = MultilineString; |
| } |
| |
| bool IsMultilineString() const { |
| return MultilineString; |
| } |
| }; |
| |
| } // end namespace swift |
| |
| |
| namespace llvm { |
| template <typename T> struct isPodLike; |
| template <> |
| struct isPodLike<swift::Token> { static const bool value = true; }; |
| } // end namespace llvm |
| |
| #endif |