include/swift/Parse/Token.h

//===--- Token.h - Token interface ------------------------------*- C++ -*-===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
//  This file defines the Token interface.
//
//===----------------------------------------------------------------------===//

#ifndef SWIFT_TOKEN_H
#define SWIFT_TOKEN_H

#include "swift/Basic/SourceLoc.h"
#include "swift/Basic/LLVM.h"
#include "swift/Syntax/TokenKinds.h"
#include "swift/Config.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSwitch.h"

namespace swift {

/// Token - This structure provides full information about a lexed token.
/// It is not intended to be space efficient, it is intended to return as much
/// information as possible about each returned token.  This is expected to be
/// compressed into a smaller form if memory footprint is important.
///
class Token {
  /// Kind - The actual flavor of token this is.
  ///
  tok Kind;

  /// Whether this token is the first token on the line.
  unsigned AtStartOfLine : 1;

  /// Whether this token is an escaped `identifier` token.
  unsigned EscapedIdentifier : 1;
  
  /// Modifiers for string literals
  unsigned MultilineString : 1;

  /// Length of custom delimiter of "raw" string literals
  unsigned CustomDelimiterLen : 8;

  // Padding bits == 32 - 11;

  /// The length of the comment that precedes the token.
  unsigned CommentLength;

  /// Text - The actual string covered by the token in the source buffer.
  StringRef Text;

  StringRef trimComment() const {
    assert(hasComment() && "Has no comment to trim.");
    StringRef Raw(Text.begin() - CommentLength, CommentLength);
    return Raw.trim();
  }

public:
  Token(tok Kind, StringRef Text, unsigned CommentLength = 0)
          : Kind(Kind), AtStartOfLine(false), EscapedIdentifier(false),
            MultilineString(false), CustomDelimiterLen(0),
            CommentLength(CommentLength), Text(Text) {}

  Token() : Token(tok::NUM_TOKENS, {}, 0) {}

  tok getKind() const { return Kind; }
  void setKind(tok K) { Kind = K; }
  void clearCommentLength() { CommentLength = 0; }
  
  /// is/isNot - Predicates to check if this token is a specific kind, as in
  /// "if (Tok.is(tok::l_brace)) {...}".
  bool is(tok K) const { return Kind == K; }
  bool isNot(tok K) const { return Kind != K; }

  // Predicates to check to see if the token is any of a list of tokens.

  bool isAny(tok K1) const {
    return is(K1);
  }
  template <typename ...T>
  bool isAny(tok K1, tok K2, T... K) const {
    if (is(K1))
      return true;
    return isAny(K2, K...);
  }

  // Predicates to check to see if the token is not the same as any of a list.
  template <typename ...T>
  bool isNot(tok K1, T... K) const { return !isAny(K1, K...); }

  bool isBinaryOperator() const {
    return Kind == tok::oper_binary_spaced || Kind == tok::oper_binary_unspaced;
  }
  
  bool isAnyOperator() const {
    return isBinaryOperator() || Kind == tok::oper_postfix ||
           Kind == tok::oper_prefix;
  }
  bool isNotAnyOperator() const {
    return !isAnyOperator();
  }

  bool isEllipsis() const {
    return isAnyOperator() && Text == "...";
  }
  bool isNotEllipsis() const {
    return !isEllipsis();
  }

  /// Determine whether this token occurred at the start of a line.
  bool isAtStartOfLine() const { return AtStartOfLine; }

  /// Set whether this token occurred at the start of a line.
  void setAtStartOfLine(bool value) { AtStartOfLine = value; }
  
  /// True if this token is an escaped identifier token.
  bool isEscapedIdentifier() const { return EscapedIdentifier; }
  /// Set whether this token is an escaped identifier token.
  void setEscapedIdentifier(bool value) {
    assert((!value || Kind == tok::identifier) &&
           "only identifiers can be escaped identifiers");
    EscapedIdentifier = value;
  }
  
  bool isContextualKeyword(StringRef ContextKW) const {
    return is(tok::identifier) && !isEscapedIdentifier() &&
           Text == ContextKW;
  }
  
  /// Return true if this is a contextual keyword that could be the start of a
  /// decl.
  bool isContextualDeclKeyword() const {
    if (isNot(tok::identifier) || isEscapedIdentifier() || Text.empty())
      return false;

    return llvm::StringSwitch<bool>(Text)
#define CONTEXTUAL_CASE(KW) .Case(#KW, true)
#define CONTEXTUAL_DECL_ATTR(KW, ...) CONTEXTUAL_CASE(KW)
#define CONTEXTUAL_DECL_ATTR_ALIAS(KW, ...) CONTEXTUAL_CASE(KW)
#define CONTEXTUAL_SIMPLE_DECL_ATTR(KW, ...) CONTEXTUAL_CASE(KW)
#include "swift/AST/Attr.def"
#undef CONTEXTUAL_CASE
      .Default(false);
  }

  bool isContextualPunctuator(StringRef ContextPunc) const {
    return isAnyOperator() && Text == ContextPunc;
  }

  /// Determine whether the token can be an argument label.
  ///
  /// This covers all identifiers and keywords except those keywords
  /// used
  bool canBeArgumentLabel() const {
    // Identifiers, escaped identifiers, and '_' can be argument labels.
    if (is(tok::identifier) || isEscapedIdentifier() || is(tok::kw__)) {
      // ... except for '__shared' and '__owned'.
      if (getRawText().equals("__shared") ||
          getRawText().equals("__owned"))
        return false;

      return true;
    }

    // 'let', 'var', and 'inout' cannot be argument labels.
    if (isAny(tok::kw_let, tok::kw_var, tok::kw_inout))
      return false;

    // All other keywords can be argument labels.
    return isKeyword();
  }

  /// True if the token is an identifier or '_'.
  bool isIdentifierOrUnderscore() const {
    return isAny(tok::identifier, tok::kw__);
  }

  /// True if the token is an l_paren token that does not start a new line.
  bool isFollowingLParen() const {
    return !isAtStartOfLine() && Kind == tok::l_paren;
  }
  
  /// True if the token is an l_square token that does not start a new line.
  bool isFollowingLSquare() const {
    return !isAtStartOfLine() && Kind == tok::l_square;
  }

  /// True if the token is any keyword.
  bool isKeyword() const {
    switch (Kind) {
#define KEYWORD(X) case tok::kw_##X: return true;
#include "swift/Syntax/TokenKinds.def"
    default: return false;
    }
  }

  /// True if the token is any literal.
  bool isLiteral() const {
    switch(Kind) {
    case tok::integer_literal:
    case tok::floating_literal:
    case tok::string_literal:
      return true;
    default:
      return false;
    }
  }

  bool isPunctuation() const {
    switch (Kind) {
#define PUNCTUATOR(Name, Str) case tok::Name: return true;
#include "swift/Syntax/TokenKinds.def"
    default: return false;
    }
  }

  /// True if the string literal token is multiline.
  bool isMultilineString() const {
    return MultilineString;
  }
  /// Count of extending escaping '#'.
  unsigned getCustomDelimiterLen() const {
    return CustomDelimiterLen;
  }
  /// Set characteristics of string literal token.
  void setStringLiteral(bool IsMultilineString, unsigned CustomDelimiterLen) {
    assert(Kind == tok::string_literal);
    this->MultilineString = IsMultilineString;
    this->CustomDelimiterLen = CustomDelimiterLen;
  }
  
  /// getLoc - Return a source location identifier for the specified
  /// offset in the current file.
  SourceLoc getLoc() const {
    return SourceLoc(llvm::SMLoc::getFromPointer(Text.begin()));
  }

  unsigned getLength() const { return Text.size(); }

  CharSourceRange getRange() const {
    return CharSourceRange(getLoc(), getLength());
  }

  CharSourceRange getRangeWithoutBackticks() const {
    SourceLoc TokLoc = getLoc();
    unsigned TokLength = getLength();
    if (isEscapedIdentifier()) {
      // Adjust to account for the backticks.
      TokLoc = TokLoc.getAdvancedLoc(1);
      TokLength -= 2;
    }
    return CharSourceRange(TokLoc, TokLength);
  }

  bool hasComment() const {
    return CommentLength != 0;
  }

  CharSourceRange getCommentRange() const {
    if (CommentLength == 0)
      return CharSourceRange(SourceLoc(llvm::SMLoc::getFromPointer(Text.begin())),
                             0);
    auto TrimedComment = trimComment();
    return CharSourceRange(
      SourceLoc(llvm::SMLoc::getFromPointer(TrimedComment.begin())),
      TrimedComment.size());
  }
  
  SourceLoc getCommentStart() const {
    if (CommentLength == 0) return SourceLoc();
    return SourceLoc(llvm::SMLoc::getFromPointer(trimComment().begin()));
  }

  StringRef getRawText() const {
    return Text;
  }

  StringRef getText() const {
    if (EscapedIdentifier) {
      // Strip off the backticks on either side.
      assert(Text.front() == '`' && Text.back() == '`');
      return Text.slice(1, Text.size() - 1);
    }
    return Text;
  }

  void setText(StringRef T) { Text = T; }

  /// Set the token to the specified kind and source range.
  void setToken(tok K, StringRef T, unsigned CommentLength = 0) {
    Kind = K;
    Text = T;
    this->CommentLength = CommentLength;
    EscapedIdentifier = false;
    this->MultilineString = false;
    this->CustomDelimiterLen = 0;
    assert(this->CustomDelimiterLen == CustomDelimiterLen &&
           "custom string delimiter length > 255");
  }
};
  
} // end namespace swift


namespace llvm {
  template <typename T> struct isPodLike;
  template <>
  struct isPodLike<swift::Token> { static const bool value = true; };
} // end namespace llvm

#endif