Google Git
Sign in
fuchsia / third_party / swift / eaefd7194696865f32ca2938abc83f5dfbc9a66f / . / lib / Parse / Parser.cpp
blob: 2c07e46dda99d29c21bc68f516a9c9fc573cb1ec [file] [log] [blame]
//===--- Parser.cpp - Swift Language Parser -------------------------------===//
//
// This source file is part of the Swift.org open source project
//
// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
// Licensed under Apache License v2.0 with Runtime Library Exception
//
// See https://swift.org/LICENSE.txt for license information
// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
//
//===----------------------------------------------------------------------===//
//
// This file implements the Swift parser.
//
//===----------------------------------------------------------------------===//
#include "swift/Parse/Parser.h"
#include "swift/Subsystems.h"
#include "swift/AST/ASTWalker.h"
#include "swift/AST/DiagnosticsParse.h"
#include "swift/AST/Module.h"
#include "swift/AST/PrettyStackTrace.h"
#include "swift/AST/SourceFile.h"
#include "swift/Basic/Defer.h"
#include "swift/Basic/SourceManager.h"
#include "swift/Basic/Timer.h"
#include "swift/Parse/Lexer.h"
#include "swift/Parse/CodeCompletionCallbacks.h"
#include "swift/Parse/ParsedSyntaxNodes.h"
#include "swift/Parse/ParsedSyntaxRecorder.h"
#include "swift/Parse/ParseSILSupport.h"
#include "swift/Parse/SyntaxParseActions.h"
#include "swift/Parse/SyntaxParsingContext.h"
#include "swift/Parse/HiddenLibSyntaxAction.h"
#include "swift/Syntax/RawSyntax.h"
#include "swift/Syntax/TokenSyntax.h"
#include "swift/SyntaxParse/SyntaxTreeCreator.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/SaveAndRestore.h"
#include "llvm/ADT/PointerUnion.h"
#include "llvm/ADT/Twine.h"
static void getStringPartTokens(const swift::Token &Tok,
const swift::LangOptions &LangOpts,
const swift::SourceManager &SM, int BufID,
std::vector<swift::Token> &Toks);
namespace swift {
template <typename DF>
void tokenize(const LangOptions &LangOpts, const SourceManager &SM,
unsigned BufferID, unsigned Offset, unsigned EndOffset,
DiagnosticEngine * Diags,
CommentRetentionMode RetainComments,
TriviaRetentionMode TriviaRetention,
bool TokenizeInterpolatedString, ArrayRef<Token> SplitTokens,
DF &&DestFunc) {
assert((TriviaRetention != TriviaRetentionMode::WithTrivia ||
!TokenizeInterpolatedString) &&
"string interpolation with trivia is not implemented yet");
if (Offset == 0 && EndOffset == 0)
EndOffset = SM.getRangeForBuffer(BufferID).getByteLength();
Lexer L(LangOpts, SM, BufferID, Diags, LexerMode::Swift,
HashbangMode::Allowed, RetainComments, TriviaRetention, Offset,
EndOffset);
auto TokComp = [&](const Token &A, const Token &B) {
return SM.isBeforeInBuffer(A.getLoc(), B.getLoc());
};
std::set<Token, decltype(TokComp)> ResetTokens(TokComp);
for (auto C = SplitTokens.begin(), E = SplitTokens.end(); C != E; ++C) {
ResetTokens.insert(*C);
}
Token Tok;
ParsedTrivia LeadingTrivia, TrailingTrivia;
do {
L.lex(Tok, LeadingTrivia, TrailingTrivia);
// If the token has the same location as a reset location,
// reset the token stream
auto F = ResetTokens.find(Tok);
if (F != ResetTokens.end()) {
assert(F->isNot(tok::string_literal));
DestFunc(*F, ParsedTrivia(), ParsedTrivia());
auto NewState = L.getStateForBeginningOfTokenLoc(
F->getLoc().getAdvancedLoc(F->getLength()));
L.restoreState(NewState);
continue;
}
if (Tok.is(tok::string_literal) && TokenizeInterpolatedString) {
std::vector<Token> StrTokens;
getStringPartTokens(Tok, LangOpts, SM, BufferID, StrTokens);
for (auto &StrTok : StrTokens) {
DestFunc(StrTok, ParsedTrivia(), ParsedTrivia());
}
} else {
DestFunc(Tok, LeadingTrivia, TrailingTrivia);
}
} while (Tok.getKind() != tok::eof);
}
} // namespace swift
using namespace swift;
using namespace swift::syntax;
void SILParserTUStateBase::anchor() { }
namespace {
/// A visitor that does delayed parsing of function bodies.
class ParseDelayedFunctionBodies : public ASTWalker {
PersistentParserState &ParserState;
CodeCompletionCallbacksFactory *CodeCompletionFactory;
public:
ParseDelayedFunctionBodies(PersistentParserState &ParserState,
CodeCompletionCallbacksFactory *Factory)
: ParserState(ParserState), CodeCompletionFactory(Factory) {}
bool walkToDeclPre(Decl *D) override {
if (auto AFD = dyn_cast<AbstractFunctionDecl>(D)) {
if (AFD->getBodyKind() != FuncDecl::BodyKind::Unparsed)
return false;
parseFunctionBody(AFD);
return true;
}
return true;
}
private:
void parseFunctionBody(AbstractFunctionDecl *AFD) {
// FIXME: This duplicates the evaluation of
// ParseAbstractFunctionBodyRequest, but installs a code completion
// factory.
assert(AFD->getBodyKind() == FuncDecl::BodyKind::Unparsed);
SourceFile &SF = *AFD->getDeclContext()->getParentSourceFile();
SourceManager &SourceMgr = SF.getASTContext().SourceMgr;
unsigned BufferID = SourceMgr.findBufferContainingLoc(AFD->getLoc());
Parser TheParser(BufferID, SF, nullptr, &ParserState, nullptr,
/*DelayBodyParsing=*/false);
TheParser.SyntaxContext->setDiscard();
std::unique_ptr<CodeCompletionCallbacks> CodeCompletion;
if (CodeCompletionFactory) {
CodeCompletion.reset(
CodeCompletionFactory->createCodeCompletionCallbacks(TheParser));
TheParser.setCodeCompletionCallbacks(CodeCompletion.get());
}
auto body = TheParser.parseAbstractFunctionBodyDelayed(AFD);
AFD->setBodyParsed(body);
if (CodeCompletion)
CodeCompletion->doneParsing();
}
};
static void parseDelayedDecl(
PersistentParserState &ParserState,
CodeCompletionCallbacksFactory *CodeCompletionFactory) {
if (!ParserState.hasDelayedDecl())
return;
SourceFile &SF = *ParserState.getDelayedDeclContext()->getParentSourceFile();
SourceManager &SourceMgr = SF.getASTContext().SourceMgr;
unsigned BufferID =
SourceMgr.findBufferContainingLoc(ParserState.getDelayedDeclLoc());
Parser TheParser(BufferID, SF, nullptr, &ParserState, nullptr);
TheParser.SyntaxContext->setDiscard();
std::unique_ptr<CodeCompletionCallbacks> CodeCompletion;
if (CodeCompletionFactory) {
CodeCompletion.reset(
CodeCompletionFactory->createCodeCompletionCallbacks(TheParser));
TheParser.setCodeCompletionCallbacks(CodeCompletion.get());
}
switch (ParserState.getDelayedDeclKind()) {
case PersistentParserState::DelayedDeclKind::TopLevelCodeDecl:
TheParser.parseTopLevelCodeDeclDelayed();
break;
case PersistentParserState::DelayedDeclKind::Decl:
TheParser.parseDeclDelayed();
break;
}
if (CodeCompletion)
CodeCompletion->doneParsing();
}
} // unnamed namespace
swift::Parser::BacktrackingScope::~BacktrackingScope() {
if (Backtrack) {
P.backtrackToPosition(PP);
DT.abort();
}
}
void swift::performDelayedParsing(
DeclContext *DC, PersistentParserState &PersistentState,
CodeCompletionCallbacksFactory *CodeCompletionFactory) {
SharedTimer timer("Parsing");
ParseDelayedFunctionBodies Walker(PersistentState,
CodeCompletionFactory);
DC->walkContext(Walker);
if (CodeCompletionFactory)
parseDelayedDecl(PersistentState, CodeCompletionFactory);
}
/// Tokenizes a string literal, taking into account string interpolation.
static void getStringPartTokens(const Token &Tok, const LangOptions &LangOpts,
const SourceManager &SM,
int BufID, std::vector<Token> &Toks) {
assert(Tok.is(tok::string_literal));
bool IsMultiline = Tok.isMultilineString();
unsigned CustomDelimiterLen = Tok.getCustomDelimiterLen();
unsigned QuoteLen = (IsMultiline ? 3 : 1) + CustomDelimiterLen;
SmallVector<Lexer::StringSegment, 4> Segments;
Lexer::getStringLiteralSegments(Tok, Segments, /*Diags=*/nullptr);
for (unsigned i = 0, e = Segments.size(); i != e; ++i) {
Lexer::StringSegment &Seg = Segments[i];
bool isFirst = i == 0;
bool isLast = i == e-1;
if (Seg.Kind == Lexer::StringSegment::Literal) {
SourceLoc Loc = Seg.Loc;
unsigned Len = Seg.Length;
if (isFirst) {
// Include the quote.
Loc = Loc.getAdvancedLoc(-QuoteLen);
Len += QuoteLen;
}
if (isLast) {
// Include the quote.
Len += QuoteLen;
}
StringRef Text = SM.extractText({ Loc, Len });
Token NewTok;
NewTok.setToken(tok::string_literal, Text);
NewTok.setStringLiteral(IsMultiline, CustomDelimiterLen);
Toks.push_back(NewTok);
} else {
assert(Seg.Kind == Lexer::StringSegment::Expr &&
"new enumerator was introduced ?");
unsigned Offset = SM.getLocOffsetInBuffer(Seg.Loc, BufID);
unsigned EndOffset = Offset + Seg.Length;
if (isFirst) {
// Add a token for the quote character.
StringRef Text = SM.extractText({ Seg.Loc.getAdvancedLoc(-2), 1 });
Token NewTok;
NewTok.setToken(tok::string_literal, Text);
Toks.push_back(NewTok);
}
std::vector<Token> NewTokens = swift::tokenize(LangOpts, SM, BufID,
Offset, EndOffset,
/*Diags=*/nullptr,
/*KeepComments=*/true);
Toks.insert(Toks.end(), NewTokens.begin(), NewTokens.end());
if (isLast) {
// Add a token for the quote character.
StringRef Text = SM.extractText({ Seg.Loc.getAdvancedLoc(Seg.Length),
1 });
Token NewTok;
NewTok.setToken(tok::string_literal, Text);
Toks.push_back(NewTok);
}
}
}
}
std::vector<Token> swift::tokenize(const LangOptions &LangOpts,
const SourceManager &SM, unsigned BufferID,
unsigned Offset, unsigned EndOffset,
DiagnosticEngine *Diags,
bool KeepComments,
bool TokenizeInterpolatedString,
ArrayRef<Token> SplitTokens) {
std::vector<Token> Tokens;
tokenize(LangOpts, SM, BufferID, Offset, EndOffset,
Diags,
KeepComments ? CommentRetentionMode::ReturnAsTokens
: CommentRetentionMode::AttachToNextToken,
TriviaRetentionMode::WithoutTrivia, TokenizeInterpolatedString,
SplitTokens,
[&](const Token &Tok, const ParsedTrivia &LeadingTrivia,
const ParsedTrivia &TrailingTrivia) { Tokens.push_back(Tok); });
assert(Tokens.back().is(tok::eof));
Tokens.pop_back(); // Remove EOF.
return Tokens;
}
std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsolutePosition>>
swift::tokenizeWithTrivia(const LangOptions &LangOpts, const SourceManager &SM,
unsigned BufferID, unsigned Offset,
unsigned EndOffset,
DiagnosticEngine *Diags) {
std::vector<std::pair<RC<syntax::RawSyntax>, syntax::AbsolutePosition>>
Tokens;
syntax::AbsolutePosition RunningPos;
tokenize(
LangOpts, SM, BufferID, Offset, EndOffset, Diags,
CommentRetentionMode::AttachToNextToken, TriviaRetentionMode::WithTrivia,
/*TokenizeInterpolatedString=*/false,
/*SplitTokens=*/ArrayRef<Token>(),
[&](const Token &Tok, const ParsedTrivia &LeadingTrivia,
const ParsedTrivia &TrailingTrivia) {
CharSourceRange TokRange = Tok.getRange();
SourceLoc LeadingTriviaLoc =
TokRange.getStart().getAdvancedLoc(-LeadingTrivia.getLength());
SourceLoc TrailingTriviaLoc =
TokRange.getStart().getAdvancedLoc(TokRange.getByteLength());
Trivia syntaxLeadingTrivia =
LeadingTrivia.convertToSyntaxTrivia(LeadingTriviaLoc, SM, BufferID);
Trivia syntaxTrailingTrivia =
TrailingTrivia.convertToSyntaxTrivia(TrailingTriviaLoc, SM, BufferID);
auto Text = OwnedString::makeRefCounted(Tok.getRawText());
auto ThisToken =
RawSyntax::make(Tok.getKind(), Text, syntaxLeadingTrivia.Pieces,
syntaxTrailingTrivia.Pieces,
SourcePresence::Present);
auto ThisTokenPos = ThisToken->accumulateAbsolutePosition(RunningPos);
Tokens.push_back({ThisToken, ThisTokenPos.getValue()});
});
return Tokens;
}
//===----------------------------------------------------------------------===//
// Setup and Helper Methods
//===----------------------------------------------------------------------===//
static LexerMode sourceFileKindToLexerMode(SourceFileKind kind) {
switch (kind) {
case swift::SourceFileKind::Interface:
return LexerMode::SwiftInterface;
case swift::SourceFileKind::SIL:
return LexerMode::SIL;
case swift::SourceFileKind::Library:
case swift::SourceFileKind::Main:
case swift::SourceFileKind::REPL:
return LexerMode::Swift;
}
llvm_unreachable("covered switch");
}
Parser::Parser(unsigned BufferID, SourceFile &SF, SILParserTUStateBase *SIL,
PersistentParserState *PersistentState,
std::shared_ptr<SyntaxParseActions> SPActions,
bool DelayBodyParsing)
: Parser(BufferID, SF, &SF.getASTContext().Diags, SIL, PersistentState,
std::move(SPActions), DelayBodyParsing) {}
Parser::Parser(unsigned BufferID, SourceFile &SF, DiagnosticEngine* LexerDiags,
SILParserTUStateBase *SIL,
PersistentParserState *PersistentState,
std::shared_ptr<SyntaxParseActions> SPActions,
bool DelayBodyParsing)
: Parser(
std::unique_ptr<Lexer>(new Lexer(
SF.getASTContext().LangOpts, SF.getASTContext().SourceMgr,
BufferID, LexerDiags,
sourceFileKindToLexerMode(SF.Kind),
SF.Kind == SourceFileKind::Main
? HashbangMode::Allowed
: HashbangMode::Disallowed,
SF.getASTContext().LangOpts.AttachCommentsToDecls
? CommentRetentionMode::AttachToNextToken
: CommentRetentionMode::None,
TriviaRetentionMode::WithTrivia)),
SF, SIL, PersistentState, std::move(SPActions), DelayBodyParsing) {}
namespace {
/// This is the token receiver that helps SourceFile to keep track of its
/// underlying corrected token stream.
class TokenRecorder: public ConsumeTokenReceiver {
ASTContext &Ctx;
SourceManager &SM;
// Token list ordered by their appearance in the source file.
std::vector<Token> &Bag;
unsigned BufferID;
// Registered token kind change. These changes are regiestered before the
// token is consumed, so we need to keep track of them here.
llvm::DenseMap<const void*, tok> TokenKindChangeMap;
std::vector<Token>::iterator lower_bound(SourceLoc Loc) {
return token_lower_bound(Bag, Loc);
}
std::vector<Token>::iterator lower_bound(Token Tok) {
return lower_bound(Tok.getLoc());
}
void relexComment(CharSourceRange CommentRange,
llvm::SmallVectorImpl<Token> &Scratch) {
Lexer L(Ctx.LangOpts, Ctx.SourceMgr, BufferID, nullptr, LexerMode::Swift,
HashbangMode::Disallowed,
CommentRetentionMode::ReturnAsTokens,
TriviaRetentionMode::WithoutTrivia,
SM.getLocOffsetInBuffer(CommentRange.getStart(), BufferID),
SM.getLocOffsetInBuffer(CommentRange.getEnd(), BufferID));
while(true) {
Token Result;
L.lex(Result);
if (Result.is(tok::eof))
break;
assert(Result.is(tok::comment));
Scratch.push_back(Result);
}
}
public:
TokenRecorder(SourceFile &SF):
Ctx(SF.getASTContext()),
SM(SF.getASTContext().SourceMgr),
Bag(SF.getTokenVector()),
BufferID(SF.getBufferID().getValue()) {};
void finalize() override {
// We should consume the comments at the end of the file that don't attach
// to any tokens.
SourceLoc TokEndLoc;
if (!Bag.empty()) {
Token Last = Bag.back();
TokEndLoc = Last.getLoc().getAdvancedLoc(Last.getLength());
} else {
// Special case: the file contains nothing but comments.
TokEndLoc = SM.getLocForBufferStart(BufferID);
}
llvm::SmallVector<Token, 4> Scratch;
relexComment(CharSourceRange(SM, TokEndLoc,
SM.getRangeForBuffer(BufferID).getEnd()),
Scratch);
// Accept these orphaned comments.
Bag.insert(Bag.end(), Scratch.begin(), Scratch.end());
}
void registerTokenKindChange(SourceLoc Loc, tok NewKind) override {
// If a token with the same location is already in the bag, update its kind.
auto Pos = lower_bound(Loc);
if (Pos != Bag.end() && Pos->getLoc().getOpaquePointerValue() ==
Loc.getOpaquePointerValue()) {
Pos->setKind(NewKind);
return;
}
// Save the update for later.
TokenKindChangeMap[Loc.getOpaquePointerValue()] = NewKind;
}
void receive(Token Tok) override {
// We filter out all tokens without valid location
if(Tok.getLoc().isInvalid())
return;
// If a token with the same location is already in the bag, skip this token.
auto Pos = lower_bound(Tok);
if (Pos != Bag.end() && Pos->getLoc().getOpaquePointerValue() ==
Tok.getLoc().getOpaquePointerValue()) {
return;
}
// Update Token kind if a kind update was regiestered before.
auto Found = TokenKindChangeMap.find(Tok.getLoc().
getOpaquePointerValue());
if (Found != TokenKindChangeMap.end()) {
Tok.setKind(Found->getSecond());
}
// If the token has comment attached to it, re-lexing these comments and
// consume them as separate tokens.
llvm::SmallVector<Token, 4> TokensToConsume;
if (Tok.hasComment()) {
relexComment(Tok.getCommentRange(), TokensToConsume);
}
TokensToConsume.push_back(Tok);
Bag.insert(Pos, TokensToConsume.begin(), TokensToConsume.end());
}
};
} // End of an anonymous namespace.
Parser::Parser(std::unique_ptr<Lexer> Lex, SourceFile &SF,
SILParserTUStateBase *SIL,
PersistentParserState *PersistentState,
std::shared_ptr<SyntaxParseActions> SPActions,
bool DelayBodyParsing)
: SourceMgr(SF.getASTContext().SourceMgr),
Diags(SF.getASTContext().Diags),
SF(SF),
L(Lex.release()),
SIL(SIL),
CurDeclContext(&SF),
Context(SF.getASTContext()),
DelayBodyParsing(DelayBodyParsing),
TokReceiver(SF.shouldCollectToken() ?
new TokenRecorder(SF) :
new ConsumeTokenReceiver()),
SyntaxContext(new SyntaxParsingContext(
SyntaxContext, SF, L->getBufferID(),
std::make_shared<HiddenLibSyntaxAction>(
SPActions,
std::make_shared<SyntaxTreeCreator>(
SF.getASTContext().SourceMgr,
L->getBufferID(),
SF.SyntaxParsingCache,
SF.getASTContext().getSyntaxArena())))),
Generator(SF.getASTContext(), *this) {
State = PersistentState;
if (!State) {
OwnedState.reset(new PersistentParserState());
State = OwnedState.get();
}
// Set the token to a sentinel so that we know the lexer isn't primed yet.
// This cannot be tok::unknown, since that is a token the lexer could produce.
Tok.setKind(tok::NUM_TOKENS);
auto ParserPos = State->takeParserPosition();
if (ParserPos.isValid() &&
L->isStateForCurrentBuffer(ParserPos.LS)) {
restoreParserPosition(ParserPos);
InPoundLineEnvironment = State->InPoundLineEnvironment;
}
}
Parser::~Parser() {
delete L;
delete TokReceiver;
delete SyntaxContext;
}
bool Parser::allowTopLevelCode() const {
return SF.isScriptMode();
}
const Token &Parser::peekToken() {
return L->peekNextToken();
}
SourceLoc Parser::consumeTokenWithoutFeedingReceiver() {
SourceLoc Loc = Tok.getLoc();
assert(Tok.isNot(tok::eof) && "Lexing past eof!");
if (IsParsingInterfaceTokens && !Tok.getText().empty()) {
SF.recordInterfaceToken(Tok.getText());
}
L->lex(Tok, LeadingTrivia, TrailingTrivia);
PreviousLoc = Loc;
return Loc;
}
void Parser::consumeExtraToken(Token Extra) {
TokReceiver->receive(Extra);
}
SourceLoc Parser::consumeToken() {
TokReceiver->receive(Tok);
SyntaxContext->addToken(Tok, LeadingTrivia, TrailingTrivia);
return consumeTokenWithoutFeedingReceiver();
}
ParsedTokenSyntax Parser::consumeTokenSyntax() {
TokReceiver->receive(Tok);
ParsedTokenSyntax ParsedToken = ParsedSyntaxRecorder::makeToken(
Tok, LeadingTrivia, TrailingTrivia, *SyntaxContext);
consumeTokenWithoutFeedingReceiver();
return ParsedToken;
}
SourceLoc Parser::getEndOfPreviousLoc() const {
return Lexer::getLocForEndOfToken(SourceMgr, PreviousLoc);
}
ParsedTokenSyntax
Parser::consumeStartingCharacterOfCurrentTokenSyntax(tok Kind, size_t Len) {
// Consumes prefix of token and returns its location.
// (like '?', '<', '>' or '!' immediately followed by '<')
assert(Len >= 1);
// Current token can be either one-character token we want to consume...
if (Tok.getLength() == Len) {
Tok.setKind(Kind);
return consumeTokenSyntax();
}
auto Loc = Tok.getLoc();
// ... or a multi-character token with the first N characters being the one
// that we want to consume as a separate token.
assert(Tok.getLength() > Len);
auto Token = markSplitTokenSyntax(Kind, Tok.getText().substr(0, Len));
auto NewState = L->getStateForBeginningOfTokenLoc(Loc.getAdvancedLoc(Len));
restoreParserPosition(ParserPosition(NewState, Loc),
/*enableDiagnostics=*/true);
return Token;
}
SourceLoc Parser::consumeStartingCharacterOfCurrentToken(tok Kind, size_t Len) {
// Consumes prefix of token and returns its location.
// (like '?', '<', '>' or '!' immediately followed by '<')
assert(Len >= 1);
// Current token can be either one-character token we want to consume...
if (Tok.getLength() == Len) {
Tok.setKind(Kind);
return consumeToken();
}
auto Loc = Tok.getLoc();
// ... or a multi-character token with the first N characters being the one
// that we want to consume as a separate token.
assert(Tok.getLength() > Len);
markSplitToken(Kind, Tok.getText().substr(0, Len));
auto NewState = L->getStateForBeginningOfTokenLoc(Loc.getAdvancedLoc(Len));
restoreParserPosition(ParserPosition(NewState, Loc),
/*enableDiagnostics=*/true);
return PreviousLoc;
}
ParsedTokenSyntax Parser::markSplitTokenSyntax(tok Kind, StringRef Txt) {
SplitTokens.emplace_back();
SplitTokens.back().setToken(Kind, Txt);
TokReceiver->receive(SplitTokens.back());
return ParsedSyntaxRecorder::makeToken(SplitTokens.back(), LeadingTrivia,
ParsedTrivia(), *SyntaxContext);
}
void Parser::markSplitToken(tok Kind, StringRef Txt) {
SplitTokens.emplace_back();
SplitTokens.back().setToken(Kind, Txt);
ParsedTrivia EmptyTrivia;
SyntaxContext->addToken(SplitTokens.back(), LeadingTrivia, EmptyTrivia);
TokReceiver->receive(SplitTokens.back());
}
ParsedTokenSyntax Parser::consumeStartingLessSyntax() {
assert(startsWithLess(Tok) && "Token does not start with '<'");
return consumeStartingCharacterOfCurrentTokenSyntax(tok::l_angle);
}
SourceLoc Parser::consumeStartingLess() {
assert(startsWithLess(Tok) && "Token does not start with '<'");
return consumeStartingCharacterOfCurrentToken(tok::l_angle);
}
ParsedTokenSyntax Parser::consumeStartingGreaterSyntax() {
assert(startsWithGreater(Tok) && "Token does not start with '>'");
return consumeStartingCharacterOfCurrentTokenSyntax(tok::r_angle);
}
SourceLoc Parser::consumeStartingGreater() {
assert(startsWithGreater(Tok) && "Token does not start with '>'");
return consumeStartingCharacterOfCurrentToken(tok::r_angle);
}
void Parser::skipSingleSyntax(SmallVectorImpl<ParsedSyntax> &Skipped) {
switch (Tok.getKind()) {
case tok::l_paren:
Skipped.push_back(consumeTokenSyntax());
skipUntilSyntax(Skipped, tok::r_paren);
if (auto RParen = consumeTokenSyntaxIf(tok::r_paren))
Skipped.push_back(std::move(*RParen));
break;
case tok::l_brace:
Skipped.push_back(consumeTokenSyntax());
skipUntilSyntax(Skipped, tok::r_brace);
if (auto RBrace = consumeTokenSyntaxIf(tok::r_brace))
Skipped.push_back(std::move(*RBrace));
break;
case tok::l_square:
Skipped.push_back(consumeTokenSyntax());
skipUntilSyntax(Skipped, tok::r_square);
if (auto RSquare = consumeTokenSyntaxIf(tok::r_square))
Skipped.push_back(std::move(*RSquare));
break;
case tok::pound_if:
case tok::pound_else:
case tok::pound_elseif:
Skipped.push_back(consumeTokenSyntax());
// skipUntil also implicitly stops at tok::pound_endif.
skipUntilSyntax(Skipped, tok::pound_else, tok::pound_elseif);
if (Tok.isAny(tok::pound_else, tok::pound_elseif))
skipSingleSyntax(Skipped);
else if (auto Endif = consumeTokenSyntaxIf(tok::pound_endif))
Skipped.push_back(std::move(*Endif));
break;
default:
Skipped.push_back(consumeTokenSyntax());
break;
}
}
void Parser::skipSingle() {
switch (Tok.getKind()) {
case tok::l_paren:
consumeToken();
skipUntil(tok::r_paren);
consumeIf(tok::r_paren);
break;
case tok::l_brace:
consumeToken();
skipUntil(tok::r_brace);
consumeIf(tok::r_brace);
break;
case tok::l_square:
consumeToken();
skipUntil(tok::r_square);
consumeIf(tok::r_square);
break;
case tok::pound_if:
case tok::pound_else:
case tok::pound_elseif:
consumeToken();
// skipUntil also implicitly stops at tok::pound_endif.
skipUntil(tok::pound_else, tok::pound_elseif);
if (Tok.isAny(tok::pound_else, tok::pound_elseif))
skipSingle();
else
consumeIf(tok::pound_endif);
break;
default:
consumeToken();
break;
}
}
void Parser::ignoreToken() {
ParsedTriviaList Skipped;
Skipped.reserve(LeadingTrivia.size() + TrailingTrivia.size() + 1 + 2);
std::move(LeadingTrivia.begin(), LeadingTrivia.end(),
std::back_inserter(Skipped));
Skipped.emplace_back(TriviaKind::GarbageText, Tok.getText().size());
std::move(TrailingTrivia.begin(), TrailingTrivia.end(),
std::back_inserter(Skipped));
TokReceiver->receive(Tok);
consumeTokenWithoutFeedingReceiver();
std::move(LeadingTrivia.begin(), LeadingTrivia.end(),
std::back_inserter(Skipped));
LeadingTrivia = {std::move(Skipped)};
}
void Parser::ignoreSingle() {
switch (Tok.getKind()) {
case tok::l_paren:
ignoreToken();
ignoreUntil(tok::r_paren);
ignoreIf(tok::r_paren);
break;
case tok::l_brace:
ignoreToken();
ignoreUntil(tok::r_brace);
ignoreIf(tok::r_brace);
break;
case tok::l_square:
ignoreToken();
ignoreUntil(tok::r_square);
ignoreIf(tok::r_square);
break;
case tok::pound_if:
case tok::pound_else:
case tok::pound_elseif:
ignoreToken();
ignoreUntil(tok::pound_endif);
ignoreIf(tok::pound_endif);
break;
default:
ignoreToken();
break;
}
}
void Parser::ignoreUntil(tok Kind) {
while (Tok.isNot(Kind, tok::eof, tok::pound_endif, tok::code_complete))
ignoreSingle();
}
void Parser::skipUntilSyntax(llvm::SmallVectorImpl<ParsedSyntax> &Skipped,
tok T1, tok T2) {
// tok::NUM_TOKENS is a sentinel that means "don't skip".
if (T1 == tok::NUM_TOKENS && T2 == tok::NUM_TOKENS) return;
while (Tok.isNot(T1, T2, tok::eof, tok::pound_endif, tok::code_complete))
skipSingleSyntax(Skipped);
}
void Parser::skipUntil(tok T1, tok T2) {
// tok::NUM_TOKENS is a sentinel that means "don't skip".
if (T1 == tok::NUM_TOKENS && T2 == tok::NUM_TOKENS) return;
while (Tok.isNot(T1, T2, tok::eof, tok::pound_endif, tok::code_complete))
skipSingle();
}
void Parser::skipUntilAnyOperator() {
while (Tok.isNot(tok::eof, tok::pound_endif, tok::code_complete) &&
Tok.isNotAnyOperator())
skipSingle();
}
bool Parser::ignoreUntilGreaterInTypeList() {
while (true) {
switch (Tok.getKind()) {
case tok::eof:
case tok::l_brace:
case tok::r_brace:
case tok::code_complete:
return false;
#define KEYWORD(X) case tok::kw_##X:
#define POUND_KEYWORD(X) case tok::pound_##X:
#include "swift/Syntax/TokenKinds.def"
if (isStartOfStmt() || isStartOfDecl() || Tok.is(tok::pound_endif))
return false;
break;
default:
if (startsWithGreater(Tok))
return true;
break;
}
ignoreSingle();
}
}
void Parser::skipUntilGreaterInTypeListSyntax(
SmallVectorImpl<ParsedSyntax> &Skipped, bool protocolComposition) {
while (true) {
switch (Tok.getKind()) {
case tok::eof:
case tok::l_brace:
case tok::r_brace:
case tok::code_complete:
return;
#define KEYWORD(X) case tok::kw_##X:
#define POUND_KEYWORD(X) case tok::pound_##X:
#include "swift/Syntax/TokenKinds.def"
// 'Self' can appear in types, skip it.
if (Tok.is(tok::kw_Self))
break;
if (isStartOfStmt() || isStartOfDecl() || Tok.is(tok::pound_endif))
return;
break;
case tok::l_paren:
case tok::r_paren:
case tok::l_square:
case tok::r_square:
// In generic type parameter list, skip '[' ']' '(' ')', because they
// can appear in types.
if (protocolComposition)
return;
break;
default:
if (Tok.isAnyOperator() && startsWithGreater(Tok)) {
Skipped.push_back(consumeStartingGreaterSyntax());
return;
}
break;
}
skipSingleSyntax(Skipped);
}
}
/// Skip until a token that starts with '>', and consume it if found.
/// Applies heuristics that are suitable when trying to find the end of a list
/// of generic parameters, generic arguments, or list of types in a protocol
/// composition.
SourceLoc Parser::skipUntilGreaterInTypeList(bool protocolComposition) {
SourceLoc lastLoc = PreviousLoc;
while (true) {
switch (Tok.getKind()) {
case tok::eof:
case tok::l_brace:
case tok::r_brace:
case tok::code_complete:
return lastLoc;
#define KEYWORD(X) case tok::kw_##X:
#define POUND_KEYWORD(X) case tok::pound_##X:
#include "swift/Syntax/TokenKinds.def"
// 'Self' can appear in types, skip it.
if (Tok.is(tok::kw_Self))
break;
if (isStartOfStmt() || isStartOfDecl() || Tok.is(tok::pound_endif))
return lastLoc;
break;
case tok::l_paren:
case tok::r_paren:
case tok::l_square:
case tok::r_square:
// In generic type parameter list, skip '[' ']' '(' ')', because they
// can appear in types.
if (protocolComposition)
return lastLoc;
break;
default:
if (Tok.isAnyOperator() && startsWithGreater(Tok))
return consumeStartingGreater();
break;
}
skipSingle();
lastLoc = PreviousLoc;
}
}
void Parser::skipUntilDeclRBrace() {
while (Tok.isNot(tok::eof, tok::r_brace, tok::pound_endif,
tok::pound_else, tok::pound_elseif,
tok::code_complete) &&
!isStartOfDecl())
skipSingle();
}
void Parser::skipUntilDeclStmtRBrace(tok T1) {
while (Tok.isNot(T1, tok::eof, tok::r_brace, tok::pound_endif,
tok::pound_else, tok::pound_elseif,
tok::code_complete) &&
!isStartOfStmt() && !isStartOfDecl()) {
skipSingle();
}
}
void Parser::skipUntilDeclStmtRBrace(tok T1, tok T2) {
while (Tok.isNot(T1, T2, tok::eof, tok::r_brace, tok::pound_endif,
tok::pound_else, tok::pound_elseif,
tok::code_complete) &&
!isStartOfStmt() && !isStartOfDecl()) {
skipSingle();
}
}
void Parser::skipListUntilDeclRBrace(SourceLoc startLoc, tok T1, tok T2) {
while (Tok.isNot(T1, T2, tok::eof, tok::r_brace, tok::pound_endif,
tok::pound_else, tok::pound_elseif)) {
bool hasDelimiter = Tok.getLoc() == startLoc || consumeIf(tok::comma);
bool possibleDeclStartsLine = Tok.isAtStartOfLine();
if (isStartOfDecl()) {
// Could have encountered something like `_ var:`
// or `let foo:` or `var:`
if (Tok.isAny(tok::kw_var, tok::kw_let)) {
if (possibleDeclStartsLine && !hasDelimiter) {
break;
}
Parser::BacktrackingScope backtrack(*this);
// Consume the let or var
consumeToken();
// If the following token is either <identifier> or :, it means that
// this `var` or `let` shoud be interpreted as a label
if ((Tok.canBeArgumentLabel() && peekToken().is(tok::colon)) ||
peekToken().is(tok::colon)) {
backtrack.cancelBacktrack();
continue;
}
}
break;
}
skipSingle();
}
}
void Parser::skipListUntilDeclRBraceSyntax(
SmallVectorImpl<ParsedSyntax> &Skipped, SourceLoc startLoc, tok T1, tok T2) {
while (Tok.isNot(T1, T2, tok::eof, tok::r_brace, tok::pound_endif,
tok::pound_else, tok::pound_elseif)) {
auto Comma = consumeTokenSyntaxIf(tok::comma);
bool hasDelimiter = Tok.getLoc() == startLoc || Comma;
bool possibleDeclStartsLine = Tok.isAtStartOfLine();
if (Comma)
Skipped.push_back(std::move(*Comma));
if (isStartOfDecl()) {
// Could have encountered something like `_ var:`
// or `let foo:` or `var:`
if (Tok.isAny(tok::kw_var, tok::kw_let)) {
if (possibleDeclStartsLine && !hasDelimiter) {
break;
}
Parser::BacktrackingScope backtrack(*this);
// Consume the let or var
auto LetOrVar = consumeTokenSyntax();
Skipped.push_back(std::move(LetOrVar));
// If the following token is either <identifier> or :, it means that
// this `var` or `let` should be interpreted as a label
if ((Tok.canBeArgumentLabel() && peekToken().is(tok::colon)) ||
peekToken().is(tok::colon)) {
backtrack.cancelBacktrack();
continue;
}
}
break;
}
skipSingleSyntax(Skipped);
}
}
void Parser::skipUntilDeclRBrace(tok T1, tok T2) {
while (Tok.isNot(T1, T2, tok::eof, tok::r_brace, tok::pound_endif,
tok::pound_else, tok::pound_elseif) &&
!isStartOfDecl()) {
skipSingle();
}
}
void Parser::skipUntilConditionalBlockClose() {
while (Tok.isNot(tok::pound_else, tok::pound_elseif, tok::pound_endif,
tok::eof)) {
skipSingle();
}
}
bool Parser::skipUntilTokenOrEndOfLine(tok T1) {
while (Tok.isNot(tok::eof, T1) && !Tok.isAtStartOfLine())
skipSingle();
return Tok.is(T1) && !Tok.isAtStartOfLine();
}
bool Parser::loadCurrentSyntaxNodeFromCache() {
// Don't do a cache lookup when not building a syntax tree since otherwise
// the corresponding AST nodes do not get created
if (!SF.shouldBuildSyntaxTree()) {
return false;
}
unsigned LexerOffset =
SourceMgr.getLocOffsetInBuffer(Tok.getLoc(), L->getBufferID());
unsigned LeadingTriviaLen = LeadingTrivia.getLength();
unsigned LeadingTriviaOffset = LexerOffset - LeadingTriviaLen;
SourceLoc LeadingTriviaLoc = Tok.getLoc().getAdvancedLoc(-LeadingTriviaLen);
if (auto TextLength = SyntaxContext->lookupNode(LeadingTriviaOffset,
LeadingTriviaLoc)) {
L->resetToOffset(LeadingTriviaOffset + TextLength);
L->lex(Tok, LeadingTrivia, TrailingTrivia);
return true;
}
return false;
}
bool Parser::parseEndIfDirective(SourceLoc &Loc) {
Loc = Tok.getLoc();
if (parseToken(tok::pound_endif, diag::expected_close_to_if_directive)) {
Loc = PreviousLoc;
skipUntilConditionalBlockClose();
return true;
} else if (!Tok.isAtStartOfLine() && Tok.isNot(tok::eof))
diagnose(Tok.getLoc(),
diag::extra_tokens_conditional_compilation_directive);
return false;
}
static Parser::StructureMarkerKind
getStructureMarkerKindForToken(const Token &tok) {
switch (tok.getKind()) {
case tok::l_brace:
return Parser::StructureMarkerKind::OpenBrace;
case tok::l_paren:
return Parser::StructureMarkerKind::OpenParen;
case tok::l_square:
return Parser::StructureMarkerKind::OpenSquare;
default:
llvm_unreachable("Not a matched token");
}
}
Parser::StructureMarkerRAII::StructureMarkerRAII(Parser &parser,
const Token &tok)
: StructureMarkerRAII(parser, tok.getLoc(),
getStructureMarkerKindForToken(tok)) {}
bool Parser::StructureMarkerRAII::pushStructureMarker(
Parser &parser, SourceLoc loc,
StructureMarkerKind kind) {
if (parser.StructureMarkers.size() < MaxDepth) {
parser.StructureMarkers.push_back({loc, kind, None});
return true;
} else {
parser.diagnose(loc, diag::structure_overflow, MaxDepth);
// We need to cut off parsing or we will stack-overflow.
// But `cutOffParsing` changes the current token to eof, and we may be in
// a place where `consumeToken()` will be expecting e.g. '[',
// since we need that to get to the callsite, so this can cause an assert.
parser.cutOffParsing();
return false;
}
}
//===----------------------------------------------------------------------===//
// Primitive Parsing
//===----------------------------------------------------------------------===//
Optional<ParsedTokenSyntax> Parser::parseIdentifierSyntax(const Diagnostic &D) {
switch (Tok.getKind()) {
case tok::kw_self:
case tok::kw_Self:
case tok::identifier:
return consumeIdentifierSyntax();
default:
checkForInputIncomplete();
diagnose(Tok, D);
return None;
}
}
bool Parser::parseIdentifier(Identifier &Result, SourceLoc &Loc,
const Diagnostic &D) {
switch (Tok.getKind()) {
case tok::kw_self:
case tok::kw_Self:
case tok::identifier:
Loc = consumeIdentifier(&Result);
return false;
default:
checkForInputIncomplete();
diagnose(Tok, D);
return true;
}
}
bool Parser::parseSpecificIdentifier(StringRef expected, SourceLoc &loc,
const Diagnostic &D) {
if (Tok.getText() != expected) {
diagnose(Tok, D);
return true;
}
loc = consumeToken(tok::identifier);
return false;
}
/// parseAnyIdentifier - Consume an identifier or operator if present and return
/// its name in Result. Otherwise, emit an error and return true.
bool Parser::parseAnyIdentifier(Identifier &Result, SourceLoc &Loc,
const Diagnostic &D) {
if (Tok.is(tok::identifier)) {
Loc = consumeIdentifier(&Result);
return false;
}
if (Tok.isAnyOperator()) {
Result = Context.getIdentifier(Tok.getText());
Loc = Tok.getLoc();
consumeToken();
return false;
}
// When we know we're supposed to get an identifier or operator, map the
// postfix '!' to an operator name.
if (Tok.is(tok::exclaim_postfix)) {
Result = Context.getIdentifier(Tok.getText());
Loc = Tok.getLoc();
consumeToken(tok::exclaim_postfix);
return false;
}
checkForInputIncomplete();
if (Tok.isKeyword()) {
diagnose(Tok, diag::keyword_cant_be_identifier, Tok.getText());
diagnose(Tok, diag::backticks_to_escape)
.fixItReplace(Tok.getLoc(), "`" + Tok.getText().str() + "`");
} else {
diagnose(Tok, D);
}
return true;
}
/// parseToken - The parser expects that 'K' is next in the input. If so, it is
/// consumed and false is returned.
///
/// If the input is malformed, this emits the specified error diagnostic.
bool Parser::parseToken(tok K, SourceLoc &TokLoc, const Diagnostic &D) {
if (Tok.is(K)) {
TokLoc = consumeToken(K);
return false;
}
checkForInputIncomplete();
diagnose(Tok, D);
return true;
}
bool Parser::parseMatchingToken(tok K, SourceLoc &TokLoc, Diag<> ErrorDiag,
SourceLoc OtherLoc) {
Diag<> OtherNote;
switch (K) {
case tok::r_paren: OtherNote = diag::opening_paren; break;
case tok::r_square: OtherNote = diag::opening_bracket; break;
case tok::r_brace: OtherNote = diag::opening_brace; break;
default: llvm_unreachable("unknown matching token!"); break;
}
if (parseToken(K, TokLoc, ErrorDiag)) {
diagnose(OtherLoc, OtherNote);
TokLoc = getLocForMissingMatchingToken();
return true;
}
return false;
}
// SWIFT_ENABLE_TENSORFLOW
bool Parser::parseUnsignedInteger(unsigned &Result, SourceLoc &Loc,
const Diagnostic &D) {
auto IntTok = Tok;
if (parseToken(tok::integer_literal, Loc, D))
return true;
if (IntTok.getText().getAsInteger(0, Result)) {
diagnose(IntTok.getLoc(), D);
return true;
}
return false;
}
Optional<ParsedTokenSyntax> Parser::parseTokenSyntax(tok K, SourceLoc &TokLoc,
const Diagnostic &D) {
if (Tok.is(K)) {
TokLoc = Tok.getLoc();
return consumeTokenSyntax();
}
checkForInputIncomplete();
diagnose(Tok, D);
return None;
}
ParsedSyntaxResult<ParsedTokenSyntax>
Parser::parseMatchingTokenSyntax(tok K, Diag<> ErrorDiag, SourceLoc OtherLoc,
bool silenceDiag) {
if (Tok.is(K))
return makeParsedResult(consumeTokenSyntax(K));
checkForInputIncomplete();
if (!silenceDiag) {
diagnose(Tok, ErrorDiag);
Diag<> OtherNote;
switch (K) {
case tok::r_paren: OtherNote = diag::opening_paren; break;
case tok::r_square: OtherNote = diag::opening_bracket; break;
case tok::r_brace: OtherNote = diag::opening_brace; break;
default: llvm_unreachable("unknown matching token!");
}
diagnose(OtherLoc, OtherNote);
}
return makeParserError();
}
SourceLoc Parser::getLocForMissingMatchingToken() const {
// At present, use the same location whether it's an error or whether
// the matching token is missing.
// Both cases supply a location for something the user didn't type.
return getErrorOrMissingLoc();
}
SourceLoc Parser::getErrorOrMissingLoc() const {
// The next token might start a new enclosing construct,
// and SourceLoc's are always at the start of a token (for example, for
// fixits, so use the previous token's SourceLoc and allow a subnode to end
// right at the same place as its supernode.
// The tricky case is when the previous token is an InterpolatedStringLiteral.
// Then, there will be names in scope whose SourceLoc is *after* the
// the location of a missing close brace.
// ASTScope tree creation will have to cope.
return PreviousLoc;
}
static SyntaxKind getListElementKind(SyntaxKind ListKind) {
switch (ListKind) {
case SyntaxKind::TupleExprElementList:
return SyntaxKind::TupleExprElement;
case SyntaxKind::ArrayElementList:
return SyntaxKind::ArrayElement;
case SyntaxKind::DictionaryElementList:
return SyntaxKind::DictionaryElement;
case SyntaxKind::FunctionParameterList:
return SyntaxKind::FunctionParameter;
case SyntaxKind::TupleTypeElementList:
return SyntaxKind::TupleTypeElement;
case SyntaxKind::TuplePatternElementList:
return SyntaxKind::TuplePatternElement;
default:
return SyntaxKind::Unknown;
}
}
ParserStatus
Parser::parseListSyntax(tok RightK, SourceLoc LeftLoc,
Optional<ParsedTokenSyntax> &LastComma,
Optional<ParsedTokenSyntax> &Right,
SmallVectorImpl<ParsedSyntax>& Junk,
bool AllowSepAfterLast, Diag<> ErrorDiag,
llvm::function_ref<ParserStatus()> callback) {
auto TokIsStringInterpolationEOF = [&]() -> bool {
return Tok.is(tok::eof) && Tok.getText() == ")" && RightK == tok::r_paren;
};
if (Tok.is(RightK)) {
Right = consumeTokenSyntax(RightK);
return makeParserSuccess();
}
if (TokIsStringInterpolationEOF()) {
Tok.setKind(RightK);
Right = consumeTokenSyntax();
return makeParserSuccess();
}
ParserStatus Status;
while (true) {
while (Tok.is(tok::comma)) {
diagnose(Tok, diag::unexpected_separator, ",")
.fixItRemove(SourceRange(Tok.getLoc()));
auto Comma = consumeTokenSyntax();
Junk.push_back(std::move(Comma));
}
SourceLoc StartLoc = Tok.getLoc();
Status |= callback();
if (Tok.is(RightK))
break;
// If the lexer stopped with an EOF token whose spelling is ")", then this
// is actually the tuple that is a string literal interpolation context.
// Just accept the ")" and build the tuple as we usually do.
if (TokIsStringInterpolationEOF()) {
Tok.setKind(RightK);
Right = consumeTokenSyntax();
return Status;
}
// If we haven't made progress, or seeing any error, skip ahead.
if (Tok.getLoc() == StartLoc || Status.isError()) {
assert(Status.isError() && "no progress without error");
skipListUntilDeclRBraceSyntax(Junk, LeftLoc, RightK, tok::comma);
if (Tok.is(RightK) || Tok.isNot(tok::comma))
break;
}
if (LastComma) {
if (Tok.isNot(RightK))
continue;
if (!AllowSepAfterLast) {
diagnose(Tok, diag::unexpected_separator, ",")
.fixItRemove(SourceRange(PreviousLoc));
}
break;
}
// If we're in a comma-separated list, the next token is at the
// beginning of a new line and can never start an element, break.
if (Tok.isAtStartOfLine() &&
(Tok.is(tok::r_brace) || isStartOfDecl() || isStartOfStmt())) {
break;
}
// If we found EOF or such, bailout.
if (Tok.isAny(tok::eof, tok::pound_endif)) {
IsInputIncomplete = true;
break;
}
diagnose(Tok, diag::expected_separator, ",")
.fixItInsertAfter(PreviousLoc, ",");
Status.setIsParseError();
}
auto RightResult = parseMatchingTokenSyntax(RightK, ErrorDiag, LeftLoc,
Status.isError());
Status |= RightResult.getStatus();
Right = RightResult.getOrNull();
return Status;
}
ParserStatus
Parser::parseList(tok RightK, SourceLoc LeftLoc, SourceLoc &RightLoc,
bool AllowSepAfterLast, Diag<> ErrorDiag, SyntaxKind Kind,
llvm::function_ref<ParserStatus()> callback) {
auto TokIsStringInterpolationEOF = [&]() -> bool {
return Tok.is(tok::eof) && Tok.getText() == ")" && RightK == tok::r_paren;
};
llvm::Optional<SyntaxParsingContext> ListContext;
ListContext.emplace(SyntaxContext, Kind);
if (Kind == SyntaxKind::Unknown)
ListContext->setTransparent();
SyntaxKind ElementKind = getListElementKind(Kind);
if (Tok.is(RightK)) {
ListContext.reset();
RightLoc = consumeToken(RightK);
return makeParserSuccess();
}
if (TokIsStringInterpolationEOF()) {
RightLoc = Tok.getLoc();
return makeParserSuccess();
}
ParserStatus Status;
while (true) {
while (Tok.is(tok::comma)) {
diagnose(Tok, diag::unexpected_separator, ",")
.fixItRemove(SourceRange(Tok.getLoc()));
consumeToken();
}
SourceLoc StartLoc = Tok.getLoc();
SyntaxParsingContext ElementContext(SyntaxContext, ElementKind);
if (ElementKind == SyntaxKind::Unknown)
ElementContext.setTransparent();
Status |= callback();
if (Tok.is(RightK))
break;
// If the lexer stopped with an EOF token whose spelling is ")", then this
// is actually the tuple that is a string literal interpolation context.
// Just accept the ")" and build the tuple as we usually do.
if (TokIsStringInterpolationEOF()) {
RightLoc = Tok.getLoc();
return Status;
}
// If we haven't made progress, or seeing any error, skip ahead.
if (Tok.getLoc() == StartLoc || Status.isError()) {
assert(Status.isError() && "no progress without error");
skipListUntilDeclRBrace(LeftLoc, RightK, tok::comma);
if (Tok.is(RightK) || Tok.isNot(tok::comma))
break;
}
if (consumeIf(tok::comma)) {
if (Tok.isNot(RightK))
continue;
if (!AllowSepAfterLast) {
diagnose(Tok, diag::unexpected_separator, ",")
.fixItRemove(SourceRange(PreviousLoc));
}
break;
}
// If we're in a comma-separated list, the next token is at the
// beginning of a new line and can never start an element, break.
if (Tok.isAtStartOfLine() &&
(Tok.is(tok::r_brace) || isStartOfDecl() || isStartOfStmt())) {
break;
}
// If we found EOF or such, bailout.
if (Tok.isAny(tok::eof, tok::pound_endif)) {
IsInputIncomplete = true;
break;
}
diagnose(Tok, diag::expected_separator, ",")
.fixItInsertAfter(PreviousLoc, ",");
Status.setIsParseError();
}
ListContext.reset();
if (Status.isError()) {
// If we've already got errors, don't emit missing RightK diagnostics.
RightLoc =
Tok.is(RightK) ? consumeToken() : getLocForMissingMatchingToken();
} else if (parseMatchingToken(RightK, RightLoc, ErrorDiag, LeftLoc)) {
Status.setIsParseError();
}
return Status;
}
/// diagnoseRedefinition - Diagnose a redefinition error, with a note
/// referring back to the original definition.
void Parser::diagnoseRedefinition(ValueDecl *Prev, ValueDecl *New) {
assert(New != Prev && "Cannot conflict with self");
diagnose(New->getLoc(), diag::decl_redefinition);
diagnose(Prev->getLoc(), diag::previous_decldef, Prev->getBaseName());
}
Optional<StringRef>
Parser::getStringLiteralIfNotInterpolated(SourceLoc Loc,
StringRef DiagText) {
assert(Tok.is(tok::string_literal));
// FIXME: Support extended escaping string literal.
if (Tok.getCustomDelimiterLen()) {
diagnose(Loc, diag::forbidden_extended_escaping_string, DiagText);
return None;
}
SmallVector<Lexer::StringSegment, 1> Segments;
L->getStringLiteralSegments(Tok, Segments);
if (Segments.size() != 1 ||
Segments.front().Kind == Lexer::StringSegment::Expr) {
diagnose(Loc, diag::forbidden_interpolated_string, DiagText);
return None;
}
return SourceMgr.extractText(CharSourceRange(Segments.front().Loc,
Segments.front().Length));
}
struct ParserUnit::Implementation {
std::shared_ptr<SyntaxParseActions> SPActions;
LangOptions LangOpts;
SearchPathOptions SearchPathOpts;
DiagnosticEngine Diags;
ASTContext &Ctx;
SourceFile *SF;
std::unique_ptr<Parser> TheParser;
Implementation(SourceManager &SM, SourceFileKind SFKind, unsigned BufferID,
const LangOptions &Opts, StringRef ModuleName,
std::shared_ptr<SyntaxParseActions> spActions)
: SPActions(std::move(spActions)),
LangOpts(Opts),
Diags(SM),
Ctx(*ASTContext::get(LangOpts, SearchPathOpts, SM, Diags)),
SF(new (Ctx) SourceFile(
*ModuleDecl::create(Ctx.getIdentifier(ModuleName), Ctx),
SFKind, BufferID,
SourceFile::ImplicitModuleImportKind::None,
Opts.CollectParsedToken,
Opts.BuildSyntaxTree)) {
}
~Implementation() {
// We need to delete the parser before the context so that it can finalize
// its SourceFileSyntax while it is still alive
TheParser.reset();
delete &Ctx;
}
};
ParserUnit::ParserUnit(SourceManager &SM, SourceFileKind SFKind, unsigned BufferID)
: ParserUnit(SM, SFKind, BufferID, LangOptions(), "input") {
}
ParserUnit::ParserUnit(SourceManager &SM, SourceFileKind SFKind, unsigned BufferID,
const LangOptions &LangOpts, StringRef ModuleName,
std::shared_ptr<SyntaxParseActions> spActions,
SyntaxParsingCache *SyntaxCache)
: Impl(*new Implementation(SM, SFKind, BufferID, LangOpts, ModuleName,
std::move(spActions))) {
Impl.SF->SyntaxParsingCache = SyntaxCache;
Impl.TheParser.reset(new Parser(BufferID, *Impl.SF, /*SIL=*/nullptr,
/*PersistentState=*/nullptr, Impl.SPActions,
/*DelayBodyParsing=*/false));
}
ParserUnit::ParserUnit(SourceManager &SM, SourceFileKind SFKind, unsigned BufferID,
unsigned Offset, unsigned EndOffset)
: Impl(*new Implementation(SM, SFKind, BufferID, LangOptions(), "input",
nullptr)) {
std::unique_ptr<Lexer> Lex;
Lex.reset(new Lexer(Impl.LangOpts, SM,
BufferID, &Impl.Diags,
LexerMode::Swift,
HashbangMode::Allowed,
CommentRetentionMode::None,
TriviaRetentionMode::WithoutTrivia,
Offset, EndOffset));
Impl.TheParser.reset(new Parser(std::move(Lex), *Impl.SF, /*SIL=*/nullptr,
/*PersistentState=*/nullptr, Impl.SPActions, /*DelayBodyParsing=*/false));
}
ParserUnit::~ParserUnit() {
delete &Impl;
}
OpaqueSyntaxNode ParserUnit::parse() {
auto &P = getParser();
bool Done = false;
while (!Done) {
P.parseTopLevel();
Done = P.Tok.is(tok::eof);
}
return P.finalizeSyntaxTree();
}
Parser &ParserUnit::getParser() {
return *Impl.TheParser;
}
DiagnosticEngine &ParserUnit::getDiagnosticEngine() {
return Impl.Diags;
}
const LangOptions &ParserUnit::getLangOptions() const {
return Impl.LangOpts;
}
SourceFile &ParserUnit::getSourceFile() {
return *Impl.SF;
}
ParsedDeclName swift::parseDeclName(StringRef name) {
if (name.empty()) return ParsedDeclName();
// Local function to handle the parsing of the base name + context.
//
// Returns true if an error occurred, without recording the base name.
ParsedDeclName result;
auto parseBaseName = [&](StringRef text) -> bool {
// Split the text into context name and base name.
StringRef contextName, baseName;
std::tie(contextName, baseName) = text.rsplit('.');
if (baseName.empty()) {
baseName = contextName;
contextName = StringRef();
} else if (contextName.empty()) {
return true;
}
auto isValidIdentifier = [](StringRef text) -> bool {
return Lexer::isIdentifier(text) && text != "_";
};
// Make sure we have an identifier for the base name.
if (!isValidIdentifier(baseName))
return true;
// If we have a context, make sure it is an identifier, or a series of
// dot-separated identifiers.
// FIXME: What about generic parameters?
if (!contextName.empty()) {
StringRef first;
StringRef rest = contextName;
do {
std::tie(first, rest) = rest.split('.');
if (!isValidIdentifier(first))
return true;
} while (!rest.empty());
}
// Record the results.
result.ContextName = contextName;
result.BaseName = baseName;
return false;
};
// If this is not a function name, just parse the base name and
// we're done.
if (name.back() != ')') {
if (Lexer::isOperator(name))
result.BaseName = name;
else if (parseBaseName(name))
return ParsedDeclName();
return result;
}
// We have a function name.
result.IsFunctionName = true;
// Split the base name from the parameters.
StringRef baseName, parameters;
std::tie(baseName, parameters) = name.split('(');
if (parameters.empty()) return ParsedDeclName();
// If the base name is prefixed by "getter:" or "setter:", it's an
// accessor.
if (baseName.startswith("getter:")) {
result.IsGetter = true;
result.IsFunctionName = false;
baseName = baseName.substr(7);
} else if (baseName.startswith("setter:")) {
result.IsSetter = true;
result.IsFunctionName = false;
baseName = baseName.substr(7);
}
// Parse the base name.
if (parseBaseName(baseName)) return ParsedDeclName();
parameters = parameters.drop_back(); // ')'
if (parameters.empty()) return result;
if (parameters.back() != ':')
return ParsedDeclName();
bool isMember = !result.ContextName.empty();
do {
StringRef NextParam;
std::tie(NextParam, parameters) = parameters.split(':');
if (!Lexer::isIdentifier(NextParam))
return ParsedDeclName();
if (NextParam == "_") {
result.ArgumentLabels.push_back("");
} else if (isMember && NextParam == "self") {
// For a member, "self" indicates the self parameter. There can
// only be one such parameter.
if (result.SelfIndex) return ParsedDeclName();
result.SelfIndex = result.ArgumentLabels.size();
} else {
result.ArgumentLabels.push_back(NextParam);
}
} while (!parameters.empty());
// Drop the argument labels for a property accessor; they aren't used.
if (result.isPropertyAccessor())
result.ArgumentLabels.clear();
return result;
}
DeclName ParsedDeclName::formDeclName(ASTContext &ctx) const {
return swift::formDeclName(ctx, BaseName, ArgumentLabels, IsFunctionName,
/*IsInitializer=*/true);
}
DeclName swift::formDeclName(ASTContext &ctx,
StringRef baseName,
ArrayRef<StringRef> argumentLabels,
bool isFunctionName,
bool isInitializer) {
// We cannot import when the base name is not an identifier.
if (baseName.empty())
return DeclName();
if (!Lexer::isIdentifier(baseName) && !Lexer::isOperator(baseName))
return DeclName();
// Get the identifier for the base name. Special-case `init`.
DeclBaseName baseNameId = ((isInitializer && baseName == "init")
? DeclBaseName::createConstructor()
: ctx.getIdentifier(baseName));
// For non-functions, just use the base name.
if (!isFunctionName) return baseNameId;
// For functions, we need to form a complete name.
// Convert the argument names.
SmallVector<Identifier, 4> argumentLabelIds;
for (auto argName : argumentLabels) {
if (argumentLabels.empty() || !Lexer::isIdentifier(argName)) {
argumentLabelIds.push_back(Identifier());
continue;
}
argumentLabelIds.push_back(ctx.getIdentifier(argName));
}
// Build the result.
return DeclName(ctx, baseNameId, argumentLabelIds);
}
DeclName swift::parseDeclName(ASTContext &ctx, StringRef name) {
return parseDeclName(name).formDeclName(ctx);
}