Merge pull request #13631 from omochi/lex-escape-backtick
[Parse] Lexer build backtick trivia around espaced identifier token
diff --git a/include/swift/Parse/Lexer.h b/include/swift/Parse/Lexer.h
index ef2d095..01cd01c 100644
--- a/include/swift/Parse/Lexer.h
+++ b/include/swift/Parse/Lexer.h
@@ -484,6 +484,7 @@
}
void formToken(tok Kind, const char *TokStart, bool MultilineString = false);
+ void formEscapedIdentifierToken(const char *TokStart);
/// Advance to the end of the line.
/// If EatNewLine is true, CurPtr will be at end of newline character.
diff --git a/lib/Parse/Lexer.cpp b/lib/Parse/Lexer.cpp
index f582123..68a09ed 100644
--- a/lib/Parse/Lexer.cpp
+++ b/lib/Parse/Lexer.cpp
@@ -271,6 +271,19 @@
NextToken.setToken(Kind, TokenText, CommentLength, MultilineString);
}
+void Lexer::formEscapedIdentifierToken(const char *TokStart) {
+ assert(CurPtr - TokStart >= 3 && "escaped identifier must be longer than or equal 3 bytes");
+ assert(TokStart[0] == '`' && "escaped identifier starts with backtick");
+ assert(CurPtr[-1] == '`' && "escaped identifier ends with backtick");
+ if (TriviaRetention == TriviaRetentionMode::WithTrivia) {
+ LeadingTrivia.push_back(TriviaPiece::backtick());
+ assert(TrailingTrivia.size() == 0 && "TrailingTrivia is empty here");
+ TrailingTrivia.push_back(TriviaPiece::backtick());
+ }
+ formToken(tok::identifier, TokStart);
+ NextToken.setEscapedIdentifier(true);
+}
+
Lexer::State Lexer::getStateForBeginningOfTokenLoc(SourceLoc Loc) const {
const char *Ptr = getBufferPtrForSourceLoc(Loc);
// Skip whitespace backwards until we hit a newline. This is needed to
@@ -1770,8 +1783,7 @@
// If we have the terminating "`", it's an escaped identifier.
if (*CurPtr == '`') {
++CurPtr;
- formToken(tok::identifier, Quote);
- NextToken.setEscapedIdentifier(true);
+ formEscapedIdentifierToken(Quote);
return;
}
}
@@ -1779,8 +1791,7 @@
// Special case; allow '`$`'.
if (Quote[1] == '$' && Quote[2] == '`') {
CurPtr = Quote + 3;
- formToken(tok::identifier, Quote);
- NextToken.setEscapedIdentifier(true);
+ formEscapedIdentifierToken(Quote);
return;
}
diff --git a/lib/Parse/Parser.cpp b/lib/Parse/Parser.cpp
index 28c22a5..bab7f7e 100644
--- a/lib/Parse/Parser.cpp
+++ b/lib/Parse/Parser.cpp
@@ -286,10 +286,6 @@
Trivia LeadingTrivia, TrailingTrivia;
do {
L.lex(Tok, LeadingTrivia, TrailingTrivia);
- if (Tok.isEscapedIdentifier()) {
- LeadingTrivia.push_back(TriviaPiece::backtick());
- TrailingTrivia.push_front(TriviaPiece::backtick());
- }
auto ThisToken = RawTokenSyntax::make(Tok.getKind(), Tok.getText(),
SourcePresence::Present, LeadingTrivia,
TrailingTrivia);
diff --git a/lib/Syntax/SyntaxParsingContext.cpp b/lib/Syntax/SyntaxParsingContext.cpp
index 795201f..21e4171 100644
--- a/lib/Syntax/SyntaxParsingContext.cpp
+++ b/lib/Syntax/SyntaxParsingContext.cpp
@@ -77,10 +77,6 @@
if (!Enabled)
return;
- if (Tok.isEscapedIdentifier()) {
- LeadingTrivia.push_back(TriviaPiece::backtick());
- TrailingTrivia.push_front(TriviaPiece::backtick());
- }
addRawSyntax(RawTokenSyntax::make(Tok.getKind(), Tok.getText(),
SourcePresence::Present, LeadingTrivia,
TrailingTrivia));
diff --git a/test/Syntax/tokens_escaped_identifier.swift b/test/Syntax/tokens_escaped_identifier.swift
new file mode 100644
index 0000000..20e3082
--- /dev/null
+++ b/test/Syntax/tokens_escaped_identifier.swift
@@ -0,0 +1,21 @@
+// RUN: %swift-syntax-test -input-source-filename %s -dump-full-tokens | %FileCheck %s
+let /*leading trivia*/ `if` = 3
+print(/*leading trivia*/ `if` )
+
+// CHECK-LABEL: 2:25
+// CHECK-NEXT:(token identifier
+// CHECK-NEXT: (trivia block_comment/*leading trivia*/)
+// CHECK-NEXT: (trivia space 1)
+// CHECK-NEXT: (trivia backtick 1)
+// CHECK-NEXT: (text="if")
+// CHECK-NEXT: (trivia backtick 1)
+// CHECK-NEXT: (trivia space 1))
+
+// CHECK-LABEL: 3:27
+// CHECK-NEXT:(token identifier
+// CHECK-NEXT: (trivia block_comment/*leading trivia*/)
+// CHECK-NEXT: (trivia space 1)
+// CHECK-NEXT: (trivia backtick 1)
+// CHECK-NEXT: (text="if")
+// CHECK-NEXT: (trivia backtick 1)
+// CHECK-NEXT: (trivia space 1))
