[Parser] Update CachedTokens while parsing ObjectiveC template argument list
Consider the following ObjC++ snippet:
--
@protocol PA;
@protocol PB;
@class NSArray<ObjectType>;
typedef int some_t;
id<PA> FA(NSArray<id<PB>> *h, some_t group);
--
This would hit an assertion in the parser after generating an annotation token
while trying to update the token cache:
Assertion failed: (CachedTokens[CachedLexPos-1].getLastLoc() == Tok.getAnnotationEndLoc() && "The annotation should be until the most recent cached token")
...
7 clang::Preprocessor::AnnotatePreviousCachedTokens(clang::Token const&) + 494
8 clang::Parser::TryAnnotateTypeOrScopeTokenAfterScopeSpec(bool, bool, clang::CXXScopeSpec&, bool) + 1163
9 clang::Parser::TryAnnotateTypeOrScopeToken(bool, bool) + 361
10 clang::Parser::isCXXDeclarationSpecifier(clang::Parser::TPResult, bool*) + 598
...
The cached preprocessor token in this case is:
greatergreater '>>' Loc=<testcase.mm:7:24>
while the annotation ("NSArray<id<PB>>") ends at "testcase.mm:7:25", hence the
assertion.
Properly update the CachedTokens during template parsing to contain
two greater tokens instead of a greatergreater.
Differential Revision: http://reviews.llvm.org/D15173
rdar://problem/23494277
git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@259311 91177308-0d34-0410-b5e6-96231b3b80d8
(cherry picked from commit a0d5dbf5e8d9532d6581c5b221d798094bc97e82)
diff --git a/include/clang/Lex/Preprocessor.h b/include/clang/Lex/Preprocessor.h
index f6154b6..5ffa482 100644
--- a/include/clang/Lex/Preprocessor.h
+++ b/include/clang/Lex/Preprocessor.h
@@ -1185,6 +1185,17 @@
return CachedTokens[CachedLexPos-1].getLastLoc();
}
+ /// \brief Whether \p Tok is the most recent token (`CachedLexPos - 1`) in
+ /// CachedTokens.
+ bool IsPreviousCachedToken(const Token &Tok) const;
+
+ /// \brief Replace token in `CachedLexPos - 1` in CachedTokens by the tokens
+ /// in \p NewToks.
+ ///
+ /// Useful when a token needs to be split in smaller ones and CachedTokens
+ /// most recent token must to be updated to reflect that.
+ void ReplacePreviousCachedToken(ArrayRef<Token> NewToks);
+
/// \brief Replace the last token with an annotation token.
///
/// Like AnnotateCachedTokens(), this routine replaces an
diff --git a/lib/Lex/PPCaching.cpp b/lib/Lex/PPCaching.cpp
index bd48ae6..4742aae 100644
--- a/lib/Lex/PPCaching.cpp
+++ b/lib/Lex/PPCaching.cpp
@@ -116,3 +116,29 @@
}
}
}
+
+bool Preprocessor::IsPreviousCachedToken(const Token &Tok) const {
+ // There's currently no cached token...
+ if (!CachedLexPos)
+ return false;
+
+ const Token LastCachedTok = CachedTokens[CachedLexPos - 1];
+ if (LastCachedTok.getKind() != Tok.getKind())
+ return false;
+
+ int RelOffset = 0;
+ if ((!getSourceManager().isInSameSLocAddrSpace(
+ Tok.getLocation(), getLastCachedTokenLocation(), &RelOffset)) ||
+ RelOffset)
+ return false;
+
+ return true;
+}
+
+void Preprocessor::ReplacePreviousCachedToken(ArrayRef<Token> NewToks) {
+ assert(CachedLexPos != 0 && "Expected to have some cached tokens");
+ CachedTokens.insert(CachedTokens.begin() + CachedLexPos - 1, NewToks.begin(),
+ NewToks.end());
+ CachedTokens.erase(CachedTokens.begin() + CachedLexPos - 1 + NewToks.size());
+ CachedLexPos += NewToks.size() - 1;
+}
diff --git a/lib/Parse/ParseTemplate.cpp b/lib/Parse/ParseTemplate.cpp
index 3a964dd..b416a1a 100644
--- a/lib/Parse/ParseTemplate.cpp
+++ b/lib/Parse/ParseTemplate.cpp
@@ -828,6 +828,7 @@
}
// Strip the initial '>' from the token.
+ Token PrevTok = Tok;
if (RemainingToken == tok::equal && Next.is(tok::equal) &&
areTokensAdjacent(Tok, Next)) {
// Join two adjacent '=' tokens into one, for cases like:
@@ -844,6 +845,17 @@
PP.getSourceManager(),
getLangOpts()));
+ // The advance from '>>' to '>' in a ObjectiveC template argument list needs
+ // to be properly reflected in the token cache to allow correct interaction
+ // between annotation and backtracking.
+ if (ObjCGenericList && PrevTok.getKind() == tok::greatergreater &&
+ RemainingToken == tok::greater && PP.IsPreviousCachedToken(PrevTok)) {
+ PrevTok.setKind(RemainingToken);
+ PrevTok.setLength(1);
+ Token NewToks[] = {PrevTok, Tok};
+ PP.ReplacePreviousCachedToken(NewToks);
+ }
+
if (!ConsumeLastToken) {
// Since we're not supposed to consume the '>' token, we need to push
// this token and revert the current token back to the '>'.
diff --git a/test/Parser/objcxx11-protocol-in-template.mm b/test/Parser/objcxx11-protocol-in-template.mm
index c5c3b6c..5c80ae9 100644
--- a/test/Parser/objcxx11-protocol-in-template.mm
+++ b/test/Parser/objcxx11-protocol-in-template.mm
@@ -8,3 +8,11 @@
vector<id<P>> v;
vector<vector<id<P>>> v2;
+
+@protocol PA;
+@protocol PB;
+
+@class NSArray<ObjectType>;
+typedef int some_t;
+
+id<PA> FA(NSArray<id<PB>> *h, some_t group);
