| //===--- Token.cpp - Tokens and token streams in the pseudoparser ---------===// |
| // |
| // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| // |
| //===----------------------------------------------------------------------===// |
| |
| #include "clang-pseudo/Token.h" |
| #include "clang/Basic/LangOptions.h" |
| #include "llvm/ADT/StringExtras.h" |
| #include "llvm/Support/Format.h" |
| #include "llvm/Support/FormatVariadic.h" |
| |
| namespace clang { |
| namespace pseudo { |
| |
| llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Token &T) { |
| OS << llvm::formatv("{0} {1}:{2} ", clang::tok::getTokenName(T.Kind), T.Line, |
| T.Indent); |
| OS << '"'; |
| llvm::printEscapedString(T.text(), OS); |
| OS << '"'; |
| if (T.Flags) |
| OS << llvm::format(" flags=%x", T.Flags); |
| return OS; |
| } |
| |
| llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const TokenStream &TS) { |
| OS << "Index Kind Line Text\n"; |
| for (const auto &T : TS.tokens()) { |
| OS << llvm::format("%5d: %16s %4d:%-2d ", TS.index(T), |
| clang::tok::getTokenName(T.Kind), T.Line, T.Indent); |
| OS << '"'; |
| llvm::printEscapedString(T.text(), OS); |
| OS << '"'; |
| if (T.Flags) |
| OS << llvm::format(" flags=%x", T.Flags); |
| OS << '\n'; |
| } |
| return OS; |
| } |
| |
| llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const Token::Range &R) { |
| OS << llvm::formatv("[{0},{1})", R.Begin, R.End); |
| return OS; |
| } |
| |
| TokenStream::TokenStream(std::shared_ptr<void> Payload) |
| : Payload(std::move(Payload)) { |
| Storage.emplace_back(); |
| Storage.back().Kind = clang::tok::eof; |
| } |
| |
| void TokenStream::finalize() { |
| assert(!isFinalized()); |
| unsigned LastLine = Storage.back().Line; |
| Storage.emplace_back(); |
| Storage.back().Kind = tok::eof; |
| Storage.back().Line = LastLine + 1; |
| |
| Tokens = Storage; |
| Tokens = Tokens.drop_front().drop_back(); |
| } |
| |
| bool TokenStream::isFinalized() const { |
| assert(!Storage.empty() && Storage.front().Kind == tok::eof); |
| if (Storage.size() == 1) |
| return false; |
| return Storage.back().Kind == tok::eof; |
| } |
| |
| void TokenStream::print(llvm::raw_ostream &OS) const { |
| bool FirstToken = true; |
| unsigned LastLine = -1; |
| StringRef LastText; |
| for (const auto &T : tokens()) { |
| StringRef Text = T.text(); |
| if (FirstToken) { |
| FirstToken = false; |
| } else if (T.Line == LastLine) { |
| if (LastText.data() + LastText.size() != Text.data()) |
| OS << ' '; |
| } else { |
| OS << '\n'; |
| OS.indent(T.Indent); |
| } |
| OS << Text; |
| LastLine = T.Line; |
| LastText = Text; |
| } |
| if (!FirstToken) |
| OS << '\n'; |
| } |
| |
| clang::LangOptions genericLangOpts(clang::Language Lang, |
| clang::LangStandard::Kind Standard) { |
| clang::LangOptions Opts; |
| std::vector<std::string> UnusedIncludes; |
| LangOptions::setLangDefaults(Opts, Lang, llvm::Triple(), UnusedIncludes, |
| Standard); |
| |
| // Some options are "on by default", but e.g. at the driver level. |
| if (Opts.CPlusPlus) |
| Opts.CXXOperatorNames = true; |
| if (Opts.CPlusPlus20) |
| Opts.Coroutines = true; |
| |
| // Some options are off by default, but define keywords we want to tolerate. |
| if (Opts.CPlusPlus) |
| Opts.MicrosoftExt = true; // kw__try, kw__finally |
| Opts.DeclSpecKeyword = true; // __declspec |
| Opts.WChar = true; |
| |
| return Opts; |
| } |
| |
| TokenStream stripComments(const TokenStream &Input) { |
| TokenStream Out(Input.getPayload()); |
| for (const Token &T : Input.tokens()) { |
| if (T.Kind == tok::comment) |
| continue; |
| Out.push(T); |
| } |
| Out.finalize(); |
| return Out; |
| } |
| |
| } // namespace pseudo |
| } // namespace clang |