lldb/unittests/ValueObject/DILLexerTests.cpp - third_party/llvm-project - Git at Google

 //===-- DILLexerTests.cpp --------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//

 #include "lldb/ValueObject/DILLexer.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Testing/Support/Error.h"
 #include "gtest/gtest.h"
 #include <string>

 using llvm::StringRef;

 using namespace lldb_private::dil;

 llvm::Expected<std::vector<std::pair<Token::Kind, std::string>>>
 ExtractTokenData(llvm::StringRef input_expr) {

   llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(input_expr);
   if (!maybe_lexer)
     return maybe_lexer.takeError();
   DILLexer lexer(*maybe_lexer);

   std::vector<std::pair<Token::Kind, std::string>> data;
   do {
     Token tok = lexer.GetCurrentToken();
     data.push_back(std::make_pair(tok.GetKind(), tok.GetSpelling()));
     lexer.Advance();
   } while (data.back().first != Token::eof);
   // Don't return the eof token.
   data.pop_back();
   return data;
 }

 TEST(DILLexerTests, SimpleTest) {
   StringRef input_expr("simple_var");
   llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(input_expr);
   ASSERT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
   DILLexer lexer(*maybe_lexer);
   Token token = lexer.GetCurrentToken();

   EXPECT_EQ(token.GetKind(), Token::identifier);
   EXPECT_EQ(token.GetSpelling(), "simple_var");
   lexer.Advance();
   token = lexer.GetCurrentToken();
   EXPECT_EQ(token.GetKind(), Token::eof);
 }

 TEST(DILLexerTests, TokenKindTest) {
   Token token = Token(Token::identifier, "ident", 0);

   EXPECT_TRUE(token.Is(Token::identifier));
   EXPECT_FALSE(token.Is(Token::l_paren));
   EXPECT_TRUE(token.IsOneOf({Token::eof, Token::identifier}));
   EXPECT_FALSE(token.IsOneOf(
       {Token::l_paren, Token::r_paren, Token::coloncolon, Token::eof}));
 }

 TEST(DILLexerTests, LookAheadTest) {
   StringRef input_expr("(anonymous namespace)::some_var");
   llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(input_expr);
   ASSERT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
   DILLexer lexer(*maybe_lexer);
   Token token = lexer.GetCurrentToken();

   // Current token is '('; check the next 4 tokens, to make
   // sure they are the identifier 'anonymous', the identifier 'namespace'
   // ')' and '::', in that order.
   EXPECT_EQ(token.GetKind(), Token::l_paren);
   EXPECT_EQ(lexer.LookAhead(1).GetKind(), Token::identifier);
   EXPECT_EQ(lexer.LookAhead(1).GetSpelling(), "anonymous");
   EXPECT_EQ(lexer.LookAhead(2).GetKind(), Token::identifier);
   EXPECT_EQ(lexer.LookAhead(2).GetSpelling(), "namespace");
   EXPECT_EQ(lexer.LookAhead(3).GetKind(), Token::r_paren);
   EXPECT_EQ(lexer.LookAhead(4).GetKind(), Token::coloncolon);

   // Our current index should still be 0, as we only looked ahead; we are still
   // officially on the '('.
   EXPECT_EQ(lexer.GetCurrentTokenIdx(), 0u);

   // Accept the 'lookahead', so our current token is '::', which has the index
   // 4 in our vector of tokens (which starts at zero).
   lexer.Advance(4);
   token = lexer.GetCurrentToken();
   EXPECT_EQ(token.GetKind(), Token::coloncolon);
   EXPECT_EQ(lexer.GetCurrentTokenIdx(), 4u);

   lexer.Advance();
   token = lexer.GetCurrentToken();
   EXPECT_EQ(token.GetKind(), Token::identifier);
   EXPECT_EQ(token.GetSpelling(), "some_var");
   EXPECT_EQ(lexer.GetCurrentTokenIdx(), 5u);
   EXPECT_EQ(token.GetLocation(), strlen("(anonymous namespace)::"));

   lexer.Advance();
   token = lexer.GetCurrentToken();
   EXPECT_EQ(token.GetKind(), Token::eof);
 }

 TEST(DILLexerTests, MultiTokenLexTest) {
   EXPECT_THAT_EXPECTED(
       ExtractTokenData("This string has (several ) ::identifiers"),
       llvm::HasValue(testing::ElementsAre(
           testing::Pair(Token::identifier, "This"),
           testing::Pair(Token::identifier, "string"),
           testing::Pair(Token::identifier, "has"),
           testing::Pair(Token::l_paren, "("),
           testing::Pair(Token::identifier, "several"),
           testing::Pair(Token::r_paren, ")"),
           testing::Pair(Token::coloncolon, "::"),
           testing::Pair(Token::identifier, "identifiers"))));
 }

 TEST(DILLexerTests, IdentifiersTest) {
   // These strings should lex into identifier tokens.
   std::vector<std::string> valid_identifiers = {
       "$My_name1", "$pc",  "abcd", "_", "_a",     "_a_",      "$",
       "a_b",       "this", "self", "a", "MyName", "namespace"};

   // The lexer can lex these strings, but they should not be identifiers.
   std::vector<std::string> invalid_identifiers = {"", "::", "(", ")", "0abc"};

   // The lexer is expected to fail attempting to lex these strings (it cannot
   // create valid tokens out of them).
   std::vector<std::string> invalid_tok_strings = {"#include", "a@a"};

   // Verify that all of the valid identifiers come out as identifier tokens.
   for (auto &str : valid_identifiers) {
     SCOPED_TRACE(str);
     EXPECT_THAT_EXPECTED(ExtractTokenData(str),
                          llvm::HasValue(testing::ElementsAre(
                              testing::Pair(Token::identifier, str))));
   }

   // Verify that the lexer fails on invalid token strings.
   for (auto &str : invalid_tok_strings) {
     SCOPED_TRACE(str);
     auto maybe_lexer = DILLexer::Create(str);
     EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Failed());
   }

   // Verify that none of the invalid identifiers come out as identifier tokens.
   for (auto &str : invalid_identifiers) {
     SCOPED_TRACE(str);
     llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(str);
     EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
     DILLexer lexer(*maybe_lexer);
     Token token = lexer.GetCurrentToken();
     EXPECT_TRUE(token.IsNot(Token::identifier));
     EXPECT_TRUE(token.IsOneOf({Token::eof, Token::coloncolon, Token::l_paren,
                                Token::r_paren, Token::integer_constant}));
   }
 }

 TEST(DILLexerTests, NumbersTest) {
   // These strings should lex into number tokens.
   std::vector<std::string> valid_integers = {"123", "0x123", "0123", "0b101"};
   std::vector<std::string> valid_floats = {
       "1.2",    ".2",    "2.f",     "0x1.2",    "0x.2",      ".2e1f",
       "2.e+1f", "0x1.f", "0x1.2P1", "0x1.p-1f", "0x1.2P+3f", "1E1",
       "1E+1",   "0x1p1", "0x1p+1",  "0xf.fp1f"};

   // The lexer can lex these strings, but they should not be numbers.
   std::vector<std::string> invalid_numbers = {"", "x123", "b123", "a.b"};

   for (auto &str : valid_integers) {
     SCOPED_TRACE(str);
     EXPECT_THAT_EXPECTED(ExtractTokenData(str),
                          llvm::HasValue(testing::ElementsAre(
                              testing::Pair(Token::integer_constant, str))));
   }
   for (auto &str : valid_floats) {
     SCOPED_TRACE(str);
     EXPECT_THAT_EXPECTED(ExtractTokenData(str),
                          llvm::HasValue(testing::ElementsAre(
                              testing::Pair(Token::float_constant, str))));
   }
   // Verify that none of the invalid numbers come out as numeric tokens.
   for (auto &str : invalid_numbers) {
     SCOPED_TRACE(str);
     llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(str);
     EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
     DILLexer lexer(*maybe_lexer);
     Token token = lexer.GetCurrentToken();
     EXPECT_TRUE(token.IsNot(Token::integer_constant));
     EXPECT_TRUE(token.IsOneOf({Token::eof, Token::identifier}));
   }

   // Verify that '-' and '+' are not lexed if they're not part of a number
   std::vector<std::string> expressions = {"1+e",     "0x1+p",      "1.1+e",
                                           "1.1e1+e", "0x1.1p-1-p", "1e-1+e",
                                           "1e1+e",   "0x1p-1-p",   "0xe+e"};
   for (auto &str : expressions) {
     SCOPED_TRACE(str);
     llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(str);
     EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
     DILLexer lexer(*maybe_lexer);
     Token token = lexer.GetCurrentToken();
     EXPECT_TRUE(
         token.IsOneOf({Token::integer_constant, Token::float_constant}));
     lexer.Advance();
     token = lexer.GetCurrentToken();
     EXPECT_TRUE(token.IsOneOf({Token::plus, Token::minus}));
     lexer.Advance();
     token = lexer.GetCurrentToken();
     EXPECT_TRUE(token.Is(Token::identifier));
   }
 }
	//===-- DILLexerTests.cpp --------------------------------------------===//
	//
	// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	// See https://llvm.org/LICENSE.txt for license information.
	// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	//
	//===----------------------------------------------------------------------===//

	#include "lldb/ValueObject/DILLexer.h"
	#include "llvm/ADT/StringRef.h"
	#include "llvm/Testing/Support/Error.h"
	#include "gtest/gtest.h"
	#include <string>

	using llvm::StringRef;

	using namespace lldb_private::dil;

	llvm::Expected<std::vector<std::pair<Token::Kind, std::string>>>
	ExtractTokenData(llvm::StringRef input_expr) {

	llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(input_expr);
	if (!maybe_lexer)
	return maybe_lexer.takeError();
	DILLexer lexer(*maybe_lexer);

	std::vector<std::pair<Token::Kind, std::string>> data;
	do {
	Token tok = lexer.GetCurrentToken();
	data.push_back(std::make_pair(tok.GetKind(), tok.GetSpelling()));
	lexer.Advance();
	} while (data.back().first != Token::eof);
	// Don't return the eof token.
	data.pop_back();
	return data;
	}

	TEST(DILLexerTests, SimpleTest) {
	StringRef input_expr("simple_var");
	llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(input_expr);
	ASSERT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
	DILLexer lexer(*maybe_lexer);
	Token token = lexer.GetCurrentToken();

	EXPECT_EQ(token.GetKind(), Token::identifier);
	EXPECT_EQ(token.GetSpelling(), "simple_var");
	lexer.Advance();
	token = lexer.GetCurrentToken();
	EXPECT_EQ(token.GetKind(), Token::eof);
	}

	TEST(DILLexerTests, TokenKindTest) {
	Token token = Token(Token::identifier, "ident", 0);

	EXPECT_TRUE(token.Is(Token::identifier));
	EXPECT_FALSE(token.Is(Token::l_paren));
	EXPECT_TRUE(token.IsOneOf({Token::eof, Token::identifier}));
	EXPECT_FALSE(token.IsOneOf(
	{Token::l_paren, Token::r_paren, Token::coloncolon, Token::eof}));
	}

	TEST(DILLexerTests, LookAheadTest) {
	StringRef input_expr("(anonymous namespace)::some_var");
	llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(input_expr);
	ASSERT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
	DILLexer lexer(*maybe_lexer);
	Token token = lexer.GetCurrentToken();

	// Current token is '('; check the next 4 tokens, to make
	// sure they are the identifier 'anonymous', the identifier 'namespace'
	// ')' and '::', in that order.
	EXPECT_EQ(token.GetKind(), Token::l_paren);
	EXPECT_EQ(lexer.LookAhead(1).GetKind(), Token::identifier);
	EXPECT_EQ(lexer.LookAhead(1).GetSpelling(), "anonymous");
	EXPECT_EQ(lexer.LookAhead(2).GetKind(), Token::identifier);
	EXPECT_EQ(lexer.LookAhead(2).GetSpelling(), "namespace");
	EXPECT_EQ(lexer.LookAhead(3).GetKind(), Token::r_paren);
	EXPECT_EQ(lexer.LookAhead(4).GetKind(), Token::coloncolon);

	// Our current index should still be 0, as we only looked ahead; we are still
	// officially on the '('.
	EXPECT_EQ(lexer.GetCurrentTokenIdx(), 0u);

	// Accept the 'lookahead', so our current token is '::', which has the index
	// 4 in our vector of tokens (which starts at zero).
	lexer.Advance(4);
	token = lexer.GetCurrentToken();
	EXPECT_EQ(token.GetKind(), Token::coloncolon);
	EXPECT_EQ(lexer.GetCurrentTokenIdx(), 4u);

	lexer.Advance();
	token = lexer.GetCurrentToken();
	EXPECT_EQ(token.GetKind(), Token::identifier);
	EXPECT_EQ(token.GetSpelling(), "some_var");
	EXPECT_EQ(lexer.GetCurrentTokenIdx(), 5u);
	EXPECT_EQ(token.GetLocation(), strlen("(anonymous namespace)::"));

	lexer.Advance();
	token = lexer.GetCurrentToken();
	EXPECT_EQ(token.GetKind(), Token::eof);
	}

	TEST(DILLexerTests, MultiTokenLexTest) {
	EXPECT_THAT_EXPECTED(
	ExtractTokenData("This string has (several ) ::identifiers"),
	llvm::HasValue(testing::ElementsAre(
	testing::Pair(Token::identifier, "This"),
	testing::Pair(Token::identifier, "string"),
	testing::Pair(Token::identifier, "has"),
	testing::Pair(Token::l_paren, "("),
	testing::Pair(Token::identifier, "several"),
	testing::Pair(Token::r_paren, ")"),
	testing::Pair(Token::coloncolon, "::"),
	testing::Pair(Token::identifier, "identifiers"))));
	}

	TEST(DILLexerTests, IdentifiersTest) {
	// These strings should lex into identifier tokens.
	std::vector<std::string> valid_identifiers = {
	"$My_name1", "$pc", "abcd", "_", "_a", "_a_", "$",
	"a_b", "this", "self", "a", "MyName", "namespace"};

	// The lexer can lex these strings, but they should not be identifiers.
	std::vector<std::string> invalid_identifiers = {"", "::", "(", ")", "0abc"};

	// The lexer is expected to fail attempting to lex these strings (it cannot
	// create valid tokens out of them).
	std::vector<std::string> invalid_tok_strings = {"#include", "a@a"};

	// Verify that all of the valid identifiers come out as identifier tokens.
	for (auto &str : valid_identifiers) {
	SCOPED_TRACE(str);
	EXPECT_THAT_EXPECTED(ExtractTokenData(str),
	llvm::HasValue(testing::ElementsAre(
	testing::Pair(Token::identifier, str))));
	}

	// Verify that the lexer fails on invalid token strings.
	for (auto &str : invalid_tok_strings) {
	SCOPED_TRACE(str);
	auto maybe_lexer = DILLexer::Create(str);
	EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Failed());
	}

	// Verify that none of the invalid identifiers come out as identifier tokens.
	for (auto &str : invalid_identifiers) {
	SCOPED_TRACE(str);
	llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(str);
	EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
	DILLexer lexer(*maybe_lexer);
	Token token = lexer.GetCurrentToken();
	EXPECT_TRUE(token.IsNot(Token::identifier));
	EXPECT_TRUE(token.IsOneOf({Token::eof, Token::coloncolon, Token::l_paren,
	Token::r_paren, Token::integer_constant}));
	}
	}

	TEST(DILLexerTests, NumbersTest) {
	// These strings should lex into number tokens.
	std::vector<std::string> valid_integers = {"123", "0x123", "0123", "0b101"};
	std::vector<std::string> valid_floats = {
	"1.2", ".2", "2.f", "0x1.2", "0x.2", ".2e1f",
	"2.e+1f", "0x1.f", "0x1.2P1", "0x1.p-1f", "0x1.2P+3f", "1E1",
	"1E+1", "0x1p1", "0x1p+1", "0xf.fp1f"};

	// The lexer can lex these strings, but they should not be numbers.
	std::vector<std::string> invalid_numbers = {"", "x123", "b123", "a.b"};

	for (auto &str : valid_integers) {
	SCOPED_TRACE(str);
	EXPECT_THAT_EXPECTED(ExtractTokenData(str),
	llvm::HasValue(testing::ElementsAre(
	testing::Pair(Token::integer_constant, str))));
	}
	for (auto &str : valid_floats) {
	SCOPED_TRACE(str);
	EXPECT_THAT_EXPECTED(ExtractTokenData(str),
	llvm::HasValue(testing::ElementsAre(
	testing::Pair(Token::float_constant, str))));
	}
	// Verify that none of the invalid numbers come out as numeric tokens.
	for (auto &str : invalid_numbers) {
	SCOPED_TRACE(str);
	llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(str);
	EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
	DILLexer lexer(*maybe_lexer);
	Token token = lexer.GetCurrentToken();
	EXPECT_TRUE(token.IsNot(Token::integer_constant));
	EXPECT_TRUE(token.IsOneOf({Token::eof, Token::identifier}));
	}

	// Verify that '-' and '+' are not lexed if they're not part of a number
	std::vector<std::string> expressions = {"1+e", "0x1+p", "1.1+e",
	"1.1e1+e", "0x1.1p-1-p", "1e-1+e",
	"1e1+e", "0x1p-1-p", "0xe+e"};
	for (auto &str : expressions) {
	SCOPED_TRACE(str);
	llvm::Expected<DILLexer> maybe_lexer = DILLexer::Create(str);
	EXPECT_THAT_EXPECTED(maybe_lexer, llvm::Succeeded());
	DILLexer lexer(*maybe_lexer);
	Token token = lexer.GetCurrentToken();
	EXPECT_TRUE(
	token.IsOneOf({Token::integer_constant, Token::float_constant}));
	lexer.Advance();
	token = lexer.GetCurrentToken();
	EXPECT_TRUE(token.IsOneOf({Token::plus, Token::minus}));
	lexer.Advance();
	token = lexer.GetCurrentToken();
	EXPECT_TRUE(token.Is(Token::identifier));
	}
	}