tools/fidl/fidlc/include/fidl/lexer.h

// Copyright 2017 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#ifndef TOOLS_FIDL_FIDLC_INCLUDE_FIDL_LEXER_H_
#define TOOLS_FIDL_FIDLC_INCLUDE_FIDL_LEXER_H_

#include <assert.h>
#include <stdint.h>

#include <map>
#include <string_view>

#include "reporter.h"
#include "source_manager.h"
#include "token.h"

namespace fidl {

using reporter::Reporter;

// The lexer does not own the data it operates on. It merely takes a
// std::string_view and produces a stream of tokens and possibly a failure
// partway through.
// See https://fuchsia.dev/fuchsia-src/development/languages/fidl/reference/compiler#_lexing
// for additional context
class Lexer {
 public:
  // The Lexer assumes the final character is 0. This substantially
  // simplifies advancing to the next character.
  Lexer(const SourceFile& source_file, Reporter* reporter)
      : source_file_(source_file), reporter_(reporter) {
    token_subkinds = {
#define TOKEN_SUBKIND(Name, Spelling) {Spelling, Token::Subkind::k##Name},
#include "fidl/token_definitions.inc"
#undef TOKEN_SUBKIND
    };
    current_ = data().data();
    end_of_file_ = current_ + data().size();
    previous_end_ = token_start_ = current_;
  }

  // Lexes and returns the next token. Must not be called again after returning
  // Token::Kind::kEndOfFile.
  Token Lex();

 private:
  std::string_view data() { return source_file_.data(); }

  constexpr char Peek() const;
  void Skip();
  char Consume();
  std::string_view Reset(Token::Kind kind);
  Token Finish(Token::Kind kind);

  void SkipWhitespace();

  Token LexEndOfStream();
  Token LexNumericLiteral();
  Token LexIdentifier();
  Token LexEscapedIdentifier();
  Token LexStringLiteral();
  Token LexCommentOrDocComment();

  const SourceFile& source_file_;
  std::map<std::string_view, Token::Subkind> token_subkinds;
  Reporter* reporter_;

  const char* current_ = nullptr;
  const char* end_of_file_ = nullptr;
  const char* token_start_ = nullptr;
  const char* previous_end_ = nullptr;
  size_t token_size_ = 0u;
};

}  // namespace fidl

#endif  // TOOLS_FIDL_FIDLC_INCLUDE_FIDL_LEXER_H_