// Copyright 2021 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

//! Defines the lexer for the Documentation Compiler.
//! It extracts all the lexical items (with their location) from a source text (Documentation).

use crate::source::Location;
use crate::source::Source;
use crate::DocCompiler;
use std::rc::Rc;
use std::str::CharIndices;

/// Defines all the lexical items we can parse.
#[derive(Clone)]
pub enum LexicalContent {
    /// A number. For example:
    /// - 1234
    /// - 0xabcd
    Number(String),

    /// A name from an English text point of view. For example:
    /// - Hello
    /// - heap-specific
    /// - doesn't
    Name(String),

    /// A reference to an existing applicative concept like a method name, a field name, ...
    /// A reference is held between back quotes.
    Reference(String),

    /// A block of text between two triple back quotes.
    CodeBlock(String),

    /// A text between single quotes. The left quote must be after a white space or at the
    /// beginning of the text. The text inside the quotes can't start with a space (in that
    /// particular case, it's a stand alone single quote).
    SingleQuoteString(String),

    /// A text between double quotes. The text inside the quotes can't start with a space (in
    /// that particular case, it's a stand alone double quote).
    DoubleQuoteString(String),

    /// A standalone single quote.
    SingleQuote,

    /// A standalone double quote.
    DoubleQuote,

    /// A comma.
    Comma,

    /// A semicolon.
    Semicolon,

    /// The plus character.
    Plus,

    /// The minus character.
    Minus,

    /// The asterisk character.
    Asterisk,

    /// The slash character (not immediately following a name).
    Slash,

    /// The percent character.
    Percent,

    /// The backslash character.
    BackSlash,

    /// The ampersand character.
    Ampersand,

    /// The hash (number sign) character.
    Hash,

    /// Two hash characters in a row.
    HashHash,

    /// The pipe character.
    Pipe,

    /// The tilde character.
    Tilde,

    /// The caret character.
    Caret,

    /// The dollar character.
    Dollar,

    /// The at sign character.
    AtSign,

    /// The unicode paragraph character.
    Paragraph,

    /// The equal character.
    Equal,

    /// Two equal characters in a row.
    EqualEqual,

    /// The left angle bracket.
    LowerThan,

    // The left angle bracket followed by an equal.
    LowerOrEqual,

    // The right angle bracket.
    GreaterThan,

    // The right angle bracket followed by an equal.
    GreaterOrEqual,

    /// A left parenthesis.
    LeftParenthesis,

    /// A Right parenthesis.
    RightParenthesis,

    /// A left bracket (square).
    LeftBracket,

    /// A Right bracket (square).
    RightBracket,

    /// A left brace (curly bracket).
    LeftBrace,

    /// A Right brace (curly bracket).
    RightBrace,
    /// A standalone unicode symbol like: ⮬ or ⮯.
    UnicodeCharacter(char),

    /// The end of an english sentence.
    /// For example ".", ":", "!", "?".
    EndOfSentence(char),

    /// Some consecutive blank spaces.
    Spaces(u32),

    /// Some consecutive new lines.
    NewLines(u32),

    /// The end of the documentation. The parser generates one and only one EndOfInput.
    /// This is always the last item in the list.
    EndOfInput,
}

/// Defines a lexical item (item + location in the source file).
#[derive(Clone)]
pub struct LexicalItem {
    pub location: Location,
    pub content: LexicalContent,
}

/// Parses some documentation and extract all the lexical items found in the documentation.
///
/// Returns the reduced items.
pub fn reduce_lexems(compiler: &mut DocCompiler, source: &Rc<Source>) -> Option<Vec<LexicalItem>> {
    let mut items: Vec<LexicalItem> = Vec::new();
    let mut ok = true;
    let mut iter = source.text.char_indices();
    let mut current = iter.next();
    while let Some((index, character)) = current {
        current = match character {
            '0'..='9' => reduce_number_or_name(&mut items, &source, index, character, &mut iter),
            'a'..='z' | 'A'..='Z' | '_' => reduce_name(&mut items, &source, index, &mut iter),
            '`' => reduce_back_quote(compiler, &mut items, &source, &mut iter),
            '\'' => {
                reduce_single_quote_string(compiler, &mut items, &source, index, &mut iter, &mut ok)
            }
            '"' => {
                reduce_double_quote_string(compiler, &mut items, &source, index, &mut iter, &mut ok)
            }
            ',' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::Comma,
            ),
            ';' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::Semicolon,
            ),
            '+' => {
                reduce_single_character(&mut items, &source, index, &mut iter, LexicalContent::Plus)
            }
            '-' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::Minus,
            ),
            '*' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::Asterisk,
            ),
            '/' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::Slash,
            ),
            '%' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::Percent,
            ),
            '\\' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::BackSlash,
            ),
            '&' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::Ampersand,
            ),
            '#' => reduce_one_or_two_characters(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::Hash,
                '#',
                LexicalContent::HashHash,
            ),
            '|' => {
                reduce_single_character(&mut items, &source, index, &mut iter, LexicalContent::Pipe)
            }
            '~' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::Tilde,
            ),
            '^' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::Caret,
            ),
            '$' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::Dollar,
            ),
            '@' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::AtSign,
            ),
            '§' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::Paragraph,
            ),
            '=' => reduce_one_or_two_characters(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::Equal,
                '=',
                LexicalContent::EqualEqual,
            ),
            '<' => reduce_one_or_two_characters(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::LowerThan,
                '=',
                LexicalContent::LowerOrEqual,
            ),
            '>' => reduce_one_or_two_characters(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::GreaterThan,
                '=',
                LexicalContent::GreaterOrEqual,
            ),
            '(' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::LeftParenthesis,
            ),
            ')' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::RightParenthesis,
            ),
            '[' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::LeftBracket,
            ),
            ']' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::RightBracket,
            ),
            '{' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::LeftBrace,
            ),
            '}' => reduce_single_character(
                &mut items,
                &source,
                index,
                &mut iter,
                LexicalContent::RightBrace,
            ),
            '⮬' => reduce_unicode_character(&mut items, &source, index, &mut iter, character),
            '⮯' => reduce_unicode_character(&mut items, &source, index, &mut iter, character),
            '⮫' => reduce_unicode_character(&mut items, &source, index, &mut iter, character),
            '⮨' => reduce_unicode_character(&mut items, &source, index, &mut iter, character),
            '⮭' => reduce_unicode_character(&mut items, &source, index, &mut iter, character),
            '⮮' => reduce_unicode_character(&mut items, &source, index, &mut iter, character),
            '⮪' => reduce_unicode_character(&mut items, &source, index, &mut iter, character),
            '⮩' => reduce_unicode_character(&mut items, &source, index, &mut iter, character),
            '↵' => reduce_unicode_character(&mut items, &source, index, &mut iter, character),
            '⌘' => reduce_unicode_character(&mut items, &source, index, &mut iter, character),
            // Reduces a sentence end character.
            '.' | ':' | '!' | '?' => {
                items.push(LexicalItem {
                    location: Location { source: Rc::clone(&source), start: index, end: index + 1 },
                    content: LexicalContent::EndOfSentence(character),
                });
                iter.next()
            }
            ' ' => reduce_spaces(&mut items, &source, index, &mut iter),
            '\n' => reduce_new_lines(&mut items, &source, index, &mut iter),
            // Unknown character.
            _ => {
                ok = false;
                compiler.add_error(
                    &Location { source: Rc::clone(&source), start: index, end: index },
                    format!("Unknown character <{}>", character),
                );
                iter.next()
            }
        }
    }
    // We reached the end of the text. We add a EndOfInput item. This way, the next stage won't
    // need to check if we are at the end of the vector when trying to reduce something.
    items.push(LexicalItem {
        location: Location {
            source: Rc::clone(&source),
            start: source.text.len(),
            end: source.text.len(),
        },
        content: LexicalContent::EndOfInput,
    });
    if ok {
        Some(items)
    } else {
        None
    }
}

/// Reduces a number.
///
/// For example:
/// - 1234
/// - 0xabcd
///
/// If we find characters which are not valid for a number but valid for a name then, the whole
/// sequence of characters (including the leading digits) are used to return a Name.
///
/// For example:
/// - 123abc
fn reduce_number_or_name(
    items: &mut Vec<LexicalItem>,
    source: &Rc<Source>,
    start: usize,
    first_character: char,
    iter: &mut CharIndices<'_>,
) -> Option<(usize, char)> {
    let mut hexadecimal = false;
    let mut number = true;
    let mut current: Option<(usize, char)> = iter.next();
    if first_character == '0' {
        // Checks for an hexadecimal number.
        if let Some((_, character)) = current {
            if character == 'x' || character == 'X' {
                hexadecimal = true;
                current = iter.next();
            }
        }
    }
    let end = loop {
        match current {
            Some((index, character)) => match character {
                '0'..='9' => {}
                'a'..='f' | 'A'..='F' => {
                    if !hexadecimal {
                        number = false;
                    }
                }
                'g'..='z' | 'G'..='Z' | '_' | '-' | '\'' => number = false,
                _ => {
                    break index;
                }
            },
            None => {
                break source.text.len();
            }
        }
        current = iter.next();
    };
    if number {
        // Only digits (potentially hexadecimal) have been found => Number.
        items.push(LexicalItem {
            location: Location { source: Rc::clone(&source), start, end },
            content: LexicalContent::Number(source.text[start..end].to_string()),
        });
    } else {
        // Not a valid number => Name.
        items.push(LexicalItem {
            location: Location { source: Rc::clone(&source), start, end },
            content: LexicalContent::Name(source.text[start..end].to_string()),
        });
    }
    current
}

/// Reduces a name.
///
/// A name is a valid name from the English language point of view.
///
/// Examples of valid names:
/// - Hello
/// - heap-specific
/// - doesn't
fn reduce_name(
    items: &mut Vec<LexicalItem>,
    source: &Rc<Source>,
    start: usize,
    iter: &mut CharIndices<'_>,
) -> Option<(usize, char)> {
    let mut current: Option<(usize, char)>;
    let end = loop {
        current = iter.next();
        match current {
            Some((index, character)) => match character {
                'a'..='z' | 'A'..='Z' | '0'..='9' | '_' | '-' | '\'' => {}
                _ => {
                    break index;
                }
            },
            None => {
                break source.text.len();
            }
        }
    };
    items.push(LexicalItem {
        location: Location { source: Rc::clone(&source), start, end },
        content: LexicalContent::Name(source.text[start..end].to_string()),
    });
    current
}

/// Reduces an item which starts with a back quote.
///
/// It can be a reference (text between two back quotes).
/// It can be a code block (text between two triple back quotes).
fn reduce_back_quote(
    compiler: &mut DocCompiler,
    items: &mut Vec<LexicalItem>,
    source: &Rc<Source>,
    iter: &mut CharIndices<'_>,
) -> Option<(usize, char)> {
    let mut current = iter.next();
    let start = if let Some((index, _)) = current { index } else { 0 };
    // A fist back quote has already been reduced. Look for a second one.
    if let Some((_, character)) = current {
        if character == '`' {
            current = iter.next();
            // Two back quotes have already been reduced. Look for a third one.
            let end = if let Some((index, character)) = current {
                if character == '`' {
                    // Three back quotes have been found. That means this is a code block.
                    return reduce_code_block(compiler, items, source, iter);
                }
                index
            } else {
                source.text.len()
            };
            // Only two back quotes have been found. It would be an empty reference. That means an
            // error.
            compiler.add_error(
                &Location { source: Rc::clone(&source), start, end },
                "Empty reference".to_owned(),
            );
            return current;
        }
    }
    loop {
        match current {
            Some((index, character)) => match character {
                '`' => {
                    items.push(LexicalItem {
                        location: Location { source: Rc::clone(&source), start, end: index },
                        content: LexicalContent::Reference(source.text[start..index].to_string()),
                    });
                    current = iter.next();
                    break;
                }
                _ => {}
            },
            None => {
                compiler.add_error(
                    &Location { source: Rc::clone(&source), start, end: source.text.len() },
                    "Unterminated reference".to_owned(),
                );
                break;
            }
        }
        current = iter.next();
    }
    current
}

/// Reduces a block of text within two triple back quotes.
///
/// The opening triple back quotes have already been reduced.
fn reduce_code_block(
    compiler: &mut DocCompiler,
    items: &mut Vec<LexicalItem>,
    source: &Rc<Source>,
    iter: &mut CharIndices<'_>,
) -> Option<(usize, char)> {
    let mut current = iter.next();
    let start = if let Some((index, _)) = current { index } else { 0 };
    // Skip the characters between the opening three back quotes and the end of the line (these
    // characters define the code syntax to use).
    loop {
        match current {
            Some((_, character)) => {
                if character == '\n' {
                    current = iter.next();
                    break;
                }
            }
            None => {
                compiler.add_error(
                    &Location { source: Rc::clone(&source), start, end: source.text.len() },
                    "Unterminated code block".to_owned(),
                );
                return current;
            }
        }
        current = iter.next();
    }
    // Reduce the code block. The end of the block is reached when three back quotes followed by a
    // new line are found at the beginning of a line.
    // If three back quote are found at the beginning of a line without a following new line, this
    // is an error.
    let start_block = if let Some((index, _)) = current { index } else { 0 };
    loop {
        match current {
            Some((_, character)) => {
                if character != '\n' {
                    current = iter.next();
                    continue;
                }
                // A new line is found.
                current = iter.next();
                if current.is_none() {
                    continue;
                }
                let (end_block, first_quote) = current.unwrap();
                if first_quote != '`' {
                    continue;
                }
                // A fist back quote is found.
                current = iter.next();
                if current.is_none() {
                    continue;
                }
                let (_, second_quote) = current.unwrap();
                if second_quote != '`' {
                    continue;
                }
                // A second back quote is found.
                current = iter.next();
                if current.is_none() {
                    continue;
                }
                let (_, third_quote) = current.unwrap();
                if third_quote != '`' {
                    continue;
                }
                // A third back quote is found.
                current = iter.next();
                let new_line_pos = if let Some((index, ending_new_line)) = current {
                    if ending_new_line == '\n' {
                        // A following new line is found. We have reduced a code block.
                        items.push(LexicalItem {
                            location: Location {
                                source: Rc::clone(&source),
                                start: start_block,
                                end: end_block,
                            },
                            content: LexicalContent::CodeBlock(
                                source.text[start_block..end_block].to_string(),
                            ),
                        });
                        return iter.next();
                    }
                    index
                } else {
                    source.text.len()
                };
                // The three back quote are not followed by a new line. This is an error.
                compiler.add_error(
                    &Location {
                        source: Rc::clone(&source),
                        start: new_line_pos,
                        end: new_line_pos,
                    },
                    "New line expected to end the code block".to_owned(),
                );
                return current;
            }
            None => {
                compiler.add_error(
                    &Location { source: Rc::clone(&source), start, end: source.text.len() },
                    "Unterminated code block".to_owned(),
                );
                break;
            }
        }
    }
    current
}

/// Reduces a string between single quotes, a single quote or a name.
///
/// If the first single quote is not after a white space or at the beginning of the text then,
/// a Name is returned.
/// For example, with the input: `a_reference`'s
/// This function is called to reduce 's and the result is Name("'s").
///
/// If the first single quote is followed by a space, the result is SingleQuote.
/// For example, the following text is valid: A ' is valid.
fn reduce_single_quote_string(
    compiler: &mut DocCompiler,
    items: &mut Vec<LexicalItem>,
    source: &Rc<Source>,
    start: usize,
    iter: &mut CharIndices<'_>,
    ok: &mut bool,
) -> Option<(usize, char)> {
    // If the quote is not at the beginning of the line and not preceded by a white space then
    // reduce a Name.
    if let Some(last) = items.last() {
        match last.content {
            LexicalContent::Spaces(_) | LexicalContent::NewLines(_) => {}
            _ => return reduce_name(items, source, start, iter),
        }
    }
    // If the quote is immediately followed by a space then only reduce a SingleQuote.
    let mut current = iter.next();
    let text_start = if let Some((index, character)) = current {
        if character == ' ' || character == '\n' {
            items.push(LexicalItem {
                location: Location { source: Rc::clone(&source), start, end: start },
                content: LexicalContent::SingleQuote,
            });
            return current;
        }
        index
    } else {
        0
    };
    // Reduces a string.
    loop {
        match current {
            Some((index, character)) => match character {
                '\'' => {
                    current = iter.next();
                    let end =
                        if let Some((index, _)) = current { index } else { source.text.len() };
                    items.push(LexicalItem {
                        location: Location { source: Rc::clone(&source), start, end: end },
                        content: LexicalContent::SingleQuoteString(
                            source.text[text_start..index].to_string(),
                        ),
                    });
                    return current;
                }
                _ => {}
            },
            None => {
                break;
            }
        }
        current = iter.next();
    }
    *ok = false;
    compiler.add_error(
        &Location { source: Rc::clone(&source), start, end: source.text.len() },
        "Unterminated string (character <'> expected).".to_owned(),
    );
    current
}

/// Reduces a string between double quotes or a double quote.
///
/// If the first double quote is followed by a space, the result is DoubleQuote.
/// For example, the following text is valid: A " is valid.
fn reduce_double_quote_string(
    compiler: &mut DocCompiler,
    items: &mut Vec<LexicalItem>,
    source: &Rc<Source>,
    start: usize,
    iter: &mut CharIndices<'_>,
    ok: &mut bool,
) -> Option<(usize, char)> {
    let mut current = iter.next();
    // If the quote is immediately followed by a space then only reduce a DoubleQuote.
    let text_start = if let Some((index, character)) = current {
        if character == ' ' || character == '\n' {
            items.push(LexicalItem {
                location: Location { source: Rc::clone(&source), start, end: start },
                content: LexicalContent::DoubleQuote,
            });
            return current;
        }
        index
    } else {
        0
    };
    // Reduces a string.
    loop {
        match current {
            Some((index, character)) => match character {
                '"' => {
                    current = iter.next();
                    let end =
                        if let Some((index, _)) = current { index } else { source.text.len() };
                    items.push(LexicalItem {
                        location: Location { source: Rc::clone(&source), start, end: end },
                        content: LexicalContent::DoubleQuoteString(
                            source.text[text_start..index].to_string(),
                        ),
                    });
                    return current;
                }
                _ => {}
            },
            None => {
                break;
            }
        }
        current = iter.next();
    }
    *ok = false;
    compiler.add_error(
        &Location { source: Rc::clone(&source), start, end: source.text.len() },
        "Unterminated string (character <\"> expected).".to_owned(),
    );
    current
}

/// Reduce a single character.
///
/// The caller already found what character it is. Only adds the content to items and returns the
/// next character to reduce.
fn reduce_single_character(
    items: &mut Vec<LexicalItem>,
    source: &Rc<Source>,
    start: usize,
    iter: &mut CharIndices<'_>,
    content: LexicalContent,
) -> Option<(usize, char)> {
    items.push(LexicalItem {
        location: Location { source: Rc::clone(&source), start, end: start },
        content,
    });
    iter.next()
}

/// Reduce one or two characters.
///
/// Add either one_character_content or two_character_content to items based on whether the next
/// character matches second_character.
fn reduce_one_or_two_characters(
    items: &mut Vec<LexicalItem>,
    source: &Rc<Source>,
    start: usize,
    iter: &mut CharIndices<'_>,
    one_character_content: LexicalContent,
    second_character: char,
    two_character_content: LexicalContent,
) -> Option<(usize, char)> {
    let current = iter.next();
    if let Some((_, character)) = current {
        if character == second_character {
            let current = iter.next();
            let end = if let Some((index, _)) = current { index } else { source.text.len() };
            items.push(LexicalItem {
                location: Location { source: Rc::clone(&source), start, end },
                content: two_character_content,
            });
            return current;
        }
    }
    items.push(LexicalItem {
        location: Location { source: Rc::clone(&source), start, end: start },
        content: one_character_content,
    });
    current
}

/// Reduce a single unicode character.
fn reduce_unicode_character(
    items: &mut Vec<LexicalItem>,
    source: &Rc<Source>,
    start: usize,
    iter: &mut CharIndices<'_>,
    character: char,
) -> Option<(usize, char)> {
    items.push(LexicalItem {
        location: Location { source: Rc::clone(&source), start, end: start },
        content: LexicalContent::UnicodeCharacter(character),
    });
    iter.next()
}

/// Reduces spaces (at least one).
fn reduce_spaces(
    items: &mut Vec<LexicalItem>,
    source: &Rc<Source>,
    start: usize,
    iter: &mut CharIndices<'_>,
) -> Option<(usize, char)> {
    // Counts the number of blank spaces found.
    let mut count = 1;
    let mut current: Option<(usize, char)>;
    let end = loop {
        current = iter.next();
        match current {
            Some((index, character)) => match character {
                ' ' => count += 1,
                _ => {
                    break index;
                }
            },
            None => {
                break source.text.len();
            }
        }
    };
    items.push(LexicalItem {
        location: Location { source: Rc::clone(&source), start, end },
        content: LexicalContent::Spaces(count),
    });
    current
}

/// Reduces new lines (at least one).
///
/// Consecutives new lines are grouped together.
/// A count greater than 1 means one new line followed by (count - 1) blank lines.
fn reduce_new_lines(
    items: &mut Vec<LexicalItem>,
    source: &Rc<Source>,
    start: usize,
    iter: &mut CharIndices<'_>,
) -> Option<(usize, char)> {
    // Counts the number of new lines found.
    let mut count = 1;
    let mut current: Option<(usize, char)>;
    let end = loop {
        current = iter.next();
        match current {
            Some((index, character)) => match character {
                '\n' => count += 1,
                _ => {
                    break index;
                }
            },
            None => {
                break source.text.len();
            }
        }
    };
    items.push(LexicalItem {
        location: Location { source: Rc::clone(&source), start, end },
        content: LexicalContent::NewLines(count),
    });
    current
}

#[cfg(test)]
mod test {
    use crate::lexer::reduce_lexems;
    use crate::source::Source;
    use crate::utils::test::lexical_items_to_errors;
    use crate::DocCompiler;
    use std::rc::Rc;

    #[test]
    fn lexer_ok() {
        let mut compiler = DocCompiler::new();
        let source = Rc::new(Source::new(
            "sdk/foo/foo.fidl".to_owned(),
            10,
            4,
            "Some documentation.\n".to_owned(),
        ));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(!items.is_none());
        assert!(compiler.errors.is_empty());
    }

    #[test]
    fn lexer_numbers() {
        let mut compiler = DocCompiler::new();
        let source = Rc::new(Source::new(
            "sdk/foo/foo.fidl".to_owned(),
            10,
            4,
            "1234 0x789abc 123abc 0x78abg".to_owned(),
        ));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(!items.is_none());
        lexical_items_to_errors(&mut compiler, &items.unwrap(), /*with_spaces=*/ false);
        assert_eq!(
            compiler.errors,
            "\
1234 0x789abc 123abc 0x78abg
^^^^
sdk/foo/foo.fidl: 10:4: Number <1234>
1234 0x789abc 123abc 0x78abg
     ^^^^^^^^
sdk/foo/foo.fidl: 10:9: Number <0x789abc>
1234 0x789abc 123abc 0x78abg
              ^^^^^^
sdk/foo/foo.fidl: 10:18: Name <123abc>
1234 0x789abc 123abc 0x78abg
                     ^^^^^^^
sdk/foo/foo.fidl: 10:25: Name <0x78abg>
"
        );
    }

    #[test]
    fn lexer_names() {
        let mut compiler = DocCompiler::new();
        let source = Rc::new(Source::new(
            "sdk/foo/foo.fidl".to_owned(),
            10,
            4,
            "It's correct to use heap-specific.\n".to_owned(),
        ));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(!items.is_none());
        lexical_items_to_errors(&mut compiler, &items.unwrap(), /*with_spaces=*/ false);
        assert_eq!(
            compiler.errors,
            "\
It's correct to use heap-specific.
^^^^
sdk/foo/foo.fidl: 10:4: Name <It's>
It's correct to use heap-specific.
     ^^^^^^^
sdk/foo/foo.fidl: 10:9: Name <correct>
It's correct to use heap-specific.
             ^^
sdk/foo/foo.fidl: 10:17: Name <to>
It's correct to use heap-specific.
                ^^^
sdk/foo/foo.fidl: 10:20: Name <use>
It's correct to use heap-specific.
                    ^^^^^^^^^^^^^
sdk/foo/foo.fidl: 10:24: Name <heap-specific>
It's correct to use heap-specific.
                                 ^
sdk/foo/foo.fidl: 10:37: EndOfSentence <.>
"
        );
    }

    #[test]
    fn lexer_reference() {
        let mut compiler = DocCompiler::new();
        let source = Rc::new(Source::new(
            "sdk/foo/foo.fidl".to_owned(),
            10,
            4,
            "`xyz` isn't `abc`.".to_owned(),
        ));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(!items.is_none());
        lexical_items_to_errors(&mut compiler, &items.unwrap(), /*with_spaces=*/ false);
        assert_eq!(
            compiler.errors,
            "\
`xyz` isn't `abc`.
 ^^^
sdk/foo/foo.fidl: 10:5: Reference <xyz>
`xyz` isn't `abc`.
      ^^^^^
sdk/foo/foo.fidl: 10:10: Name <isn't>
`xyz` isn't `abc`.
             ^^^
sdk/foo/foo.fidl: 10:17: Reference <abc>
`xyz` isn't `abc`.
                 ^
sdk/foo/foo.fidl: 10:21: EndOfSentence <.>
"
        );
    }

    #[test]
    fn lexer_reference_with_apostrophe() {
        let mut compiler = DocCompiler::new();
        let source =
            Rc::new(Source::new("sdk/foo/foo.fidl".to_owned(), 10, 4, "`xyz`'s.".to_owned()));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(!items.is_none());
        lexical_items_to_errors(&mut compiler, &items.unwrap(), /*with_spaces=*/ false);
        assert_eq!(
            compiler.errors,
            "\
`xyz`'s.
 ^^^
sdk/foo/foo.fidl: 10:5: Reference <xyz>
`xyz`'s.
     ^^
sdk/foo/foo.fidl: 10:9: Name <'s>
`xyz`'s.
       ^
sdk/foo/foo.fidl: 10:11: EndOfSentence <.>
"
        );
    }

    #[test]
    fn lexer_reference_with_apostrophe_at_the_end() {
        let mut compiler = DocCompiler::new();
        let source =
            Rc::new(Source::new("sdk/foo/foo.fidl".to_owned(), 10, 4, "`xyz`'s".to_owned()));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(!items.is_none());
        lexical_items_to_errors(&mut compiler, &items.unwrap(), /*with_spaces=*/ false);
        assert_eq!(
            compiler.errors,
            "\
`xyz`'s
 ^^^
sdk/foo/foo.fidl: 10:5: Reference <xyz>
`xyz`'s
     ^^
sdk/foo/foo.fidl: 10:9: Name <'s>
"
        );
    }

    #[test]
    fn lexer_empty_reference() {
        let mut compiler = DocCompiler::new();
        let source =
            Rc::new(Source::new("sdk/foo/foo.fidl".to_owned(), 10, 4, "`` abc".to_owned()));
        let _items = reduce_lexems(&mut compiler, &source);
        assert_eq!(
            compiler.errors,
            "\
`` abc
 ^
sdk/foo/foo.fidl: 10:5: Empty reference
"
        );
    }

    #[test]
    fn lexer_code_block() {
        let mut compiler = DocCompiler::new();
        let source = Rc::new(Source::new(
            "sdk/foo/foo.fidl".to_owned(),
            10,
            4,
            "```c++\nint a;\n```\n".to_owned(),
        ));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(!items.is_none());
        lexical_items_to_errors(&mut compiler, &items.unwrap(), /*with_spaces=*/ false);
        assert_eq!(
            compiler.errors,
            "\
int a;
^^^^^^
sdk/foo/foo.fidl: 11:4: CodeBlock <int a;
>
"
        );
    }

    #[test]
    fn lexer_unterminated_code_block_1() {
        let mut compiler = DocCompiler::new();
        let source =
            Rc::new(Source::new("sdk/foo/foo.fidl".to_owned(), 10, 4, "```c++".to_owned()));
        let _items = reduce_lexems(&mut compiler, &source);
        assert_eq!(
            compiler.errors,
            "\
```c++
   ^^^
sdk/foo/foo.fidl: 10:7: Unterminated code block
"
        );
    }

    #[test]
    fn lexer_unterminated_code_block_2() {
        let mut compiler = DocCompiler::new();
        let source = Rc::new(Source::new(
            "sdk/foo/foo.fidl".to_owned(),
            10,
            4,
            "```c++\nint a;\n".to_owned(),
        ));
        let _items = reduce_lexems(&mut compiler, &source);
        assert_eq!(
            compiler.errors,
            "\
```c++
   ^^^
sdk/foo/foo.fidl: 10:7: Unterminated code block
"
        );
    }

    #[test]
    fn lexer_code_block_missing_ending_new_line_1() {
        let mut compiler = DocCompiler::new();
        let source = Rc::new(Source::new(
            "sdk/foo/foo.fidl".to_owned(),
            10,
            4,
            "```c++\nint a;\n```".to_owned(),
        ));
        let _items = reduce_lexems(&mut compiler, &source);
        assert_eq!(
            compiler.errors,
            "\
```
   ^
sdk/foo/foo.fidl: 12:7: New line expected to end the code block
"
        );
    }

    #[test]
    fn lexer_code_block_missing_ending_new_line_2() {
        let mut compiler = DocCompiler::new();
        let source = Rc::new(Source::new(
            "sdk/foo/foo.fidl".to_owned(),
            10,
            4,
            "```c++\nint a;\n```xyz".to_owned(),
        ));
        let _items = reduce_lexems(&mut compiler, &source);
        assert_eq!(
            compiler.errors,
            "\
```xyz
   ^
sdk/foo/foo.fidl: 12:7: New line expected to end the code block
"
        );
    }

    #[test]
    fn lexer_single_quotes() {
        let mut compiler = DocCompiler::new();
        let source = Rc::new(Source::new(
            "sdk/foo/foo.fidl".to_owned(),
            10,
            4,
            "'abcd' ' abc's 'xyz' '' `abc`'s".to_owned(),
        ));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(!items.is_none());
        lexical_items_to_errors(&mut compiler, &items.unwrap(), /*with_spaces=*/ false);
        assert_eq!(
            compiler.errors,
            "\
'abcd' ' abc's 'xyz' '' `abc`'s
^^^^^^
sdk/foo/foo.fidl: 10:4: SingleQuoteString <abcd>
'abcd' ' abc's 'xyz' '' `abc`'s
       ^
sdk/foo/foo.fidl: 10:11: SingleQuote
'abcd' ' abc's 'xyz' '' `abc`'s
         ^^^^^
sdk/foo/foo.fidl: 10:13: Name <abc's>
'abcd' ' abc's 'xyz' '' `abc`'s
               ^^^^^
sdk/foo/foo.fidl: 10:19: SingleQuoteString <xyz>
'abcd' ' abc's 'xyz' '' `abc`'s
                     ^^
sdk/foo/foo.fidl: 10:25: SingleQuoteString <>
'abcd' ' abc's 'xyz' '' `abc`'s
                         ^^^
sdk/foo/foo.fidl: 10:29: Reference <abc>
'abcd' ' abc's 'xyz' '' `abc`'s
                             ^^
sdk/foo/foo.fidl: 10:33: Name <'s>
"
        );
    }

    #[test]
    /// Checks that we can have a backslashes including at the end of a string
    fn lexer_single_quotes_with_backslash() {
        let mut compiler = DocCompiler::new();
        let source =
            Rc::new(Source::new("sdk/foo/foo.fidl".to_owned(), 10, 4, "'xxx\\ \\'".to_owned()));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(!items.is_none());
        lexical_items_to_errors(&mut compiler, &items.unwrap(), /*with_spaces=*/ false);
        assert_eq!(
            compiler.errors,
            "\
'xxx\\ \\'
^^^^^^^^
sdk/foo/foo.fidl: 10:4: SingleQuoteString <xxx\\ \\>
"
        );
    }

    #[test]
    fn lexer_unterminated_single_quotes_1() {
        let mut compiler = DocCompiler::new();
        let source = Rc::new(Source::new("sdk/foo/foo.fidl".to_owned(), 10, 4, "'abcd".to_owned()));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(items.is_none());
        assert_eq!(
            compiler.errors,
            "\
'abcd
^^^^^
sdk/foo/foo.fidl: 10:4: Unterminated string (character <'> expected).
"
        );
    }

    #[test]
    fn lexer_unterminated_single_quotes_2() {
        let mut compiler = DocCompiler::new();
        let source =
            Rc::new(Source::new("sdk/foo/foo.fidl".to_owned(), 10, 4, "'abcd\\".to_owned()));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(items.is_none());
        assert_eq!(
            compiler.errors,
            "\
'abcd\\
^^^^^^
sdk/foo/foo.fidl: 10:4: Unterminated string (character <'> expected).
"
        );
    }

    #[test]
    fn lexer_double_quotes() {
        let mut compiler = DocCompiler::new();
        let source = Rc::new(Source::new(
            "sdk/foo/foo.fidl".to_owned(),
            10,
            4,
            "\"abcd\" \" \"\" \"xyz\"".to_owned(),
        ));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(!items.is_none());
        lexical_items_to_errors(&mut compiler, &items.unwrap(), /*with_spaces=*/ false);
        assert_eq!(
            compiler.errors,
            "\
\"abcd\" \" \"\" \"xyz\"
^^^^^^
sdk/foo/foo.fidl: 10:4: DoubleQuoteString <abcd>
\"abcd\" \" \"\" \"xyz\"
       ^
sdk/foo/foo.fidl: 10:11: DoubleQuote
\"abcd\" \" \"\" \"xyz\"
         ^^
sdk/foo/foo.fidl: 10:13: DoubleQuoteString <>
\"abcd\" \" \"\" \"xyz\"
            ^^^^^
sdk/foo/foo.fidl: 10:16: DoubleQuoteString <xyz>
"
        );
    }

    #[test]
    /// Checks that we can have a backslashes including at the end of a string
    fn lexer_double_quotes_with_backslash() {
        let mut compiler = DocCompiler::new();
        let source =
            Rc::new(Source::new("sdk/foo/foo.fidl".to_owned(), 10, 4, "\"xxx\\ \\\"".to_owned()));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(!items.is_none());
        lexical_items_to_errors(&mut compiler, &items.unwrap(), /*with_spaces=*/ false);
        assert_eq!(
            compiler.errors,
            "\
\"xxx\\ \\\"
^^^^^^^^
sdk/foo/foo.fidl: 10:4: DoubleQuoteString <xxx\\ \\>
"
        );
    }

    #[test]
    fn lexer_unterminated_double_quotes_1() {
        let mut compiler = DocCompiler::new();
        let source =
            Rc::new(Source::new("sdk/foo/foo.fidl".to_owned(), 10, 4, "\"abcd".to_owned()));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(items.is_none());
        assert_eq!(
            compiler.errors,
            "\
\"abcd
^^^^^
sdk/foo/foo.fidl: 10:4: Unterminated string (character <\"> expected).
"
        );
    }

    #[test]
    fn lexer_unterminated_double_quotes_2() {
        let mut compiler = DocCompiler::new();
        let source =
            Rc::new(Source::new("sdk/foo/foo.fidl".to_owned(), 10, 4, "\"abcd\\".to_owned()));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(items.is_none());
        assert_eq!(
            compiler.errors,
            "\
\"abcd\\
^^^^^^
sdk/foo/foo.fidl: 10:4: Unterminated string (character <\"> expected).
"
        );
    }

    #[test]
    fn lexer_symbols_and_punctuation() {
        let mut compiler = DocCompiler::new();
        let source = Rc::new(Source::new(
            "sdk/foo/foo.fidl".to_owned(),
            10,
            4,
            ", ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\".to_owned(),
        ));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(!items.is_none());
        lexical_items_to_errors(&mut compiler, &items.unwrap(), /*with_spaces=*/ false);
        assert_eq!(
            compiler.errors,
            "\
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
^
sdk/foo/foo.fidl: 10:4: Comma
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
  ^
sdk/foo/foo.fidl: 10:6: Semicolon
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
    ^
sdk/foo/foo.fidl: 10:8: Plus
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
      ^
sdk/foo/foo.fidl: 10:10: Minus
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
        ^
sdk/foo/foo.fidl: 10:12: Asterisk
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
          ^
sdk/foo/foo.fidl: 10:14: Slash
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
            ^
sdk/foo/foo.fidl: 10:16: Percent
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
              ^
sdk/foo/foo.fidl: 10:18: Ampersand
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                ^
sdk/foo/foo.fidl: 10:20: Hash
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                  ^^
sdk/foo/foo.fidl: 10:22: HashHash
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                     ^
sdk/foo/foo.fidl: 10:25: Pipe
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                       ^
sdk/foo/foo.fidl: 10:27: Tilde
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                         ^
sdk/foo/foo.fidl: 10:29: Caret
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                           ^
sdk/foo/foo.fidl: 10:31: Dollar
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                             ^
sdk/foo/foo.fidl: 10:33: AtSign
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                               ^
sdk/foo/foo.fidl: 10:35: Paragraph
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                                 ^
sdk/foo/foo.fidl: 10:37: Equal
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                                   ^^
sdk/foo/foo.fidl: 10:39: EqualEqual
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                                      ^
sdk/foo/foo.fidl: 10:42: LowerThan
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                                        ^^
sdk/foo/foo.fidl: 10:44: LowerOrEqual
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                                           ^
sdk/foo/foo.fidl: 10:47: GreaterThan
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                                             ^^
sdk/foo/foo.fidl: 10:49: GreaterOrEqual
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                                                ^
sdk/foo/foo.fidl: 10:52: LeftParenthesis
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                                                  ^
sdk/foo/foo.fidl: 10:54: RightParenthesis
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                                                    ^
sdk/foo/foo.fidl: 10:56: LeftBracket
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                                                      ^
sdk/foo/foo.fidl: 10:58: RightBracket
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                                                        ^
sdk/foo/foo.fidl: 10:60: LeftBrace
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                                                          ^
sdk/foo/foo.fidl: 10:62: RightBrace
, ; + - * / % & # ## | ~ ^ $ @ § = == < <= > >= ( ) [ ] { } \\
                                                            ^
sdk/foo/foo.fidl: 10:64: BackSlash
"
        );
    }

    #[test]
    fn lexer_unicode_symbols() {
        let mut compiler = DocCompiler::new();
        let source = Rc::new(Source::new(
            "sdk/foo/foo.fidl".to_owned(),
            10,
            4,
            "⮬ ⮯ ⮫ ⮨ ⮭ ⮮ ⮪ ⮩ ↵ ⌘".to_owned(),
        ));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(!items.is_none());
        lexical_items_to_errors(&mut compiler, &items.unwrap(), /*with_spaces=*/ false);
        assert_eq!(
            compiler.errors,
            "\
⮬ ⮯ ⮫ ⮨ ⮭ ⮮ ⮪ ⮩ ↵ ⌘
^
sdk/foo/foo.fidl: 10:4: UnicodeCharacter <⮬>
⮬ ⮯ ⮫ ⮨ ⮭ ⮮ ⮪ ⮩ ↵ ⌘
  ^
sdk/foo/foo.fidl: 10:6: UnicodeCharacter <⮯>
⮬ ⮯ ⮫ ⮨ ⮭ ⮮ ⮪ ⮩ ↵ ⌘
    ^
sdk/foo/foo.fidl: 10:8: UnicodeCharacter <⮫>
⮬ ⮯ ⮫ ⮨ ⮭ ⮮ ⮪ ⮩ ↵ ⌘
      ^
sdk/foo/foo.fidl: 10:10: UnicodeCharacter <⮨>
⮬ ⮯ ⮫ ⮨ ⮭ ⮮ ⮪ ⮩ ↵ ⌘
        ^
sdk/foo/foo.fidl: 10:12: UnicodeCharacter <⮭>
⮬ ⮯ ⮫ ⮨ ⮭ ⮮ ⮪ ⮩ ↵ ⌘
          ^
sdk/foo/foo.fidl: 10:14: UnicodeCharacter <⮮>
⮬ ⮯ ⮫ ⮨ ⮭ ⮮ ⮪ ⮩ ↵ ⌘
            ^
sdk/foo/foo.fidl: 10:16: UnicodeCharacter <⮪>
⮬ ⮯ ⮫ ⮨ ⮭ ⮮ ⮪ ⮩ ↵ ⌘
              ^
sdk/foo/foo.fidl: 10:18: UnicodeCharacter <⮩>
⮬ ⮯ ⮫ ⮨ ⮭ ⮮ ⮪ ⮩ ↵ ⌘
                ^
sdk/foo/foo.fidl: 10:20: UnicodeCharacter <↵>
⮬ ⮯ ⮫ ⮨ ⮭ ⮮ ⮪ ⮩ ↵ ⌘
                  ^
sdk/foo/foo.fidl: 10:22: UnicodeCharacter <⌘>
"
        );
    }

    #[test]
    fn lexer_end_of_sentence() {
        let mut compiler = DocCompiler::new();
        let source = Rc::new(Source::new(
            "sdk/foo/foo.fidl".to_owned(),
            10,
            4,
            "Aa. Bb? Cc! Dd:".to_owned(),
        ));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(!items.is_none());
        lexical_items_to_errors(&mut compiler, &items.unwrap(), /*with_spaces=*/ false);
        assert_eq!(
            compiler.errors,
            "\
Aa. Bb? Cc! Dd:
^^
sdk/foo/foo.fidl: 10:4: Name <Aa>
Aa. Bb? Cc! Dd:
  ^
sdk/foo/foo.fidl: 10:6: EndOfSentence <.>
Aa. Bb? Cc! Dd:
    ^^
sdk/foo/foo.fidl: 10:8: Name <Bb>
Aa. Bb? Cc! Dd:
      ^
sdk/foo/foo.fidl: 10:10: EndOfSentence <?>
Aa. Bb? Cc! Dd:
        ^^
sdk/foo/foo.fidl: 10:12: Name <Cc>
Aa. Bb? Cc! Dd:
          ^
sdk/foo/foo.fidl: 10:14: EndOfSentence <!>
Aa. Bb? Cc! Dd:
            ^^
sdk/foo/foo.fidl: 10:16: Name <Dd>
Aa. Bb? Cc! Dd:
              ^
sdk/foo/foo.fidl: 10:18: EndOfSentence <:>
"
        );
    }

    #[test]
    fn lexer_spaces_and_new_lines() {
        let mut compiler = DocCompiler::new();
        let source = Rc::new(Source::new(
            "sdk/foo/foo.fidl".to_owned(),
            10,
            4,
            "Some     documentation.\n\n\nAnd   spaces.".to_owned(),
        ));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(!items.is_none());
        lexical_items_to_errors(&mut compiler, &items.unwrap(), /*with_spaces=*/ true);
        assert_eq!(
            compiler.errors,
            "\
Some     documentation.
^^^^
sdk/foo/foo.fidl: 10:4: Name <Some>
Some     documentation.
    ^^^^^
sdk/foo/foo.fidl: 10:8: Spaces (5)
Some     documentation.
         ^^^^^^^^^^^^^
sdk/foo/foo.fidl: 10:13: Name <documentation>
Some     documentation.
                      ^
sdk/foo/foo.fidl: 10:26: EndOfSentence <.>
Some     documentation.
                       ^
sdk/foo/foo.fidl: 10:27: NewLines (3)
And   spaces.
^^^
sdk/foo/foo.fidl: 13:4: Name <And>
And   spaces.
   ^^^
sdk/foo/foo.fidl: 13:7: Spaces (3)
And   spaces.
      ^^^^^^
sdk/foo/foo.fidl: 13:10: Name <spaces>
And   spaces.
            ^
sdk/foo/foo.fidl: 13:16: EndOfSentence <.>
And   spaces.
             ^
sdk/foo/foo.fidl: 13:17: End
"
        );
    }

    #[test]
    fn lexer_bad_character() {
        let mut compiler = DocCompiler::new();
        let source =
            Rc::new(Source::new("sdk/foo/foo.fidl".to_owned(), 10, 4, "En dash —.\n".to_owned()));
        let items = reduce_lexems(&mut compiler, &source);
        assert!(items.is_none());
        assert_eq!(
            compiler.errors,
            "\
En dash —.
        ^
sdk/foo/foo.fidl: 10:12: Unknown character <—>
"
        );
    }
}