| //! Convert macro-by-example tokens which are specific to macro expansion into a | |
| //! format that works for our parser. | |
| use rustc_hash::FxHashMap; | |
| use span::{Edition, SyntaxContext}; | |
| use syntax::{SyntaxKind, SyntaxKind::*, T}; | |
| pub fn to_parser_input( | |
| buffer: tt::TokenTreesView<'_>, | |
| span_to_edition: &mut dyn FnMut(SyntaxContext) -> Edition, | |
| ) -> parser::Input { | |
| let mut res = parser::Input::with_capacity(buffer.len()); | |
| let mut current = buffer.cursor(); | |
| let mut syntax_context_to_edition_cache = FxHashMap::default(); | |
| let mut ctx_edition = | |
| |ctx| *syntax_context_to_edition_cache.entry(ctx).or_insert_with(|| span_to_edition(ctx)); | |
| while !current.eof() { | |
| let tt = current.token_tree(); | |
| // Check if it is lifetime | |
| if let Some(tt::TokenTree::Leaf(tt::Leaf::Punct(punct))) = tt | |
| && punct.char == '\'' | |
| { | |
| current.bump(); | |
| match current.token_tree() { | |
| Some(tt::TokenTree::Leaf(tt::Leaf::Ident(ident))) => { | |
| res.push(LIFETIME_IDENT, ctx_edition(ident.span.ctx)); | |
| current.bump(); | |
| continue; | |
| } | |
| _ => panic!("Next token must be ident"), | |
| } | |
| } | |
| match tt { | |
| Some(tt::TokenTree::Leaf(leaf)) => { | |
| match leaf { | |
| tt::Leaf::Literal(lit) => { | |
| let kind = match lit.kind { | |
| tt::LitKind::Byte => SyntaxKind::BYTE, | |
| tt::LitKind::Char => SyntaxKind::CHAR, | |
| tt::LitKind::Integer => SyntaxKind::INT_NUMBER, | |
| tt::LitKind::Float => SyntaxKind::FLOAT_NUMBER, | |
| tt::LitKind::Str | tt::LitKind::StrRaw(_) => SyntaxKind::STRING, | |
| tt::LitKind::ByteStr | tt::LitKind::ByteStrRaw(_) => { | |
| SyntaxKind::BYTE_STRING | |
| } | |
| tt::LitKind::CStr | tt::LitKind::CStrRaw(_) => SyntaxKind::C_STRING, | |
| tt::LitKind::Err(_) => SyntaxKind::ERROR, | |
| }; | |
| res.push(kind, ctx_edition(lit.span.ctx)); | |
| if kind == FLOAT_NUMBER && !lit.text().ends_with('.') { | |
| // Tag the token as joint if it is float with a fractional part | |
| // we use this jointness to inform the parser about what token split | |
| // event to emit when we encounter a float literal in a field access | |
| res.was_joint(); | |
| } | |
| } | |
| tt::Leaf::Ident(ident) => { | |
| let edition = ctx_edition(ident.span.ctx); | |
| match ident.sym.as_str() { | |
| "_" => res.push(T![_], edition), | |
| i if i.starts_with('\'') => res.push(LIFETIME_IDENT, edition), | |
| _ if ident.is_raw.yes() => res.push(IDENT, edition), | |
| text => match SyntaxKind::from_keyword(text, edition) { | |
| Some(kind) => res.push(kind, edition), | |
| None => { | |
| let contextual_keyword = | |
| SyntaxKind::from_contextual_keyword(text, edition) | |
| .unwrap_or(SyntaxKind::IDENT); | |
| res.push_ident(contextual_keyword, edition); | |
| } | |
| }, | |
| } | |
| } | |
| tt::Leaf::Punct(punct) => { | |
| let kind = SyntaxKind::from_char(punct.char) | |
| .unwrap_or_else(|| panic!("{punct:#?} is not a valid punct")); | |
| res.push(kind, ctx_edition(punct.span.ctx)); | |
| if punct.spacing == tt::Spacing::Joint { | |
| res.was_joint(); | |
| } | |
| } | |
| } | |
| current.bump(); | |
| } | |
| Some(tt::TokenTree::Subtree(subtree)) => { | |
| if let Some(kind) = match subtree.delimiter.kind { | |
| tt::DelimiterKind::Parenthesis => Some(T!['(']), | |
| tt::DelimiterKind::Brace => Some(T!['{']), | |
| tt::DelimiterKind::Bracket => Some(T!['[']), | |
| tt::DelimiterKind::Invisible => None, | |
| } { | |
| res.push(kind, ctx_edition(subtree.delimiter.open.ctx)); | |
| } | |
| current.bump(); | |
| } | |
| None => { | |
| let subtree = current.end(); | |
| if let Some(kind) = match subtree.delimiter.kind { | |
| tt::DelimiterKind::Parenthesis => Some(T![')']), | |
| tt::DelimiterKind::Brace => Some(T!['}']), | |
| tt::DelimiterKind::Bracket => Some(T![']']), | |
| tt::DelimiterKind::Invisible => None, | |
| } { | |
| res.push(kind, ctx_edition(subtree.delimiter.close.ctx)); | |
| } | |
| } | |
| }; | |
| } | |
| res | |
| } |