Auto merge of #15303 - oxalica:fix/byte-escape-highlight, r=lowr
Fix highlighting of byte escape sequences
Currently non-UTF8 escape sequences in byte strings and any escape sequences in byte literals are ignored.
diff --git a/crates/hir-def/src/body/pretty.rs b/crates/hir-def/src/body/pretty.rs
index 0c6cf0b..eeaed87 100644
--- a/crates/hir-def/src/body/pretty.rs
+++ b/crates/hir-def/src/body/pretty.rs
@@ -634,7 +634,7 @@
match literal {
Literal::String(it) => w!(self, "{:?}", it),
Literal::ByteString(it) => w!(self, "\"{}\"", it.escape_ascii()),
- Literal::CString(it) => w!(self, "\"{}\\0\"", it),
+ Literal::CString(it) => w!(self, "\"{}\\0\"", it.escape_ascii()),
Literal::Char(it) => w!(self, "'{}'", it.escape_debug()),
Literal::Bool(it) => w!(self, "{}", it),
Literal::Int(i, suffix) => {
diff --git a/crates/hir-def/src/hir.rs b/crates/hir-def/src/hir.rs
index 500e880..8a140a1 100644
--- a/crates/hir-def/src/hir.rs
+++ b/crates/hir-def/src/hir.rs
@@ -85,7 +85,7 @@
pub enum Literal {
String(Box<str>),
ByteString(Box<[u8]>),
- CString(Box<str>),
+ CString(Box<[u8]>),
Char(char),
Bool(bool),
Int(i128, Option<BuiltinInt>),
diff --git a/crates/hir-ty/src/mir/lower.rs b/crates/hir-ty/src/mir/lower.rs
index e443c58..9f25175 100644
--- a/crates/hir-ty/src/mir/lower.rs
+++ b/crates/hir-ty/src/mir/lower.rs
@@ -1355,7 +1355,6 @@
return Ok(Operand::from_concrete_const(data, mm, ty));
}
hir_def::hir::Literal::CString(b) => {
- let b = b.as_bytes();
let bytes = b.iter().copied().chain(iter::once(0)).collect::<Vec<_>>();
let mut data = Vec::with_capacity(mem::size_of::<usize>() * 2);
diff --git a/crates/ide/src/syntax_highlighting.rs b/crates/ide/src/syntax_highlighting.rs
index 577bd2b..ae97236 100644
--- a/crates/ide/src/syntax_highlighting.rs
+++ b/crates/ide/src/syntax_highlighting.rs
@@ -24,7 +24,7 @@
use crate::{
syntax_highlighting::{
- escape::{highlight_escape_char, highlight_escape_string},
+ escape::{highlight_escape_byte, highlight_escape_char, highlight_escape_string},
format::highlight_format_string,
highlights::Highlights,
macro_::MacroHighlighter,
@@ -471,6 +471,14 @@
};
highlight_escape_char(hl, &char, range.start())
+ } else if ast::Byte::can_cast(token.kind())
+ && ast::Byte::can_cast(descended_token.kind())
+ {
+ let Some(byte) = ast::Byte::cast(token) else {
+ continue;
+ };
+
+ highlight_escape_byte(hl, &byte, range.start())
}
}
diff --git a/crates/ide/src/syntax_highlighting/escape.rs b/crates/ide/src/syntax_highlighting/escape.rs
index 211e358..5913ca5 100644
--- a/crates/ide/src/syntax_highlighting/escape.rs
+++ b/crates/ide/src/syntax_highlighting/escape.rs
@@ -1,7 +1,7 @@
//! Syntax highlighting for escape sequences
use crate::syntax_highlighting::highlights::Highlights;
use crate::{HlRange, HlTag};
-use syntax::ast::{Char, IsString};
+use syntax::ast::{Byte, Char, IsString};
use syntax::{AstToken, TextRange, TextSize};
pub(super) fn highlight_escape_string<T: IsString>(
@@ -10,14 +10,14 @@
start: TextSize,
) {
string.escaped_char_ranges(&mut |piece_range, char| {
- if char.is_err() {
- return;
- }
-
if string.text()[piece_range.start().into()..].starts_with('\\') {
+ let highlight = match char {
+ Ok(_) => HlTag::EscapeSequence,
+ Err(_) => HlTag::InvalidEscapeSequence,
+ };
stack.add(HlRange {
range: piece_range + start,
- highlight: HlTag::EscapeSequence.into(),
+ highlight: highlight.into(),
binding_hash: None,
});
}
@@ -26,6 +26,9 @@
pub(super) fn highlight_escape_char(stack: &mut Highlights, char: &Char, start: TextSize) {
if char.value().is_none() {
+ // We do not emit invalid escapes highlighting here. The lexer would likely be in a bad
+ // state and this token contains junks, since `'` is not a reliable delimiter (consider
+ // lifetimes). Nonetheless, parser errors should already be emitted.
return;
}
@@ -43,3 +46,24 @@
TextRange::new(start + TextSize::from(1), start + TextSize::from(text.len() as u32 + 1));
stack.add(HlRange { range, highlight: HlTag::EscapeSequence.into(), binding_hash: None })
}
+
+pub(super) fn highlight_escape_byte(stack: &mut Highlights, byte: &Byte, start: TextSize) {
+ if byte.value().is_none() {
+ // See `highlight_escape_char` for why no error highlighting here.
+ return;
+ }
+
+ let text = byte.text();
+ if !text.starts_with("b'") || !text.ends_with('\'') {
+ return;
+ }
+
+ let text = &text[2..text.len() - 1];
+ if !text.starts_with('\\') {
+ return;
+ }
+
+ let range =
+ TextRange::new(start + TextSize::from(2), start + TextSize::from(text.len() as u32 + 2));
+ stack.add(HlRange { range, highlight: HlTag::EscapeSequence.into(), binding_hash: None })
+}
diff --git a/crates/ide/src/syntax_highlighting/html.rs b/crates/ide/src/syntax_highlighting/html.rs
index 2c78230..bbc6b55 100644
--- a/crates/ide/src/syntax_highlighting/html.rs
+++ b/crates/ide/src/syntax_highlighting/html.rs
@@ -109,6 +109,7 @@
.control { font-style: italic; }
.reference { font-style: italic; font-weight: bold; }
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
</style>
";
diff --git a/crates/ide/src/syntax_highlighting/tags.rs b/crates/ide/src/syntax_highlighting/tags.rs
index f983109..6d4cdd0 100644
--- a/crates/ide/src/syntax_highlighting/tags.rs
+++ b/crates/ide/src/syntax_highlighting/tags.rs
@@ -29,6 +29,7 @@
Comment,
EscapeSequence,
FormatSpecifier,
+ InvalidEscapeSequence,
Keyword,
NumericLiteral,
Operator(HlOperator),
@@ -166,6 +167,7 @@
HlTag::CharLiteral => "char_literal",
HlTag::Comment => "comment",
HlTag::EscapeSequence => "escape_sequence",
+ HlTag::InvalidEscapeSequence => "invalid_escape_sequence",
HlTag::FormatSpecifier => "format_specifier",
HlTag::Keyword => "keyword",
HlTag::Punctuation(punct) => match punct {
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_assoc_functions.html b/crates/ide/src/syntax_highlighting/test_data/highlight_assoc_functions.html
index 9ed65fb..4dcbfe4 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_assoc_functions.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_assoc_functions.html
@@ -40,7 +40,8 @@
.control { font-style: italic; }
.reference { font-style: italic; font-weight: bold; }
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
</style>
<pre><code><span class="keyword">fn</span> <span class="function declaration">not_static</span><span class="parenthesis">(</span><span class="parenthesis">)</span> <span class="brace">{</span><span class="brace">}</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_attributes.html b/crates/ide/src/syntax_highlighting/test_data/highlight_attributes.html
index 567ab8c..bf5505c 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_attributes.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_attributes.html
@@ -40,7 +40,8 @@
.control { font-style: italic; }
.reference { font-style: italic; font-weight: bold; }
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
</style>
<pre><code><span class="attribute_bracket attribute">#</span><span class="attribute_bracket attribute">[</span><span class="builtin_attr attribute library">allow</span><span class="parenthesis attribute">(</span><span class="none attribute">dead_code</span><span class="parenthesis attribute">)</span><span class="attribute_bracket attribute">]</span>
<span class="attribute_bracket attribute">#</span><span class="attribute_bracket attribute">[</span><span class="tool_module attribute library">rustfmt</span><span class="operator attribute">::</span><span class="tool_module attribute library">skip</span><span class="attribute_bracket attribute">]</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_crate_root.html b/crates/ide/src/syntax_highlighting/test_data/highlight_crate_root.html
index 1e4c06d..0d1b3c1 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_crate_root.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_crate_root.html
@@ -40,7 +40,8 @@
.control { font-style: italic; }
.reference { font-style: italic; font-weight: bold; }
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
</style>
<pre><code><span class="keyword">extern</span> <span class="keyword">crate</span> <span class="module crate_root library">foo</span><span class="semicolon">;</span>
<span class="keyword">use</span> <span class="module crate_root default_library library">core</span><span class="operator">::</span><span class="module default_library library">iter</span><span class="semicolon">;</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_default_library.html b/crates/ide/src/syntax_highlighting/test_data/highlight_default_library.html
index 5d66f83..dd1528e 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_default_library.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_default_library.html
@@ -40,7 +40,8 @@
.control { font-style: italic; }
.reference { font-style: italic; font-weight: bold; }
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
</style>
<pre><code><span class="keyword">use</span> <span class="module crate_root default_library library">core</span><span class="operator">::</span><span class="module default_library library">iter</span><span class="semicolon">;</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html b/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html
index 35f240d..d5f92aa 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html
@@ -40,7 +40,8 @@
.control { font-style: italic; }
.reference { font-style: italic; font-weight: bold; }
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
</style>
<pre><code><span class="comment documentation">//! This is a module to test doc injection.</span>
<span class="comment documentation">//! ```</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_extern_crate.html b/crates/ide/src/syntax_highlighting/test_data/highlight_extern_crate.html
index 87b9da4..b15f7bc 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_extern_crate.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_extern_crate.html
@@ -40,7 +40,8 @@
.control { font-style: italic; }
.reference { font-style: italic; font-weight: bold; }
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
</style>
<pre><code><span class="keyword">extern</span> <span class="keyword">crate</span> <span class="module crate_root default_library library">std</span><span class="semicolon">;</span>
<span class="keyword">extern</span> <span class="keyword">crate</span> <span class="module crate_root default_library library">alloc</span> <span class="keyword">as</span> <span class="module crate_root default_library declaration library">abc</span><span class="semicolon">;</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_general.html b/crates/ide/src/syntax_highlighting/test_data/highlight_general.html
index 6b049f3..bdeb09d 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_general.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_general.html
@@ -40,7 +40,8 @@
.control { font-style: italic; }
.reference { font-style: italic; font-weight: bold; }
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
</style>
<pre><code><span class="keyword">use</span> <span class="module">inner</span><span class="operator">::</span><span class="brace">{</span><span class="self_keyword">self</span> <span class="keyword">as</span> <span class="module declaration">inner_mod</span><span class="brace">}</span><span class="semicolon">;</span>
<span class="keyword">mod</span> <span class="module declaration">inner</span> <span class="brace">{</span><span class="brace">}</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_injection.html b/crates/ide/src/syntax_highlighting/test_data/highlight_injection.html
index d9c3db6..f9c33b8 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_injection.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_injection.html
@@ -40,7 +40,8 @@
.control { font-style: italic; }
.reference { font-style: italic; font-weight: bold; }
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
</style>
<pre><code><span class="keyword">fn</span> <span class="function declaration">fixture</span><span class="parenthesis">(</span><span class="value_param declaration reference">ra_fixture</span><span class="colon">:</span> <span class="punctuation">&</span><span class="builtin_type">str</span><span class="parenthesis">)</span> <span class="brace">{</span><span class="brace">}</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_keywords.html b/crates/ide/src/syntax_highlighting/test_data/highlight_keywords.html
index 3900959..fd3b398 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_keywords.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_keywords.html
@@ -40,7 +40,8 @@
.control { font-style: italic; }
.reference { font-style: italic; font-weight: bold; }
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
</style>
<pre><code><span class="keyword">extern</span> <span class="keyword">crate</span> <span class="self_keyword crate_root public">self</span><span class="semicolon">;</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_lifetimes.html b/crates/ide/src/syntax_highlighting/test_data/highlight_lifetimes.html
index f98e0b1..ec39998 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_lifetimes.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_lifetimes.html
@@ -40,7 +40,8 @@
.control { font-style: italic; }
.reference { font-style: italic; font-weight: bold; }
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
</style>
<pre><code>
<span class="attribute_bracket attribute">#</span><span class="attribute_bracket attribute">[</span><span class="attribute attribute default_library library">derive</span><span class="parenthesis attribute">(</span><span class="parenthesis attribute">)</span><span class="attribute_bracket attribute">]</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_macros.html b/crates/ide/src/syntax_highlighting/test_data/highlight_macros.html
index 2cbbf69..c5fcec7 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_macros.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_macros.html
@@ -40,7 +40,8 @@
.control { font-style: italic; }
.reference { font-style: italic; font-weight: bold; }
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
</style>
<pre><code><span class="module crate_root library">proc_macros</span><span class="operator">::</span><span class="macro library">mirror</span><span class="macro_bang">!</span> <span class="brace macro">{</span>
<span class="brace macro">{</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_inline.html b/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_inline.html
index 8a1d698..4dcf8e5 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_inline.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_inline.html
@@ -40,7 +40,8 @@
.control { font-style: italic; }
.reference { font-style: italic; font-weight: bold; }
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
</style>
<pre><code><span class="comment documentation">//! </span><span class="struct documentation injected intra_doc_link">[Struct]</span>
<span class="comment documentation">//! This is an intra doc injection test for modules</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_outline.html b/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_outline.html
index c4c3e3d..084bbf2 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_outline.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_outline.html
@@ -40,7 +40,8 @@
.control { font-style: italic; }
.reference { font-style: italic; font-weight: bold; }
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
</style>
<pre><code><span class="comment documentation">/// </span><span class="struct documentation injected intra_doc_link">[crate::foo::Struct]</span>
<span class="comment documentation">/// This is an intra doc injection test for modules</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_operators.html b/crates/ide/src/syntax_highlighting/test_data/highlight_operators.html
index 2369071..1af4bcf 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_operators.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_operators.html
@@ -40,7 +40,8 @@
.control { font-style: italic; }
.reference { font-style: italic; font-weight: bold; }
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
</style>
<pre><code><span class="keyword">fn</span> <span class="function declaration">main</span><span class="parenthesis">(</span><span class="parenthesis">)</span> <span class="brace">{</span>
<span class="numeric_literal">1</span> <span class="arithmetic">+</span> <span class="numeric_literal">1</span> <span class="arithmetic">-</span> <span class="numeric_literal">1</span> <span class="arithmetic">*</span> <span class="numeric_literal">1</span> <span class="arithmetic">/</span> <span class="numeric_literal">1</span> <span class="arithmetic">%</span> <span class="numeric_literal">1</span> <span class="bitwise">|</span> <span class="numeric_literal">1</span> <span class="bitwise">&</span> <span class="numeric_literal">1</span> <span class="logical">!</span> <span class="numeric_literal">1</span> <span class="bitwise">^</span> <span class="numeric_literal">1</span> <span class="bitwise">>></span> <span class="numeric_literal">1</span> <span class="bitwise"><<</span> <span class="numeric_literal">1</span><span class="semicolon">;</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_rainbow.html b/crates/ide/src/syntax_highlighting/test_data/highlight_rainbow.html
index bff35c8..ec18c3e 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_rainbow.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_rainbow.html
@@ -40,7 +40,8 @@
.control { font-style: italic; }
.reference { font-style: italic; font-weight: bold; }
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
</style>
<pre><code><span class="keyword">fn</span> <span class="function declaration">main</span><span class="parenthesis">(</span><span class="parenthesis">)</span> <span class="brace">{</span>
<span class="keyword">let</span> <span class="variable declaration reference" data-binding-hash="8121853618659664005" style="color: hsl(273,88%,88%);">hello</span> <span class="operator">=</span> <span class="string_literal">"hello"</span><span class="semicolon">;</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
index f4f164a..dcac8eb 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
@@ -40,7 +40,8 @@
.control { font-style: italic; }
.reference { font-style: italic; font-weight: bold; }
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
</style>
<pre><code><span class="keyword">macro_rules</span><span class="macro_bang">!</span> <span class="macro declaration">println</span> <span class="brace">{</span>
<span class="parenthesis">(</span><span class="punctuation">$</span><span class="parenthesis">(</span><span class="punctuation">$</span>arg<span class="colon">:</span>tt<span class="parenthesis">)</span><span class="punctuation">*</span><span class="parenthesis">)</span> <span class="operator">=</span><span class="angle">></span> <span class="parenthesis">(</span><span class="brace">{</span>
@@ -105,6 +106,8 @@
<span class="keyword">let</span> <span class="variable declaration">a</span> <span class="operator">=</span> <span class="char_literal">'</span><span class="escape_sequence">\x65</span><span class="char_literal">'</span><span class="semicolon">;</span>
<span class="keyword">let</span> <span class="variable declaration">a</span> <span class="operator">=</span> <span class="char_literal">'</span><span class="escape_sequence">\x00</span><span class="char_literal">'</span><span class="semicolon">;</span>
+ <span class="keyword">let</span> <span class="variable declaration">a</span> <span class="operator">=</span> <span class="byte_literal">b'</span><span class="escape_sequence">\xFF</span><span class="byte_literal">'</span><span class="semicolon">;</span>
+
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"Hello </span><span class="escape_sequence">{{</span><span class="string_literal macro">Hello</span><span class="escape_sequence">}}</span><span class="string_literal macro">"</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>
<span class="comment">// from https://doc.rust-lang.org/std/fmt/index.html</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"Hello"</span><span class="parenthesis macro">)</span><span class="semicolon">;</span> <span class="comment">// => "Hello"</span>
@@ -159,8 +162,9 @@
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"Hello</span><span class="escape_sequence">\n</span><span class="string_literal macro">World"</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"</span><span class="escape_sequence">\u{48}</span><span class="escape_sequence">\x65</span><span class="escape_sequence">\x6C</span><span class="escape_sequence">\x6C</span><span class="escape_sequence">\x6F</span><span class="string_literal macro"> World"</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>
- <span class="keyword">let</span> <span class="punctuation">_</span> <span class="operator">=</span> <span class="string_literal">"</span><span class="escape_sequence">\x28</span><span class="escape_sequence">\x28</span><span class="escape_sequence">\x00</span><span class="escape_sequence">\x63</span><span class="escape_sequence">\n</span><span class="string_literal">"</span><span class="semicolon">;</span>
- <span class="keyword">let</span> <span class="punctuation">_</span> <span class="operator">=</span> <span class="string_literal">b"</span><span class="escape_sequence">\x28</span><span class="escape_sequence">\x28</span><span class="escape_sequence">\x00</span><span class="escape_sequence">\x63</span><span class="escape_sequence">\n</span><span class="string_literal">"</span><span class="semicolon">;</span>
+ <span class="keyword">let</span> <span class="punctuation">_</span> <span class="operator">=</span> <span class="string_literal">"</span><span class="escape_sequence">\x28</span><span class="escape_sequence">\x28</span><span class="escape_sequence">\x00</span><span class="escape_sequence">\x63</span><span class="invalid_escape_sequence">\xFF</span><span class="escape_sequence">\u{FF}</span><span class="escape_sequence">\n</span><span class="string_literal">"</span><span class="semicolon">;</span> <span class="comment">// invalid non-UTF8 escape sequences</span>
+ <span class="keyword">let</span> <span class="punctuation">_</span> <span class="operator">=</span> <span class="string_literal">b"</span><span class="escape_sequence">\x28</span><span class="escape_sequence">\x28</span><span class="escape_sequence">\x00</span><span class="escape_sequence">\x63</span><span class="escape_sequence">\xFF</span><span class="invalid_escape_sequence">\u{FF}</span><span class="escape_sequence">\n</span><span class="string_literal">"</span><span class="semicolon">;</span> <span class="comment">// valid bytes, invalid unicodes</span>
+ <span class="keyword">let</span> <span class="punctuation">_</span> <span class="operator">=</span> <span class="string_literal">c"</span><span class="escape_sequence">\u{FF}</span><span class="escape_sequence">\xFF</span><span class="string_literal">"</span><span class="semicolon">;</span> <span class="comment">// valid bytes, valid unicodes</span>
<span class="keyword">let</span> <span class="variable declaration reference">backslash</span> <span class="operator">=</span> <span class="string_literal">r"\\"</span><span class="semicolon">;</span>
<span class="macro">println</span><span class="macro_bang">!</span><span class="parenthesis macro">(</span><span class="string_literal macro">"</span><span class="format_specifier">{</span><span class="escape_sequence">\x41</span><span class="format_specifier">}</span><span class="string_literal macro">"</span><span class="comma macro">,</span> <span class="none macro">A</span> <span class="operator macro">=</span> <span class="numeric_literal macro">92</span><span class="parenthesis macro">)</span><span class="semicolon">;</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_unsafe.html b/crates/ide/src/syntax_highlighting/test_data/highlight_unsafe.html
index 654d51b..c72ea54 100644
--- a/crates/ide/src/syntax_highlighting/test_data/highlight_unsafe.html
+++ b/crates/ide/src/syntax_highlighting/test_data/highlight_unsafe.html
@@ -40,7 +40,8 @@
.control { font-style: italic; }
.reference { font-style: italic; font-weight: bold; }
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
</style>
<pre><code><span class="keyword">macro_rules</span><span class="macro_bang">!</span> <span class="macro declaration">id</span> <span class="brace">{</span>
<span class="parenthesis">(</span><span class="punctuation">$</span><span class="parenthesis">(</span><span class="punctuation">$</span>tt<span class="colon">:</span>tt<span class="parenthesis">)</span><span class="punctuation">*</span><span class="parenthesis">)</span> <span class="operator">=</span><span class="angle">></span> <span class="brace">{</span>
diff --git a/crates/ide/src/syntax_highlighting/tests.rs b/crates/ide/src/syntax_highlighting/tests.rs
index 1ee451a..696aa59 100644
--- a/crates/ide/src/syntax_highlighting/tests.rs
+++ b/crates/ide/src/syntax_highlighting/tests.rs
@@ -451,6 +451,8 @@
let a = '\x65';
let a = '\x00';
+ let a = b'\xFF';
+
println!("Hello {{Hello}}");
// from https://doc.rust-lang.org/std/fmt/index.html
println!("Hello"); // => "Hello"
@@ -505,8 +507,9 @@
println!("Hello\nWorld");
println!("\u{48}\x65\x6C\x6C\x6F World");
- let _ = "\x28\x28\x00\x63\n";
- let _ = b"\x28\x28\x00\x63\n";
+ let _ = "\x28\x28\x00\x63\xFF\u{FF}\n"; // invalid non-UTF8 escape sequences
+ let _ = b"\x28\x28\x00\x63\xFF\u{FF}\n"; // valid bytes, invalid unicodes
+ let _ = c"\u{FF}\xFF"; // valid bytes, valid unicodes
let backslash = r"\\";
println!("{\x41}", A = 92);
diff --git a/crates/rust-analyzer/src/semantic_tokens.rs b/crates/rust-analyzer/src/semantic_tokens.rs
index d4bb20c..1fe02fc 100644
--- a/crates/rust-analyzer/src/semantic_tokens.rs
+++ b/crates/rust-analyzer/src/semantic_tokens.rs
@@ -78,6 +78,7 @@
(DERIVE_HELPER, "deriveHelper") => DECORATOR,
(DOT, "dot"),
(ESCAPE_SEQUENCE, "escapeSequence") => STRING,
+ (INVALID_ESCAPE_SEQUENCE, "invalidEscapeSequence") => STRING,
(FORMAT_SPECIFIER, "formatSpecifier") => STRING,
(GENERIC, "generic") => TYPE_PARAMETER,
(LABEL, "label"),
diff --git a/crates/rust-analyzer/src/to_proto.rs b/crates/rust-analyzer/src/to_proto.rs
index ba3421b..7a89533 100644
--- a/crates/rust-analyzer/src/to_proto.rs
+++ b/crates/rust-analyzer/src/to_proto.rs
@@ -640,6 +640,7 @@
HlTag::CharLiteral => semantic_tokens::CHAR,
HlTag::Comment => semantic_tokens::COMMENT,
HlTag::EscapeSequence => semantic_tokens::ESCAPE_SEQUENCE,
+ HlTag::InvalidEscapeSequence => semantic_tokens::INVALID_ESCAPE_SEQUENCE,
HlTag::FormatSpecifier => semantic_tokens::FORMAT_SPECIFIER,
HlTag::Keyword => semantic_tokens::KEYWORD,
HlTag::None => semantic_tokens::GENERIC,
diff --git a/crates/syntax/src/ast/token_ext.rs b/crates/syntax/src/ast/token_ext.rs
index 090eb89..87fd51d 100644
--- a/crates/syntax/src/ast/token_ext.rs
+++ b/crates/syntax/src/ast/token_ext.rs
@@ -2,7 +2,9 @@
use std::borrow::Cow;
-use rustc_lexer::unescape::{unescape_byte, unescape_char, unescape_literal, Mode};
+use rustc_lexer::unescape::{
+ unescape_byte, unescape_c_string, unescape_char, unescape_literal, CStrUnit, Mode,
+};
use crate::{
ast::{self, AstToken},
@@ -146,6 +148,7 @@
pub trait IsString: AstToken {
const RAW_PREFIX: &'static str;
+ const MODE: Mode;
fn is_raw(&self) -> bool {
self.text().starts_with(Self::RAW_PREFIX)
}
@@ -181,7 +184,7 @@
let text = &self.text()[text_range_no_quotes - start];
let offset = text_range_no_quotes.start() - start;
- unescape_literal(text, Mode::Str, &mut |range, unescaped_char| {
+ unescape_literal(text, Self::MODE, &mut |range, unescaped_char| {
let text_range =
TextRange::new(range.start.try_into().unwrap(), range.end.try_into().unwrap());
cb(text_range + offset, unescaped_char);
@@ -196,6 +199,7 @@
impl IsString for ast::String {
const RAW_PREFIX: &'static str = "r";
+ const MODE: Mode = Mode::Str;
}
impl ast::String {
@@ -213,7 +217,7 @@
let mut buf = String::new();
let mut prev_end = 0;
let mut has_error = false;
- unescape_literal(text, Mode::Str, &mut |char_range, unescaped_char| match (
+ unescape_literal(text, Self::MODE, &mut |char_range, unescaped_char| match (
unescaped_char,
buf.capacity() == 0,
) {
@@ -239,6 +243,7 @@
impl IsString for ast::ByteString {
const RAW_PREFIX: &'static str = "br";
+ const MODE: Mode = Mode::ByteStr;
}
impl ast::ByteString {
@@ -256,7 +261,7 @@
let mut buf: Vec<u8> = Vec::new();
let mut prev_end = 0;
let mut has_error = false;
- unescape_literal(text, Mode::ByteStr, &mut |char_range, unescaped_char| match (
+ unescape_literal(text, Self::MODE, &mut |char_range, unescaped_char| match (
unescaped_char,
buf.capacity() == 0,
) {
@@ -282,42 +287,70 @@
impl IsString for ast::CString {
const RAW_PREFIX: &'static str = "cr";
+ const MODE: Mode = Mode::CStr;
+
+ fn escaped_char_ranges(
+ &self,
+ cb: &mut dyn FnMut(TextRange, Result<char, rustc_lexer::unescape::EscapeError>),
+ ) {
+ let text_range_no_quotes = match self.text_range_between_quotes() {
+ Some(it) => it,
+ None => return,
+ };
+
+ let start = self.syntax().text_range().start();
+ let text = &self.text()[text_range_no_quotes - start];
+ let offset = text_range_no_quotes.start() - start;
+
+ unescape_c_string(text, Self::MODE, &mut |range, unescaped_char| {
+ let text_range =
+ TextRange::new(range.start.try_into().unwrap(), range.end.try_into().unwrap());
+ // XXX: This method should only be used for highlighting ranges. The unescaped
+ // char/byte is not used. For simplicity, we return an arbitrary placeholder char.
+ cb(text_range + offset, unescaped_char.map(|_| ' '));
+ });
+ }
}
impl ast::CString {
- pub fn value(&self) -> Option<Cow<'_, str>> {
+ pub fn value(&self) -> Option<Cow<'_, [u8]>> {
if self.is_raw() {
let text = self.text();
let text =
&text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
- return Some(Cow::Borrowed(text));
+ return Some(Cow::Borrowed(text.as_bytes()));
}
let text = self.text();
let text = &text[self.text_range_between_quotes()? - self.syntax().text_range().start()];
- let mut buf = String::new();
+ let mut buf = Vec::new();
let mut prev_end = 0;
let mut has_error = false;
- unescape_literal(text, Mode::Str, &mut |char_range, unescaped_char| match (
- unescaped_char,
+ let mut char_buf = [0u8; 4];
+ let mut extend_unit = |buf: &mut Vec<u8>, unit: CStrUnit| match unit {
+ CStrUnit::Byte(b) => buf.push(b),
+ CStrUnit::Char(c) => buf.extend(c.encode_utf8(&mut char_buf).as_bytes()),
+ };
+ unescape_c_string(text, Self::MODE, &mut |char_range, unescaped| match (
+ unescaped,
buf.capacity() == 0,
) {
- (Ok(c), false) => buf.push(c),
+ (Ok(u), false) => extend_unit(&mut buf, u),
(Ok(_), true) if char_range.len() == 1 && char_range.start == prev_end => {
prev_end = char_range.end
}
- (Ok(c), true) => {
+ (Ok(u), true) => {
buf.reserve_exact(text.len());
- buf.push_str(&text[..prev_end]);
- buf.push(c);
+ buf.extend(text[..prev_end].as_bytes());
+ extend_unit(&mut buf, u);
}
(Err(_), _) => has_error = true,
});
match (has_error, buf.capacity() == 0) {
(true, _) => None,
- (false, true) => Some(Cow::Borrowed(text)),
+ (false, true) => Some(Cow::Borrowed(text.as_bytes())),
(false, false) => Some(Cow::Owned(buf)),
}
}
diff --git a/editors/code/package.json b/editors/code/package.json
index 31c483b..a489789 100644
--- a/editors/code/package.json
+++ b/editors/code/package.json
@@ -1804,13 +1804,17 @@
},
{
"id": "escapeSequence",
- "description": "Style for char escapes in strings"
+ "description": "Style for char or byte escapes in strings"
},
{
"id": "formatSpecifier",
"description": "Style for {} placeholders in format strings"
},
{
+ "id": "invalidEscapeSequence",
+ "description": "Style for invalid char or byte escapes in strings"
+ },
+ {
"id": "label",
"description": "Style for labels"
},
