feat: add syntax highlighting to text buffers
This commit is contained in:
Valdemar Lorenzen
+703
@@ -0,0 +1,703 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::LazyLock;
|
||||
use syntect::highlighting::ScopeSelectors;
|
||||
use syntect::parsing::{
|
||||
ParseState, ScopeError, ScopeStack, ScopeStackOp, SyntaxReference, SyntaxSet,
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub enum SyntaxKind {
|
||||
Comment,
|
||||
String,
|
||||
Keyword,
|
||||
Number,
|
||||
Type,
|
||||
Function,
|
||||
Markup,
|
||||
}
|
||||
|
||||
/// A semantic foreground range in the original line's bytes, excluding its newline.
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub struct SyntaxSpan {
|
||||
pub start_byte: usize,
|
||||
pub end_byte: usize,
|
||||
pub kind: SyntaxKind,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct CachedLine {
|
||||
source: String,
|
||||
spans: Vec<SyntaxSpan>,
|
||||
after: Option<(ParseState, ScopeStack)>,
|
||||
}
|
||||
|
||||
/// Frontend-owned, per-buffer parser state. Revision and path identify unchanged input.
|
||||
#[derive(Default)]
|
||||
pub struct SyntaxCache {
|
||||
revision: Option<u64>,
|
||||
path: Option<PathBuf>,
|
||||
syntax: Option<&'static SyntaxReference>,
|
||||
lines: Vec<CachedLine>,
|
||||
parse_line: String,
|
||||
error: Option<String>,
|
||||
#[cfg(test)]
|
||||
parsed_lines: usize,
|
||||
}
|
||||
|
||||
impl SyntaxCache {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// `revision` must change whenever line content changes; path changes are detected separately.
|
||||
pub fn update(&mut self, lines: &[String], path: Option<&Path>, revision: u64) {
|
||||
let path_changed = self.path.as_deref() != path;
|
||||
if self.revision == Some(revision) && !path_changed {
|
||||
return;
|
||||
}
|
||||
|
||||
let prefix = self
|
||||
.lines
|
||||
.iter()
|
||||
.zip(lines)
|
||||
.take_while(|(cached, source)| cached.source == **source)
|
||||
.count();
|
||||
let source_changed = prefix != self.lines.len() || prefix != lines.len();
|
||||
let syntaxes = syntax_set();
|
||||
let syntax = if self.revision.is_none() || path_changed || (source_changed && prefix == 0) {
|
||||
detect_syntax(syntaxes, path, lines.first().map(String::as_str))
|
||||
} else {
|
||||
self.syntax
|
||||
};
|
||||
let syntax_changed = self.language() != syntax.map(|syntax| syntax.name.as_str());
|
||||
|
||||
self.revision = Some(revision);
|
||||
if path_changed {
|
||||
self.path = path.map(Path::to_path_buf);
|
||||
}
|
||||
self.syntax = syntax;
|
||||
if !source_changed && !syntax_changed {
|
||||
return;
|
||||
}
|
||||
|
||||
self.lines.truncate(lines.len());
|
||||
for (row, source) in lines.iter().enumerate().skip(prefix) {
|
||||
if let Some(cached) = self.lines.get_mut(row) {
|
||||
if cached.source != *source {
|
||||
cached.source.clone_from(source);
|
||||
}
|
||||
} else {
|
||||
self.lines.push(CachedLine {
|
||||
source: source.clone(),
|
||||
..CachedLine::default()
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let Some(syntax) = syntax else {
|
||||
self.clear_highlighting();
|
||||
self.error = None;
|
||||
return;
|
||||
};
|
||||
let start = if syntax_changed || self.error.is_some() {
|
||||
0
|
||||
} else {
|
||||
prefix
|
||||
};
|
||||
self.error = None;
|
||||
if start < self.lines.len() {
|
||||
self.parse_from(start, syntax, syntaxes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Sorted, non-overlapping, end-exclusive ranges. Unclassified text uses the normal foreground.
|
||||
pub fn line(&self, row: usize) -> &[SyntaxSpan] {
|
||||
self.lines.get(row).map_or(&[], |line| &line.spans)
|
||||
}
|
||||
|
||||
/// Bundled grammar name, or `None` for unknown languages and Plain Text.
|
||||
pub fn language(&self) -> Option<&str> {
|
||||
self.syntax.map(|syntax| syntax.name.as_str())
|
||||
}
|
||||
|
||||
/// A genuine parser/scope error disables highlighting for the whole buffer until input changes.
|
||||
pub fn error(&self) -> Option<&str> {
|
||||
self.error.as_deref()
|
||||
}
|
||||
|
||||
fn clear_highlighting(&mut self) {
|
||||
for line in &mut self.lines {
|
||||
line.spans.clear();
|
||||
line.after = None;
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_from(&mut self, start: usize, syntax: &SyntaxReference, syntaxes: &SyntaxSet) {
|
||||
let (mut parser, mut scopes) = if start == 0 {
|
||||
(ParseState::new(syntax), ScopeStack::new())
|
||||
} else {
|
||||
self.lines[start - 1]
|
||||
.after
|
||||
.as_ref()
|
||||
.expect("unchanged prefix has a parser checkpoint")
|
||||
.clone()
|
||||
};
|
||||
|
||||
// ponytail: parse the dirty suffix to EOF; stop at converged checkpoints if this becomes costly.
|
||||
for row in start..self.lines.len() {
|
||||
let cached = &mut self.lines[row];
|
||||
self.parse_line.clear();
|
||||
self.parse_line.push_str(&cached.source);
|
||||
self.parse_line.push('\n');
|
||||
#[cfg(test)]
|
||||
{
|
||||
self.parsed_lines += 1;
|
||||
}
|
||||
let result = parser
|
||||
.parse_line(&self.parse_line, syntaxes)
|
||||
.map_err(|error| error.to_string())
|
||||
.and_then(|changes| {
|
||||
collect_spans(
|
||||
cached.source.len(),
|
||||
&changes,
|
||||
&mut scopes,
|
||||
&mut cached.spans,
|
||||
)
|
||||
.map_err(|error| error.to_string())
|
||||
});
|
||||
if let Err(error) = result {
|
||||
self.error = Some(format!("{} syntax, line {}: {error}", syntax.name, row + 1));
|
||||
self.clear_highlighting();
|
||||
return;
|
||||
}
|
||||
cached.after = Some((parser.clone(), scopes.clone()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn syntax_set() -> &'static SyntaxSet {
|
||||
static SYNTAXES: LazyLock<SyntaxSet> = LazyLock::new(SyntaxSet::load_defaults_newlines);
|
||||
&SYNTAXES
|
||||
}
|
||||
|
||||
fn detect_syntax<'a>(
|
||||
syntaxes: &'a SyntaxSet,
|
||||
path: Option<&Path>,
|
||||
first_line: Option<&str>,
|
||||
) -> Option<&'a SyntaxReference> {
|
||||
let by_path = path.and_then(|path| {
|
||||
path.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.and_then(|name| syntaxes.find_syntax_by_extension(name))
|
||||
.or_else(|| {
|
||||
path.extension()
|
||||
.and_then(|extension| extension.to_str())
|
||||
.and_then(|extension| syntaxes.find_syntax_by_extension(extension))
|
||||
})
|
||||
});
|
||||
by_path
|
||||
.or_else(|| first_line.and_then(|line| syntaxes.find_syntax_by_first_line(line)))
|
||||
.filter(|syntax| syntax.name != "Plain Text")
|
||||
}
|
||||
|
||||
fn classify(scopes: &ScopeStack) -> Option<SyntaxKind> {
|
||||
static SELECTORS: LazyLock<[(ScopeSelectors, SyntaxKind); 7]> = LazyLock::new(|| {
|
||||
[
|
||||
("comment", SyntaxKind::Comment),
|
||||
("string", SyntaxKind::String),
|
||||
("keyword, storage, constant.language", SyntaxKind::Keyword),
|
||||
("constant.numeric", SyntaxKind::Number),
|
||||
(
|
||||
"entity.name.type, entity.name.class, entity.name.struct, entity.name.enum, \
|
||||
entity.name.trait, support.type, support.class",
|
||||
SyntaxKind::Type,
|
||||
),
|
||||
(
|
||||
"entity.name.function, support.function",
|
||||
SyntaxKind::Function,
|
||||
),
|
||||
("markup, entity.name.tag", SyntaxKind::Markup),
|
||||
]
|
||||
.map(|(selector, kind)| {
|
||||
(
|
||||
selector.parse().expect("fixed semantic scope selector"),
|
||||
kind,
|
||||
)
|
||||
})
|
||||
});
|
||||
SELECTORS
|
||||
.iter()
|
||||
.filter_map(|(selector, kind)| {
|
||||
selector
|
||||
.does_match(scopes.as_slice())
|
||||
.map(|power| (power, *kind))
|
||||
})
|
||||
.max_by_key(|(power, _)| *power)
|
||||
.map(|(_, kind)| kind)
|
||||
}
|
||||
|
||||
fn collect_spans(
|
||||
line_len: usize,
|
||||
changes: &[(usize, ScopeStackOp)],
|
||||
scopes: &mut ScopeStack,
|
||||
spans: &mut Vec<SyntaxSpan>,
|
||||
) -> Result<(), ScopeError> {
|
||||
spans.clear();
|
||||
let mut start = 0;
|
||||
let mut changes = changes.iter().peekable();
|
||||
while let Some((offset, operation)) = changes.next() {
|
||||
let end = (*offset).min(line_len);
|
||||
if start < end {
|
||||
append_span(spans, start, end, classify(scopes));
|
||||
}
|
||||
scopes.apply(operation)?;
|
||||
while changes.peek().is_some_and(|(next, _)| next == offset) {
|
||||
scopes.apply(&changes.next().unwrap().1)?;
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
if start < line_len {
|
||||
append_span(spans, start, line_len, classify(scopes));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn append_span(spans: &mut Vec<SyntaxSpan>, start: usize, end: usize, kind: Option<SyntaxKind>) {
|
||||
if start >= end {
|
||||
return;
|
||||
}
|
||||
let Some(kind) = kind else {
|
||||
return;
|
||||
};
|
||||
if let Some(previous) = spans.last_mut()
|
||||
&& previous.end_byte == start
|
||||
&& previous.kind == kind
|
||||
{
|
||||
previous.end_byte = end;
|
||||
} else {
|
||||
spans.push(SyntaxSpan {
|
||||
start_byte: start,
|
||||
end_byte: end,
|
||||
kind,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use syntect::parsing::{ClearAmount, Scope};
|
||||
|
||||
fn lines(source: &str) -> Vec<String> {
|
||||
source.split('\n').map(str::to_owned).collect()
|
||||
}
|
||||
|
||||
fn kind_at(cache: &SyntaxCache, row: usize, byte: usize) -> Option<SyntaxKind> {
|
||||
cache
|
||||
.line(row)
|
||||
.iter()
|
||||
.find_map(|span| (span.start_byte <= byte && byte < span.end_byte).then_some(span.kind))
|
||||
}
|
||||
|
||||
fn assert_token(
|
||||
cache: &SyntaxCache,
|
||||
source: &[String],
|
||||
row: usize,
|
||||
token: &str,
|
||||
kind: SyntaxKind,
|
||||
) {
|
||||
let start = source[row].find(token).unwrap();
|
||||
for (byte, _) in source[row][start..start + token.len()].char_indices() {
|
||||
assert_eq!(
|
||||
kind_at(cache, row, start + byte),
|
||||
Some(kind),
|
||||
"{token:?} on row {row}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_fresh(cache: &SyntaxCache, source: &[String], path: Option<&Path>) {
|
||||
let mut fresh = SyntaxCache::new();
|
||||
fresh.update(source, path, 0);
|
||||
assert_eq!(cache.language(), fresh.language());
|
||||
assert_eq!(cache.error(), fresh.error());
|
||||
for row in 0..source.len() {
|
||||
assert_eq!(cache.line(row), fresh.line(row), "row {row}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bundled_languages_supply_semantic_scopes() {
|
||||
let cases = [
|
||||
(
|
||||
"sample.rs",
|
||||
"Rust",
|
||||
"fn main() { let value = 42; let text = \"hello\"; /* note */ }",
|
||||
"fn",
|
||||
"main",
|
||||
),
|
||||
(
|
||||
"sample.rb",
|
||||
"Ruby",
|
||||
"def hello\n value = 42\n text = \"hello\" # note\nend",
|
||||
"def",
|
||||
"hello",
|
||||
),
|
||||
(
|
||||
"sample.py",
|
||||
"Python",
|
||||
"def hello():\n value = 42\n text = \"hello\" # note",
|
||||
"def",
|
||||
"hello",
|
||||
),
|
||||
];
|
||||
for (path, language, text, keyword, function) in cases {
|
||||
let source = lines(text);
|
||||
let mut cache = SyntaxCache::new();
|
||||
cache.update(&source, Some(Path::new(path)), 0);
|
||||
assert_eq!(cache.language(), Some(language));
|
||||
assert_eq!(cache.error(), None);
|
||||
assert_token(&cache, &source, 0, keyword, SyntaxKind::Keyword);
|
||||
assert_token(&cache, &source, 0, function, SyntaxKind::Function);
|
||||
let number_row = if language == "Rust" { 0 } else { 1 };
|
||||
let string_row = if language == "Rust" { 0 } else { 2 };
|
||||
assert_token(&cache, &source, number_row, "42", SyntaxKind::Number);
|
||||
// Select the quoted occurrence, not a function with the same name.
|
||||
let quoted = source[string_row].find("\"hello\"").unwrap();
|
||||
assert_eq!(
|
||||
kind_at(&cache, string_row, quoted + 1),
|
||||
Some(SyntaxKind::String)
|
||||
);
|
||||
assert_token(&cache, &source, string_row, "note", SyntaxKind::Comment);
|
||||
}
|
||||
|
||||
let source = lines("{\"message\":\"hello\",\"n\":42,\"enabled\":true}");
|
||||
let mut cache = SyntaxCache::new();
|
||||
cache.update(&source, Some(Path::new("sample.json")), 0);
|
||||
assert_eq!(cache.language(), Some("JSON"));
|
||||
assert_token(&cache, &source, 0, "hello", SyntaxKind::String);
|
||||
assert_token(&cache, &source, 0, "42", SyntaxKind::Number);
|
||||
assert_token(&cache, &source, 0, "true", SyntaxKind::Keyword);
|
||||
assert_eq!(cache.error(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn detects_bundled_special_filenames_extensions_and_actual_shebangs() {
|
||||
let source = lines("all:\n\techo hello");
|
||||
let mut cache = SyntaxCache::new();
|
||||
cache.update(&source, Some(Path::new("/not/read/from/disk/Makefile")), 0);
|
||||
assert_eq!(cache.language(), Some("Makefile"));
|
||||
|
||||
let source = lines("#!/usr/bin/env bash\necho \"hello\" # note");
|
||||
cache.update(&source, Some(Path::new("/not/read/from/disk/script")), 1);
|
||||
assert_eq!(cache.language(), Some("Bourne Again Shell (bash)"));
|
||||
assert_token(&cache, &source, 1, "hello", SyntaxKind::String);
|
||||
assert_token(&cache, &source, 1, "note", SyntaxKind::Comment);
|
||||
assert_eq!(cache.error(), None);
|
||||
|
||||
let source = lines("echo \"hello\" # note");
|
||||
cache.update(&source, Some(Path::new(".bashrc")), 1);
|
||||
assert_eq!(cache.language(), Some("Bourne Again Shell (bash)"));
|
||||
assert_token(&cache, &source, 0, "hello", SyntaxKind::String);
|
||||
|
||||
let source = lines("\u{feff}#!/usr/bin/env python3\nreturn 42");
|
||||
cache.update(&source, None, 2);
|
||||
assert_eq!(cache.language(), Some("Python"));
|
||||
assert_token(&cache, &source, 1, "return", SyntaxKind::Keyword);
|
||||
cache.update(&source, Some(Path::new("sample.RS")), 2);
|
||||
assert_eq!(cache.language(), Some("Rust"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_and_plain_text_have_no_spans_or_errors() {
|
||||
let source = lines("fn main() { \"hello\"; 42; }\n/* note");
|
||||
let mut cache = SyntaxCache::new();
|
||||
for path in [
|
||||
None,
|
||||
Some(Path::new("sample.unknown-extension")),
|
||||
Some(Path::new("sample.txt")),
|
||||
] {
|
||||
cache.update(&source, path, 0);
|
||||
assert_eq!(cache.language(), None);
|
||||
assert_eq!(cache.error(), None);
|
||||
assert!(cache.line(0).is_empty());
|
||||
assert!(cache.line(1).is_empty());
|
||||
assert!(cache.line(500).is_empty());
|
||||
}
|
||||
cache.update(&[], Some(Path::new("empty.rs")), 1);
|
||||
assert_eq!(cache.language(), Some("Rust"));
|
||||
assert!(cache.line(0).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_multiline_comments_and_edits_before_visible_rows() {
|
||||
let mut source = lines("/* outer\n/* inner */ still outer\nlast */ fn main() {}\n// tail");
|
||||
let path = Some(Path::new("sample.rs"));
|
||||
let mut cache = SyntaxCache::new();
|
||||
cache.update(&source, path, 0);
|
||||
assert_token(&cache, &source, 1, "inner", SyntaxKind::Comment);
|
||||
assert_token(&cache, &source, 1, "still outer", SyntaxKind::Comment);
|
||||
assert_token(&cache, &source, 2, "last", SyntaxKind::Comment);
|
||||
assert_token(&cache, &source, 2, "fn", SyntaxKind::Keyword);
|
||||
|
||||
source[0] = "fn earlier() {}".into();
|
||||
cache.update(&source, path, 1);
|
||||
assert_eq!(
|
||||
kind_at(&cache, 1, source[1].find("still outer").unwrap()),
|
||||
None
|
||||
);
|
||||
assert_eq!(kind_at(&cache, 2, 0), None);
|
||||
assert_fresh(&cache, &source, path);
|
||||
|
||||
source[0] = "/* outer".into();
|
||||
cache.update(&source, path, 2);
|
||||
assert_token(&cache, &source, 2, "last", SyntaxKind::Comment);
|
||||
assert_fresh(&cache, &source, path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn multiline_strings_preserve_unicode_byte_ranges_and_tolerate_unfinished_source() {
|
||||
let mut source = lines(
|
||||
"text = \"\"\"start\n\t界e\u{301}👩💻 middle\nfinish\"\"\"\nreturn 42 # 界e\u{301}👩💻",
|
||||
);
|
||||
let path = Some(Path::new("sample.py"));
|
||||
let mut cache = SyntaxCache::new();
|
||||
cache.update(&source, path, 0);
|
||||
assert_token(
|
||||
&cache,
|
||||
&source,
|
||||
1,
|
||||
"\t界e\u{301}👩💻 middle",
|
||||
SyntaxKind::String,
|
||||
);
|
||||
assert_token(&cache, &source, 2, "finish", SyntaxKind::String);
|
||||
assert_token(&cache, &source, 3, "return", SyntaxKind::Keyword);
|
||||
assert_token(&cache, &source, 3, "界e\u{301}👩💻", SyntaxKind::Comment);
|
||||
for (row, line) in source.iter().enumerate() {
|
||||
let mut end = 0;
|
||||
for span in cache.line(row) {
|
||||
assert!(end <= span.start_byte && span.start_byte < span.end_byte);
|
||||
assert!(span.end_byte <= line.len());
|
||||
assert!(line.is_char_boundary(span.start_byte));
|
||||
assert!(line.is_char_boundary(span.end_byte));
|
||||
end = span.end_byte;
|
||||
}
|
||||
}
|
||||
source[2] = "unfinished".into();
|
||||
cache.update(&source, path, 1);
|
||||
assert_token(&cache, &source, 3, "return", SyntaxKind::String);
|
||||
assert_eq!(cache.error(), None);
|
||||
assert_fresh(&cache, &source, path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn blank_lines_keep_scopes_and_unicode_before_tokens_keeps_byte_offsets() {
|
||||
let source = lines("/* first\n\n\t界e\u{301}👩💻 last */ fn main() { let value = 42; }");
|
||||
let mut cache = SyntaxCache::new();
|
||||
cache.update(&source, Some(Path::new("sample.rs")), 0);
|
||||
assert!(cache.line(1).is_empty());
|
||||
assert_token(&cache, &source, 2, "界e\u{301}👩💻 last", SyntaxKind::Comment);
|
||||
assert_token(&cache, &source, 2, "fn", SyntaxKind::Keyword);
|
||||
assert_token(&cache, &source, 2, "main", SyntaxKind::Function);
|
||||
assert_token(&cache, &source, 2, "42", SyntaxKind::Number);
|
||||
for span in cache.line(2) {
|
||||
assert!(source[2].is_char_boundary(span.start_byte));
|
||||
assert!(source[2].is_char_boundary(span.end_byte));
|
||||
assert!(span.end_byte <= source[2].len());
|
||||
}
|
||||
assert_eq!(cache.error(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn revisions_reuse_prefix_checkpoints_and_unchanged_source_snapshots() {
|
||||
let mut source = lines("fn main() {\n /* note\n continued */\n let value = 42;\n}");
|
||||
let path = Some(Path::new("sample.rs"));
|
||||
let mut cache = SyntaxCache::new();
|
||||
cache.update(&source, path, 10);
|
||||
assert_eq!(cache.parsed_lines, 5);
|
||||
let prefix_source = cache.lines[0].source.as_ptr();
|
||||
let prefix_spans = cache.lines[0].spans.as_ptr();
|
||||
cache.update(&source, path, 10);
|
||||
cache.update(&source, path, 11);
|
||||
assert_eq!(cache.parsed_lines, 5);
|
||||
|
||||
source[2] = " changed */".into();
|
||||
cache.update(&source, path, 12);
|
||||
assert_eq!(cache.parsed_lines, 8);
|
||||
assert_eq!(cache.lines[0].source.as_ptr(), prefix_source);
|
||||
assert_eq!(cache.lines[0].spans.as_ptr(), prefix_spans);
|
||||
assert_token(&cache, &source, 2, "changed", SyntaxKind::Comment);
|
||||
assert_fresh(&cache, &source, path);
|
||||
|
||||
source[2] = " continued */".into();
|
||||
cache.update(&source, path, 13);
|
||||
assert_eq!(cache.parsed_lines, 11);
|
||||
assert_fresh(&cache, &source, path);
|
||||
cache.update(&source, path, 14);
|
||||
assert_eq!(cache.parsed_lines, 11);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn insertion_removal_and_appending_preserve_multiline_state() {
|
||||
let path = Some(Path::new("sample.py"));
|
||||
let mut source = lines("text = \"\"\"start\nmiddle\nend\"\"\"\nreturn 42");
|
||||
let mut cache = SyntaxCache::new();
|
||||
cache.update(&source, path, 0);
|
||||
assert_eq!(cache.parsed_lines, 4);
|
||||
|
||||
source.insert(1, "inserted 界".into());
|
||||
cache.update(&source, path, 1);
|
||||
assert_eq!(cache.parsed_lines, 8);
|
||||
assert_token(&cache, &source, 2, "middle", SyntaxKind::String);
|
||||
assert_fresh(&cache, &source, path);
|
||||
|
||||
source.remove(1);
|
||||
cache.update(&source, path, 2);
|
||||
assert_eq!(cache.parsed_lines, 11);
|
||||
assert_fresh(&cache, &source, path);
|
||||
|
||||
source.push("# appended".into());
|
||||
cache.update(&source, path, 3);
|
||||
assert_eq!(cache.parsed_lines, 12);
|
||||
assert_token(&cache, &source, 4, "appended", SyntaxKind::Comment);
|
||||
source.pop();
|
||||
cache.update(&source, path, 4);
|
||||
assert_eq!(cache.parsed_lines, 12);
|
||||
assert!(cache.line(4).is_empty());
|
||||
assert_fresh(&cache, &source, path);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rename_at_the_same_revision_and_shebang_edits_redetect_language() {
|
||||
let mut source = lines("fn main() { let value = 42; }");
|
||||
let mut cache = SyntaxCache::new();
|
||||
cache.update(&source, Some(Path::new("old.rs")), 7);
|
||||
assert_eq!(cache.parsed_lines, 1);
|
||||
cache.update(&source, Some(Path::new("new.RS")), 7);
|
||||
assert_eq!(cache.parsed_lines, 1);
|
||||
cache.update(&source, Some(Path::new("new.txt")), 7);
|
||||
assert_eq!(cache.language(), None);
|
||||
assert!(cache.line(0).is_empty());
|
||||
cache.update(&source, Some(Path::new("new.rs")), 7);
|
||||
assert_eq!(cache.parsed_lines, 2);
|
||||
assert_token(&cache, &source, 0, "fn", SyntaxKind::Keyword);
|
||||
|
||||
source = lines("#!/usr/bin/env bash\necho \"hello\"");
|
||||
cache.update(&source, None, 8);
|
||||
assert_eq!(cache.language(), Some("Bourne Again Shell (bash)"));
|
||||
source[0] = "#!/usr/bin/env python3".into();
|
||||
source[1] = "return \"hello\"".into();
|
||||
cache.update(&source, None, 9);
|
||||
assert_eq!(cache.language(), Some("Python"));
|
||||
assert_token(&cache, &source, 1, "return", SyntaxKind::Keyword);
|
||||
assert_fresh(&cache, &source, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn type_function_and_markup_categories_come_from_scopes() {
|
||||
let mut cache = SyntaxCache::new();
|
||||
let source = lines("struct Widget;\nfn build() {}");
|
||||
cache.update(&source, Some(Path::new("sample.rs")), 0);
|
||||
assert_token(&cache, &source, 0, "Widget", SyntaxKind::Type);
|
||||
assert_token(&cache, &source, 1, "build", SyntaxKind::Function);
|
||||
let source = lines("# Heading");
|
||||
cache.update(&source, Some(Path::new("sample.md")), 1);
|
||||
assert_token(&cache, &source, 0, "Heading", SyntaxKind::Markup);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deepest_scope_match_wins_and_clear_restore_survives_checkpoints() {
|
||||
let mut scopes: ScopeStack = "source.python string.quoted.double.python".parse().unwrap();
|
||||
let mut spans = Vec::new();
|
||||
let changes = [
|
||||
(2, ScopeStackOp::Clear(ClearAmount::All)),
|
||||
(2, ScopeStackOp::Push(Scope::new("source.rust").unwrap())),
|
||||
(
|
||||
2,
|
||||
ScopeStackOp::Push(Scope::new("comment.block.rust").unwrap()),
|
||||
),
|
||||
];
|
||||
collect_spans(4, &changes, &mut scopes, &mut spans).unwrap();
|
||||
assert_eq!(
|
||||
spans,
|
||||
[
|
||||
SyntaxSpan {
|
||||
start_byte: 0,
|
||||
end_byte: 2,
|
||||
kind: SyntaxKind::String
|
||||
},
|
||||
SyntaxSpan {
|
||||
start_byte: 2,
|
||||
end_byte: 4,
|
||||
kind: SyntaxKind::Comment
|
||||
},
|
||||
]
|
||||
);
|
||||
let mut restored = scopes.clone();
|
||||
collect_spans(
|
||||
4,
|
||||
&[(1, ScopeStackOp::Pop(2)), (1, ScopeStackOp::Restore)],
|
||||
&mut restored,
|
||||
&mut spans,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
spans,
|
||||
[
|
||||
SyntaxSpan {
|
||||
start_byte: 0,
|
||||
end_byte: 1,
|
||||
kind: SyntaxKind::Comment
|
||||
},
|
||||
SyntaxSpan {
|
||||
start_byte: 1,
|
||||
end_byte: 4,
|
||||
kind: SyntaxKind::String
|
||||
},
|
||||
]
|
||||
);
|
||||
let before = restored.clone();
|
||||
collect_spans(
|
||||
3,
|
||||
&[
|
||||
(1, ScopeStackOp::Clear(ClearAmount::TopN(1))),
|
||||
(
|
||||
1,
|
||||
ScopeStackOp::Push(Scope::new("constant.numeric.python").unwrap()),
|
||||
),
|
||||
(2, ScopeStackOp::Pop(1)),
|
||||
(2, ScopeStackOp::Restore),
|
||||
],
|
||||
&mut restored,
|
||||
&mut spans,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(restored, before);
|
||||
assert_eq!(spans[1].kind, SyntaxKind::Number);
|
||||
restored.push(Scope::new("keyword.operator.python").unwrap());
|
||||
assert_eq!(classify(&restored), Some(SyntaxKind::Keyword));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn real_parser_errors_clear_the_entire_buffer_and_recover_on_an_edit() {
|
||||
let path = Some(Path::new("sample.rs"));
|
||||
let mut source = lines("fn main() {}\nlet value = 42;");
|
||||
let mut cache = SyntaxCache::new();
|
||||
cache.update(&source, path, 0);
|
||||
assert!(!cache.line(0).is_empty());
|
||||
// Missing grammar contexts make Syntect return a genuine ParsingError, not a mock failure.
|
||||
cache.parse_from(0, cache.syntax.unwrap(), &SyntaxSet::new());
|
||||
assert!(cache.error().unwrap().contains("Missing context"));
|
||||
assert!(cache.error().unwrap().contains("line 1"));
|
||||
assert!(cache.line(0).is_empty());
|
||||
assert!(cache.line(1).is_empty());
|
||||
cache.update(&source, path, 1);
|
||||
assert!(cache.error().is_some());
|
||||
source[1] = "let value = 43;".into();
|
||||
cache.update(&source, path, 2);
|
||||
assert_eq!(cache.error(), None);
|
||||
assert_token(&cache, &source, 0, "fn", SyntaxKind::Keyword);
|
||||
assert_token(&cache, &source, 1, "43", SyntaxKind::Number);
|
||||
assert_fresh(&cache, &source, path);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user