use super::*; fn lines(source: &str) -> Vec { source.split('\n').map(str::to_owned).collect() } #[test] fn scopes_follow_real_declarations_and_each_sources_own_tree() { let original = "impl State {\n fn old() {\n call();\n }\n}\n\nfn next() {\n call();\n}\n"; let before = cache(original, "old.rs"); let mut live = cache(&original.replace("old()", "renamed()"), "new.rs"); assert_eq!( before.scope_at(2), Some(ScopeContext { start_row: 1, end_row: 4, name: Point::new(1, 7), label: "impl State › old", }) ); assert_eq!(before.scope_at(3).unwrap().label, "impl State › old"); assert_eq!(before.scope_at(4).unwrap().label, "impl State"); assert_eq!(before.scope_at(5), None); assert_eq!( before.scope_at(6), Some(ScopeContext { start_row: 6, end_row: 9, name: Point::new(6, 3), label: "next", }) ); assert_eq!(before.scope_at(9), None); assert_eq!(before.scope_at(usize::MAX), None); let changed = lines(&original.replace("old()", "updated()")); live.update_visible(&changed, Some(Path::new("new.rs")), 1, &[2..3]); assert_eq!(live.scope_at(2).unwrap().label, "impl State › updated"); assert_eq!(before.scope_at(2).unwrap().label, "impl State › old"); let counts = live.parse_counts(); live.update_visible(&changed, Some(Path::new("new.rs")), 2, &[6..9]); assert_eq!(live.parse_counts(), counts); live.update_visible(&changed, Some(Path::new("renamed.txt")), 2, &[0..1]); assert_eq!(live.scope_at(2), None); assert_eq!(before.scope_at(2).unwrap().label, "impl State › old"); let incomplete = cache("impl State {\n fn run() {\n let = ;\n }\n}", "a.rs"); assert_eq!(incomplete.scope_at(2).unwrap().label, "impl State › run"); let python = cache( "class State:\n @decorator\n def run(self):\n work()\n\n def next(self):\n work()\n", "a.py", ); assert_eq!( python.scope_at(1), Some(ScopeContext { start_row: 1, end_row: 4, name: Point::new(2, 8), label: "State › run", }) ); assert_eq!(python.scope_at(4).unwrap().label, "State"); assert_eq!(python.scope_at(5).unwrap().label, "State › next"); assert_eq!(python.scope_at(7), None); for (path, source, expected) in [ ( "a.rs", "trait State {\n fn run(&self) {\n work();\n }\n}", Some("State › run"), ), ( "a.rb", "class State\n def run\n work\n end\nend", Some("State › run"), ), ( "a.js", "function outer() {\n function inner() {\n work();\n }\n}", Some("outer › inner"), ), ( "a.ts", "class State {\n run = () => {\n work();\n };\n}", Some("State › run"), ), ( "a.tsx", "const View = () => {\n work();\n return
;\n};", Some("View"), ), ( "a.js", "const handlers = { run: () => {\n work();\n} };", Some("run"), ), ( "a.js", "State.run = () => {\n work();\n};", Some("State.run"), ), ( "a.go", "package main\nfunc (s *State) run() {\n work()\n}", Some("*State › run"), ), ( "a.go", "package main\nfunc run() {\n work()\n}", Some("run"), ), ( "a.c", "int run(void) {\n work();\n return 7;\n}", Some("run"), ), ( "a.cpp", "class State {\n void run() {\n work();\n }\n};", Some("State › run"), ), ( "a.cpp", "void State::run() {\n work();\n}", Some("State::run"), ), ("a.sh", "run() {\n work\n}", Some("run")), ( "a.zig", "const State = struct {\n pub fn run() void {\n work();\n }\n};", Some("State › run"), ), ("a.md", "```rust\nfn run() {\n work();\n}\n```", Some("run")), ( "a.html", "", Some("run"), ), ("a.js", "use(() => {\n work();\n});", None), ( "a.js", "const handlers = { [key]: () => {\n work();\n} };", None, ), ("a.txt", "fn run() {\n work();\n}", None), ("a.unknown", "fn run() {\n work();\n}", None), ("a.json", "{\"work\": \"function run() {}\"}", None), ("a.yaml", "work: function run() {}", None), ] { let syntax = cache(source, path); let row = source[..source.find("work").unwrap()] .bytes() .filter(|byte| *byte == b'\n') .count(); assert_eq!( syntax.scope_at(row).map(|scope| scope.label), expected, "{path}: {source}" ); if let Some(scope) = syntax.scope_at(row) { assert!(scope.start_row <= row && row < scope.end_row, "{path}"); } } } fn source_range(source: &str) -> Range { Range { start_byte: 0, end_byte: source.len(), start_point: Point::new(0, 0), end_point: point_at(source, source.len()), } } #[test] fn string_newlines_remain_literal_across_visible_row_caching_and_edits() { let source = "text = \"\"\"a\n\nb\"\"\"\nplain\n"; let input = lines(source); let path = Path::new("strings.py"); let mut cache = SyntaxCache::new(); let includes_newline = |cache: &SyntaxCache, row: usize, length: usize| { cache .string_spans(row) .iter() .any(|span| span.start_byte <= length && span.end_byte > length) }; cache.update_visible(&input, Some(path), 0, std::slice::from_ref(&(0..1))); assert!(includes_newline(&cache, 0, input[0].len())); cache.update_visible(&input, Some(path), 0, std::slice::from_ref(&(1..4))); assert!(includes_newline(&cache, 0, input[0].len())); assert!(includes_newline(&cache, 1, input[1].len())); assert!(!includes_newline(&cache, 2, input[2].len())); assert!(!includes_newline(&cache, 3, input[3].len())); for (row, line) in input.iter().enumerate() { assert!( cache .line(row) .iter() .all(|span| span.end_byte <= line.len()) ); } cache.update(&lines("# a\n\nb\nplain\n"), Some(path), 1); assert!(!includes_newline(&cache, 0, 3)); assert!(!includes_newline(&cache, 1, 0)); } #[test] fn visible_windows_match_full_colors_after_edits_and_scrolling_without_reparsing() { let cases = [ ( "a.rs", "fn main() {\n/* start\ninside é 👩‍💻\nend */\nlet n = 42;\nlet text = \"unfinished", ), ("a.py", "text = '''first\ninside é\nlast'''\nanswer = 42"), ( "a.js", "const action = () => 42;\nfunction hello(require) {\n require('local');\n action;\n}\nrequire('global');\naction;", ), ( "a.md", "# Heading **bold**\n\n```rust\nfn main() {\n let n = 42;\n}\n```\n\n```python\ndef hello():\n return 7\n```\n\n```unknown\nfn plain() {}\n```", ), ( "a.md", "> # Heading **bold**\n>\n> ```rust\n> fn main() {\n> let n = 42;\n> }\n> ```", ), ( "a.md", "```html\n\n```", ), ( "a.html", "
é
\n\n", ), ( "a.js", "const page = html`
${value}
\n`;", ), ]; for (path, original) in cases { let mut visible = SyntaxCache::new(); for (revision, source) in [ original.to_owned(), original.replace("42", "123"), format!("\n{original}"), original.replace("() => 42", "42"), ] .iter() .enumerate() { let input = lines(source); let middle = input.len() / 2; visible.update_visible( &input, Some(Path::new(path)), revision as u64, std::slice::from_ref(&(middle..middle + 1)), ); let full = cache(source, path); assert_eq!(visible.error(), None, "{path}"); assert_eq!( visible.line(middle), full.line(middle), "{path} revision {revision}" ); assert!( visible .lines .iter() .enumerate() .all(|(row, spans)| row == middle || spans.is_empty()) ); let parses = visible.parse_counts(); let context = visible.context.as_ref().unwrap().captures.as_ptr(); // Unsorted, overlapping and out-of-bounds windows use original source rows. for row in (0..input.len()).rev() { visible.update_visible( &input, Some(Path::new(path)), revision as u64, &[ row..row + 1, middle..middle + 1, row..row + 1, input.len()..usize::MAX, ], ); assert_eq!( visible.line(row), full.line(row), "{path} revision {revision} row {row}" ); assert_eq!( visible.parse_counts(), parses, "{path}: scrolling reparsed a tree" ); assert_eq!(visible.context.as_ref().unwrap().captures.as_ptr(), context); } } } } #[test] fn visible_highlight_collection_stays_bounded_by_the_requested_rows() { let mut input: Vec<_> = (0..1000) .map(|row| format!("fn function_{row}() {{ let value = 42; }}")) .collect(); let path = Path::new("large.rs"); let mut visible = SyntaxCache::new(); visible.update_visible(&input, Some(path), 0, std::slice::from_ref(&(500..502))); assert_eq!(visible.parses, 1); assert!(visible.captures.len() < 50); assert!(visible.highlights.len() < 30); assert!(visible.lines[..500].iter().all(Vec::is_empty)); assert!(visible.lines[502..].iter().all(Vec::is_empty)); input[500] = "fn changed() { let value = \"é 👩‍💻\"; }".to_owned(); visible.update_visible(&input, Some(path), 1, std::slice::from_ref(&(500..502))); assert_eq!(visible.incremental_parses, 1); assert!(visible.captures.len() < 50); let mut full = SyntaxCache::new(); full.update(&input, Some(path), 1); assert!(full.captures.len() > 1000); for row in 500..502 { assert_eq!(visible.line(row), full.line(row)); } let parses = visible.parse_counts(); visible.update_visible(&input, Some(path), 1, std::slice::from_ref(&(950..952))); assert_eq!(visible.parse_counts(), parses); assert!(visible.captures.len() < 50); for row in 950..952 { assert_eq!(visible.line(row), full.line(row)); } } #[test] fn visible_cache_supports_full_updates_and_same_revision_path_changes() { let input = lines("const element =
;\nconst answer = 42;"); let mut visible = SyntaxCache::new(); visible.update_visible( &input, Some(Path::new("a.ts")), 0, std::slice::from_ref(&(1..2)), ); let parses = visible.parse_counts(); visible.update(&input, Some(Path::new("a.ts")), 0); assert_eq!(visible.parse_counts(), parses); assert_fresh(&visible, &input.join("\n"), "a.ts"); visible.update_visible( &input, Some(Path::new("a.tsx")), 0, std::slice::from_ref(&(0..1)), ); assert_eq!(visible.parses, parses.0 + 1); assert_eq!(visible.line(0), cache(&input.join("\n"), "a.tsx").line(0)); assert!(visible.line(1).is_empty()); visible.update_text(&input.join("\n"), Path::new("a.tsx")); assert_fresh(&visible, &input.join("\n"), "a.tsx"); visible.update_visible( &input, Some(Path::new("a.txt")), 0, std::slice::from_ref(&(0..2)), ); assert_eq!(visible.language(), None); assert!(visible.lines.iter().all(Vec::is_empty)); } fn cache(source: &str, path: &str) -> SyntaxCache { let mut cache = SyntaxCache::new(); cache.update(&lines(source), Some(Path::new(path)), 0); assert_eq!(cache.error(), None, "{path}"); cache } fn kind_at(cache: &SyntaxCache, row: usize, byte: usize) -> Option { cache .line(row) .iter() .find_map(|span| (span.start_byte <= byte && byte < span.end_byte).then_some(span.kind)) } fn assert_token(cache: &SyntaxCache, source: &str, row: usize, token: &str, kind: SyntaxKind) { let line = source.split('\n').nth(row).unwrap(); let start = line.find(token).unwrap(); for (byte, _) in line[start..start + token.len()].char_indices() { assert_eq!( kind_at(cache, row, start + byte), Some(kind), "{token:?} in {}", cache.language().unwrap() ); } } fn assert_fresh(cache: &SyntaxCache, source: &str, path: &str) { let fresh = self::cache(source, path); assert_eq!(cache.language(), fresh.language()); assert_eq!(cache.error(), fresh.error()); assert_eq!( cache.tree.as_ref().map(|tree| tree.root_node().to_sexp()), fresh.tree.as_ref().map(|tree| tree.root_node().to_sexp()) ); let source = lines(source); for (row, line) in source.iter().enumerate() { assert_eq!(cache.line(row), fresh.line(row), "row {row}"); let mut previous = 0; for span in cache.line(row) { assert!(previous <= span.start_byte && span.start_byte < span.end_byte); assert!(span.end_byte <= line.len()); assert!(line.is_char_boundary(span.start_byte) && line.is_char_boundary(span.end_byte)); previous = span.end_byte; } } } #[test] fn every_bundled_query_compiles_and_common_languages_color_real_tokens() { for language in languages::all() { configuration(language).unwrap_or_else(|error| panic!("{}: {error}", language.name)); } let cases = [ ( "a.rs", "Rust", "fn main() { let x = 42; let s = \"hi\"; /* note */ }", "fn", SyntaxKind::Keyword, ), ( "a.rb", "Ruby", "def hello\n 42\nend", "def", SyntaxKind::Keyword, ), ( "a.py", "Python", "def hello():\n return 42", "def", SyntaxKind::Keyword, ), ( "a.json", "JSON", "{\"n\":42,\"yes\":true}", "42", SyntaxKind::Number, ), ( "a.sh", "Bash", "echo \"hi\" # note", "hi", SyntaxKind::String, ), ( "Makefile", "Makefile", "# note\nall:\n\techo hi", "note", SyntaxKind::Comment, ), ( "a.md", "Markdown", "# Heading\n\n**bold**", "Heading", SyntaxKind::Markup, ), ( "a.js", "JavaScript", "function hello() { return 42; }", "function", SyntaxKind::Keyword, ), ( "a.ts", "TypeScript", "function hello(): number { return 42; }", "function", SyntaxKind::Keyword, ), ( "a.html", "HTML", "
text
", "div", SyntaxKind::Markup, ), ( "a.css", "CSS", "body { width: 42px; }", "42", SyntaxKind::Number, ), ( "a.yaml", "YAML", "message: \"hi\"\ncount: 42", "hi", SyntaxKind::String, ), ( "Cargo.toml", "TOML", "message = \"hi\"\ncount = 42", "hi", SyntaxKind::String, ), ( "a.c", "C", "int main(void) { return 42; }", "return", SyntaxKind::Keyword, ), ( "a.cpp", "C++", "int main() { return 42; }", "return", SyntaxKind::Keyword, ), ( "a.go", "Go", "package main\nfunc main() { return }", "package", SyntaxKind::Keyword, ), ]; for (path, language, source, token, kind) in cases { let cache = self::cache(source, path); assert_eq!(cache.language(), Some(language)); assert_token(&cache, source, 0, token, kind); assert_fresh(&cache, source, path); } } #[test] fn semantic_colors_and_text_predicates_preserve_the_ui_contract() { let source = "fn main() { let n = 42; let f = 3.25; let b = true; let s = \"hello\"; /* note */ }"; let cache = self::cache(source, "a.rs"); for (token, kind) in [ ("fn", SyntaxKind::Keyword), ("let", SyntaxKind::Keyword), ("main", SyntaxKind::Function), ("42", SyntaxKind::Number), ("3.25", SyntaxKind::Number), ("true", SyntaxKind::Keyword), ("hello", SyntaxKind::String), ("note", SyntaxKind::Comment), ] { assert_token(&cache, source, 0, token, kind); } let source = "const foo = 42; const UPPER = 7; foo; UPPER;"; let cache = self::cache(source, "a.js"); assert_eq!(kind_at(&cache, 0, source.find("foo").unwrap()), None); // JavaScript's bundled constant predicate must not color every identifier. assert_ne!( kind_at(&cache, 0, source.find("foo;").unwrap()), Some(SyntaxKind::Keyword) ); } #[test] fn zig_highlights_literals_builtins_and_predicate_matched_identifiers() { let source = r#"const std = @import("std"); const MAX_COUNT: u32 = 42; pub fn main() void { const local = @as(u32, 7); const value = Point; const text = \\first é \\second ; // note }"#; let cache = self::cache(source, "main.zig"); for (row, token, kind) in [ (0, "const", SyntaxKind::Keyword), (0, "@import", SyntaxKind::Keyword), (0, "\"std\"", SyntaxKind::String), (1, "MAX_COUNT", SyntaxKind::Keyword), (1, "u32", SyntaxKind::Type), (1, "42", SyntaxKind::Number), (2, "pub", SyntaxKind::Keyword), (2, "fn", SyntaxKind::Keyword), (2, "main", SyntaxKind::Function), (2, "void", SyntaxKind::Type), (3, "@as", SyntaxKind::Function), (4, "Point", SyntaxKind::Type), (6, "first é", SyntaxKind::String), (7, "second", SyntaxKind::String), (9, "note", SyntaxKind::Comment), ] { assert_token(&cache, source, row, token, kind); } let local = source.lines().nth(3).unwrap().find("local").unwrap(); assert_eq!(kind_at(&cache, 3, local), None); } #[test] fn honey_highlights_declarations_literals_edits_and_markdown_fences() { let source = "Point :: struct { x f64 }\n\ add proc(a, b i32) i32 {\n\ text := \"hello é\"\n\ enabled := true\n\ value := 42\n\ # note\n\ return a + b\n\ }"; let mut cache = self::cache(source, "main.hon"); assert_eq!(cache.language(), Some("Honey")); assert!(!cache.tree.as_ref().unwrap().root_node().has_error()); for (row, token, kind) in [ (0, "Point", SyntaxKind::Type), (0, "struct", SyntaxKind::Keyword), (0, "f64", SyntaxKind::Type), (1, "add", SyntaxKind::Function), (1, "proc", SyntaxKind::Keyword), (1, "i32", SyntaxKind::Type), (2, "hello é", SyntaxKind::String), (3, "true", SyntaxKind::Keyword), (4, "42", SyntaxKind::Number), (5, "note", SyntaxKind::Comment), (6, "return", SyntaxKind::Keyword), ] { assert_token(&cache, source, row, token, kind); } assert_eq!(kind_at(&cache, 6, 7), None); let changed = source.replace("\"hello é\"", "7"); cache.update(&lines(&changed), Some(Path::new("main.hon")), 1); assert_token(&cache, &changed, 2, "7", SyntaxKind::Number); assert_fresh(&cache, &changed, "main.hon"); for tag in ["honey", "honeylang", "hon"] { let markdown = format!("```{tag}\n{source}\n```"); let injected = self::cache(&markdown, "README.md"); assert_token(&injected, &markdown, 2, "add", SyntaxKind::Function); assert_token(&injected, &markdown, 3, "hello é", SyntaxKind::String); } } #[test] fn typst_highlights_markup_code_edits_and_fenced_languages() { let source = "= Heading\n#let message = \"hello é\"\n#let count = 42\n\ #text(message) // note\n*strong* and _emphasis_\n\ ```rust\nfn main() { let n = 7; }\n```"; let mut cache = self::cache(source, "main.typ"); assert_eq!(cache.language(), Some("Typst")); assert!(!cache.tree.as_ref().unwrap().root_node().has_error()); for (row, token, kind) in [ (0, "Heading", SyntaxKind::Markup), (1, "let", SyntaxKind::Keyword), (1, "hello é", SyntaxKind::String), (2, "42", SyntaxKind::Number), (3, "text", SyntaxKind::Function), (3, "// note", SyntaxKind::Comment), (4, "strong", SyntaxKind::Markup), (4, "emphasis", SyntaxKind::Markup), (6, "fn", SyntaxKind::Keyword), (6, "7", SyntaxKind::Number), ] { assert_token(&cache, source, row, token, kind); } let edited = source.replace("\"hello é\"", "false"); cache.update(&lines(&edited), Some(Path::new("main.typ")), 1); assert_token(&cache, &edited, 1, "false", SyntaxKind::Keyword); assert_fresh(&cache, &edited, "main.typ"); for tag in ["typst", "typ"] { let markdown = format!("```{tag}\n#let count = 42\n```"); let injected = self::cache(&markdown, "README.md"); assert_token(&injected, &markdown, 1, "let", SyntaxKind::Keyword); assert_token(&injected, &markdown, 1, "42", SyntaxKind::Number); } } #[test] fn filenames_shebang_save_as_and_tsx_use_actual_grammar_identity() { let source = lines("\u{feff}#!/usr/bin/env python3\nreturn 42"); let mut cache = SyntaxCache::new(); cache.update(&source, None, 0); assert_eq!(cache.language(), Some("Python")); cache.update(&source, Some(Path::new("a.RS")), 0); assert_eq!(cache.language(), Some("Rust")); assert_eq!(cache.parses, 2); cache.update(&source, Some(Path::new("renamed.rs")), 0); assert_eq!(cache.parses, 2); cache.update(&lines("all:\n\techo hi"), Some(Path::new("GNUmakefile")), 1); assert_eq!(cache.language(), Some("Makefile")); let source = "const element =
;"; cache.update(&lines(source), Some(Path::new("a.ts")), 2); let typescript = cache.syntax.unwrap(); let parses = cache.parses; cache.update(&lines(source), Some(Path::new("a.tsx")), 2); assert_eq!(cache.language(), Some("TypeScript")); assert!(!std::ptr::eq(typescript, cache.syntax.unwrap())); assert_eq!(cache.parses, parses + 1); assert_token(&cache, source, 0, "div", SyntaxKind::Markup); assert_fresh(&cache, source, "a.tsx"); let mut script = SyntaxCache::new(); script.update(&lines("#!/usr/bin/env python3\nreturn 42"), None, 0); assert_eq!(script.language(), Some("Python")); script.update(&lines("#!/usr/bin/env bash\necho \"hi\""), None, 1); assert_eq!(script.language(), Some("Bash")); assert_token( &script, "#!/usr/bin/env bash\necho \"hi\"", 1, "hi", SyntaxKind::String, ); } #[test] fn unknown_and_former_non_common_languages_are_honest_plain_text() { for path in [ "a.unknown", "a.txt", "a.hs", "a.java", "a.php", "a.swift", "a.sql", ] { let cache = self::cache("fn main() { \"hi\"; 42; }", path); assert_eq!(cache.language(), None, "{path}"); assert!(cache.lines.iter().all(Vec::is_empty)); assert_eq!(cache.parses, 0); } } #[test] fn unchanged_revision_and_same_source_do_not_allocate_or_parse() { let mut cache = self::cache("fn main() {}\n", "a.rs"); let source = lines("fn main() {}\n"); let source_pointer = cache.source.as_ptr(); let scratch_pointer = cache.next_source.as_ptr(); let tree_id = cache.tree.as_ref().unwrap().root_node().id(); cache.update(&source, Some(Path::new("a.rs")), 0); assert_eq!(cache.source.as_ptr(), source_pointer); assert_eq!(cache.next_source.as_ptr(), scratch_pointer); assert_eq!(cache.tree.as_ref().unwrap().root_node().id(), tree_id); cache.update(&source, Some(Path::new("a.rs")), 1); cache.update(&source, Some(Path::new("b.rs")), 1); assert_eq!(cache.parses, 1); assert_eq!(cache.incremental_parses, 0); assert_eq!(cache.tree.as_ref().unwrap().root_node().id(), tree_id); assert!(cache.line(1).is_empty()); cache.update(&source, Some(Path::new("b.txt")), 1); assert_eq!(cache.language(), None); assert!(cache.tree.is_none()); assert!(cache.lines.iter().all(Vec::is_empty)); cache.update(&source, Some(Path::new("b.rs")), 1); assert_eq!(cache.parses, 2); assert_eq!(cache.incremental_parses, 0); } fn assert_plain_text_storage(cache: &SyntaxCache) { assert!(cache.tree.is_none()); assert!(cache.context.is_none()); assert!(cache.scopes.get().is_none()); assert_eq!(cache.source.capacity(), 0); assert_eq!(cache.next_source.capacity(), 0); assert_eq!(cache.line_starts.capacity(), 0); assert_eq!(cache.lines.capacity(), 0); assert_eq!(cache.string_spans.capacity(), 0); assert_eq!(cache.highlighted.capacity(), 0); assert_eq!(cache.injections.capacity(), 0); assert_eq!(cache.captures.capacity(), 0); assert_eq!(cache.highlights.capacity(), 0); assert_eq!(cache.events.capacity(), 0); assert!(cache.line(0).is_empty()); assert!(cache.string_spans(0).is_empty()); assert!(cache.scope_at(0).is_none()); } #[test] fn syntax_budget_crossing_releases_state_and_reenables_after_shrinking() { let small = "fn main() { let value = \"ok\"; }"; let mut input = lines(small); let mut remaining = MAX_SYNTAX_BYTES - small.len(); while remaining > 0 { let length = (remaining - 1).min(16 * 1024); input.push(" ".repeat(length)); remaining -= length + 1; } let path = Some(Path::new("budget.rs")); let mut cache = SyntaxCache::new(); cache.update_visible(&input, path, 0, std::slice::from_ref(&(0..1))); assert_eq!(cache.source.len(), MAX_SYNTAX_BYTES); assert_eq!(cache.error(), None); assert_eq!(cache.parses, 1); assert_token(&cache, small, 0, "fn", SyntaxKind::Keyword); assert!(cache.scope_at(0).is_some()); input.last_mut().unwrap().push(' '); cache.update_visible(&input, path, 1, std::slice::from_ref(&(0..1))); assert_eq!(cache.language(), Some("Rust")); assert!(cache.error().unwrap().contains("plain text")); assert_eq!(cache.parses, 1); assert_plain_text_storage(&cache); let notice = cache.error.as_ref().unwrap().as_ptr(); cache.update(&input, path, 1); assert_eq!(cache.error.as_ref().unwrap().as_ptr(), notice); assert_eq!(cache.parses, 1); assert_plain_text_storage(&cache); cache.update(&input, Some(Path::new("budget.py")), 1); assert_eq!(cache.language(), Some("Python")); assert!(cache.error().unwrap().starts_with("Python")); assert_plain_text_storage(&cache); cache.update(&input, Some(Path::new("budget.txt")), 1); assert_eq!(cache.language(), None); assert_eq!(cache.error(), None); assert_plain_text_storage(&cache); cache.update(&lines(small), path, 2); assert_eq!(cache.parses, 2); assert_eq!(cache.error(), None); assert_token(&cache, small, 0, "fn", SyntaxKind::Keyword); assert!(cache.scope_at(0).is_some()); } #[test] fn diff_syntax_budget_counts_the_separate_final_newline() { let mut input = vec!["//".to_owned() + &" ".repeat(16 * 1024 - 2); 63]; input.push("//".to_owned() + &" ".repeat(16 * 1024 - 65)); let path = Some(Path::new("budget.rs")); let mut cache = SyntaxCache::new(); cache.update_diff(&input, path, 0, false, None); assert_eq!(cache.source.len(), MAX_SYNTAX_BYTES); assert_eq!(cache.parses, 1); assert_eq!(kind_at(&cache, 0, 0), Some(SyntaxKind::Comment)); cache.update_diff(&input, path, 0, true, None); assert_eq!(cache.parses, 1); assert_plain_text_storage(&cache); cache.update_diff(&input, path, 0, false, Some(std::slice::from_ref(&(0..1)))); assert_eq!(cache.parses, 2); assert_eq!(cache.error(), None); assert_eq!(kind_at(&cache, 0, 0), Some(SyntaxKind::Comment)); } #[test] fn long_lines_and_large_text_never_copy_or_parse_and_clear_injections() { let markdown = "# Title\n```rust\nfn main() { \"ok\"; }\n```"; let mut cache = self::cache(markdown, "hover.md"); assert!(!cache.injections.is_empty()); assert!(cache.scope_at(2).is_some()); let parses = cache.parse_counts(); let long_line = "//".to_owned() + &" ".repeat(MAX_SYNTAX_LINE_BYTES - 1); cache.update_text(&long_line, Path::new("large.rs")); assert_eq!(cache.language(), Some("Rust")); assert_eq!(cache.parse_counts(), parses); assert!(cache.error().unwrap().contains("line")); assert_plain_text_storage(&cache); let large = " ".repeat(52 * 1024 * 1024); cache.update_text(&large, Path::new("parser.c")); assert_eq!(cache.language(), Some("C")); assert_eq!(cache.parse_counts(), parses); assert_plain_text_storage(&cache); let notice = cache.error.as_ref().unwrap().as_ptr(); cache.update_text(&large, Path::new("parser.c")); assert_eq!(cache.error.as_ref().unwrap().as_ptr(), notice); assert_plain_text_storage(&cache); let large_lines = vec![large]; cache.update(&large_lines, Some(Path::new("parser.c")), 10); assert_eq!(cache.language(), Some("C")); assert_eq!(cache.parse_counts(), parses); assert_plain_text_storage(&cache); cache.update_diff(&large_lines, Some(Path::new("parser.c")), 10, true, None); assert_eq!(cache.parse_counts(), parses); assert_plain_text_storage(&cache); cache.update_text(markdown, Path::new("hover.md")); assert!(cache.parse_counts().0 > parses.0); assert_eq!(cache.error(), None); assert!(!cache.injections.is_empty()); assert!(cache.scope_at(2).is_some()); let boundary = "//".to_owned() + &" ".repeat(MAX_SYNTAX_LINE_BYTES - 2); cache.update(&lines(&boundary), Some(Path::new("line.rs")), 50); assert_eq!(cache.error(), None); assert_eq!(kind_at(&cache, 0, 0), Some(SyntaxKind::Comment)); cache.update(&lines(&long_line), Some(Path::new("line.rs")), 51); assert_plain_text_storage(&cache); cache.update_text("", Path::new("line.rs")); assert_eq!(cache.language(), Some("Rust")); assert_eq!(cache.error(), None); assert!(cache.budget_exceeded.is_none()); assert!(cache.line(0).is_empty()); assert!(cache.scope_at(0).is_none()); let mut script = vec!["#!/usr/bin/env python3".to_owned(), long_line.clone()]; cache.update(&script, None, 60); assert_eq!(cache.language(), Some("Python")); assert_plain_text_storage(&cache); script[0] = "#!/usr/bin/env ruby".to_owned(); cache.update(&script, None, 61); assert_eq!(cache.language(), Some("Ruby")); assert!(cache.error().unwrap().starts_with("Ruby")); assert_plain_text_storage(&cache); script.pop(); cache.update(&script, None, 62); assert_eq!(cache.language(), Some("Ruby")); assert_eq!(cache.error(), None); assert!(cache.tree.is_some()); } #[test] fn unicode_minimal_edits_use_byte_points_and_never_split_codepoints() { let old = "x\n\t\"é 👩‍💻\"\n"; let new = "x\n\t\"ê 👩‍💻\"\n"; let edit = minimal_edit(old, new); assert_eq!(edit.start_byte, 4); assert_eq!(edit.start_position, Point::new(1, 2)); assert_eq!(edit.old_end_byte, 6); assert_eq!(edit.new_end_byte, 6); assert_eq!(edit.old_end_position, Point::new(1, 4)); let edit = minimal_edit("a\nβ\n", "a\n\nβ\n"); assert_eq!(edit.start_position, Point::new(1, 0)); assert_eq!(edit.old_end_position, Point::new(1, 0)); assert_eq!(edit.new_end_position, Point::new(2, 0)); let edit = minimal_edit("α", ""); assert_eq!( (edit.start_byte, edit.old_end_byte, edit.new_end_byte), (0, 2, 0) ); let edit = minimal_edit("xα", "xѱ"); assert_eq!( (edit.start_byte, edit.old_end_byte, edit.new_end_byte), (1, 3, 3) ); assert_eq!(edit.new_end_position, Point::new(0, 3)); } #[test] fn incremental_unicode_newline_insert_remove_and_undo_match_fresh_trees() { let original = "fn main() {\n\tlet text = \"é é 👩‍💻\";\n\t/* nested /* inner */ note */\n}\n"; let mut cache = self::cache(original, "a.rs"); let edits = [ original.replace("é", "ê"), original.replace("let text", "\nlet text"), original.replace("\n\t/* nested /* inner */ note */", ""), original.replace("👩‍💻", "👩‍🚀"), original.replace("\";", "\";\n\tlet n = 42;"), original.to_owned(), String::new(), original.to_owned(), ]; for (revision, source) in edits.iter().enumerate() { cache.update(&lines(source), Some(Path::new("a.rs")), revision as u64 + 1); assert_fresh(&cache, source, "a.rs"); } assert_eq!(cache.parses, edits.len() + 1); assert_eq!(cache.incremental_parses, edits.len()); } #[test] fn unfinished_source_and_multiline_offscreen_state_remain_highlighted() { let source = "fn main() {\n/* start\ninside 👩‍💻\nend */\nlet n = 42;\nlet text = \"unfinished"; let cache = self::cache(source, "a.rs"); assert!(cache.tree.as_ref().unwrap().root_node().has_error()); assert_token(&cache, source, 0, "fn", SyntaxKind::Keyword); assert_token(&cache, source, 2, "inside", SyntaxKind::Comment); assert_token(&cache, source, 4, "42", SyntaxKind::Number); assert_eq!(cache.error(), None); let source = "text = '''first\ninside\nlast'''\nanswer = 42"; let cache = self::cache(source, "a.py"); assert_token(&cache, source, 1, "inside", SyntaxKind::String); assert_token(&cache, source, 3, "42", SyntaxKind::Number); } #[test] fn markdown_heading_markers_share_title_colors_without_coloring_plain_hashes() { let source = "# One\n## Two\n### Three\n#### Four\n##### Five\n###### Six\n\nSetext\n======\n\nplain # text"; let cache = self::cache(source, "headings.md"); for row in 0..6 { assert_token( &cache, source, row, &"#".repeat(row + 1), SyntaxKind::Markup, ); } assert_token(&cache, source, 0, "One", SyntaxKind::Markup); assert_token(&cache, source, 7, "Setext", SyntaxKind::Markup); assert_token(&cache, source, 8, "======", SyntaxKind::Markup); assert_eq!(kind_at(&cache, 10, 6), None); } #[test] fn markdown_list_markers_color_bullets_and_numbers_without_coloring_plain_text() { let source = "- item\n - nested\n 12. ordered\n\n+ item\n\n* item\n\n1. ordered\n2. next\n\n3) ordered\n\n> - quoted\n> 4. ordered\n\nplain - text and 1. text\n\n\\- escaped"; let mut cache = self::cache(source, "lists.md"); for (row, marker) in [ (0, "-"), (1, "-"), (2, "12."), (4, "+"), (6, "*"), (8, "1."), (9, "2."), (11, "3)"), (13, "-"), (14, "4."), ] { assert_token(&cache, source, row, marker, SyntaxKind::Markup); } assert_eq!(kind_at(&cache, 0, 2), None); assert_eq!(kind_at(&cache, 16, 6), None); assert_eq!(kind_at(&cache, 16, 17), None); assert_ne!(kind_at(&cache, 18, 1), Some(SyntaxKind::Markup)); let edited = source.replacen("- item", "plain - item", 1); cache.update(&lines(&edited), Some(Path::new("lists.md")), 1); assert_eq!(kind_at(&cache, 0, 6), None); assert_fresh(&cache, &edited, "lists.md"); } #[test] fn markdown_inline_and_fences_inject_known_languages_and_update_incrementally() { let original = "A **bold** and `code` link.\n\n```rust\nfn main() { let n = 42; }\n```\n\n```python\ndef hello():\n return 7\n```\n\n```not-a-language\nfn not_rust() { 99; }\n```"; let mut cache = self::cache(original, "a.md"); assert_token(&cache, original, 0, "bold", SyntaxKind::Markup); assert_token(&cache, original, 0, "code", SyntaxKind::Markup); assert_token(&cache, original, 3, "fn", SyntaxKind::Keyword); assert_token(&cache, original, 3, "main", SyntaxKind::Function); assert_token(&cache, original, 3, "42", SyntaxKind::Number); assert_token(&cache, original, 7, "def", SyntaxKind::Keyword); assert_ne!(kind_at(&cache, 12, 0), Some(SyntaxKind::Keyword)); let parses = cache.injection_parses; let source = original.replace("42", "123"); cache.update(&lines(&source), Some(Path::new("a.md")), 1); assert_token(&cache, &source, 3, "123", SyntaxKind::Number); assert_fresh(&cache, &source, "a.md"); assert!(cache.injection_parses > parses); let source = source.replace("```rust", "```unknown"); cache.update(&lines(&source), Some(Path::new("a.md")), 2); assert_ne!(kind_at(&cache, 3, 0), Some(SyntaxKind::Keyword)); assert_fresh(&cache, &source, "a.md"); } #[test] fn markdown_injections_keep_anonymous_syntax_but_not_named_block_continuations() { let source = "> # Heading **bold**\n>\n> ```rust\n> fn main() {\n> let n = 3.25;\n> }\n> ```"; let cache = self::cache(source, "a.md"); assert_token(&cache, source, 0, "Heading", SyntaxKind::Markup); assert_token(&cache, source, 0, "bold", SyntaxKind::Markup); assert_token(&cache, source, 3, "main", SyntaxKind::Function); assert_token(&cache, source, 4, "3.25", SyntaxKind::Number); let rust = languages::by_name("rust").unwrap(); let layer = cache .injections .iter() .find(|layer| std::ptr::eq(layer.syntax, rust)) .unwrap(); for range in &layer.ranges { assert!(!source[range.start_byte..range.end_byte].contains('>')); } let braces = [source.find('{').unwrap(), source.find('}').unwrap()]; for byte in braces { assert!( layer .ranges .iter() .any(|range| range.start_byte <= byte && byte < range.end_byte) ); } assert_fresh(&cache, source, "a.md"); } #[test] fn html_script_style_injections_override_host_and_reuse_unchanged_trees() { let original = "
hello
\n\n"; let mut cache = self::cache(original, "a.html"); assert_token(&cache, original, 0, "div", SyntaxKind::Markup); assert_token(&cache, original, 1, "function", SyntaxKind::Keyword); assert_token(&cache, original, 1, "hello", SyntaxKind::Function); assert_token(&cache, original, 1, "42", SyntaxKind::Number); assert_token(&cache, original, 2, "7", SyntaxKind::Number); let injection_parses = cache.injection_parses; let source = original.replace("
hello", "
world"); cache.update(&lines(&source), Some(Path::new("a.html")), 1); assert_eq!(cache.injection_parses, injection_parses); assert_fresh(&cache, &source, "a.html"); let source = source.replace("42", "123"); cache.update(&lines(&source), Some(Path::new("a.html")), 2); assert_token(&cache, &source, 1, "123", SyntaxKind::Number); assert_fresh(&cache, &source, "a.html"); // Inserting at an included range's exact end must reparse, not merely edit its old tree. let source = source.replace("return 123; }", "return 123; } function added() {}"); cache.update(&lines(&source), Some(Path::new("a.html")), 3); assert_token(&cache, &source, 1, "added", SyntaxKind::Function); assert_fresh(&cache, &source, "a.html"); } #[test] fn local_definitions_disable_builtin_captures_and_propagate_highlights() { let source = "function hello(require) { require('local'); }\nrequire('global');"; let cache = self::cache(source, "a.js"); let local = source.find("require('local')").unwrap(); assert_eq!(kind_at(&cache, 0, local), None); assert_token(&cache, source, 1, "require", SyntaxKind::Function); let source = "const action = () => 42;\naction;"; let cache = self::cache(source, "a.js"); assert_token(&cache, source, 0, "action", SyntaxKind::Function); assert_token(&cache, source, 1, "action", SyntaxKind::Function); } #[test] fn overlapping_capture_specificity_priority_and_plain_injection_are_nonoverlapping() { let source = "αβγ"; let mut spans = vec![Vec::new()]; let mut events = Vec::new(); let highlights = vec![ Highlight { start: 0, end: 6, kind: Some(SyntaxKind::String), depth: 0, priority: 0, pattern: 99, }, Highlight { start: 2, end: 4, kind: Some(SyntaxKind::Function), depth: 0, priority: 0, pattern: 1, }, Highlight { start: 4, end: 6, kind: None, depth: 1, priority: i32::MIN, pattern: 0, }, Highlight { start: 0, end: 2, kind: Some(SyntaxKind::Keyword), depth: 0, priority: 10, pattern: 0, }, ]; render_spans( source, &[0], &mut spans, &mut vec![Vec::new(); 1], &highlights, &mut events, &[source_range(source)], ); assert_eq!( spans[0], vec![ SyntaxSpan { start_byte: 0, end_byte: 2, kind: SyntaxKind::Keyword }, SyntaxSpan { start_byte: 2, end_byte: 4, kind: SyntaxKind::Function }, ] ); } #[test] fn local_initializer_visibility_positive_predicates_and_noninheriting_scopes() { let syntax = languages::by_name("javascript").unwrap(); let locals = r#" (program) @local.scope ((function_declaration) @local.scope (#set! local.scope-inherits false)) (variable_declarator name: (identifier) @local.definition value: (_) @local.definition-value) (identifier) @local.reference "#; let source_query = format!( "{locals}\n{}", r#" ((identifier) @function (#is-not? local)) ((identifier) @string (#is? local)) (variable_declarator name: (identifier) @type) "# ); let query = Query::new(&syntax.language, &source_query).unwrap(); let highlights_start = (0..query.pattern_count()) .filter(|pattern| query.start_byte_for_pattern(*pattern) < locals.len() + 1) .count(); let configuration = Configuration::new(syntax, query, 0, highlights_start); let source = "const value = value;\nvalue;\nfunction inner() { value; }"; let cache = self::cache(source, "a.js"); let mut highlights = Vec::new(); collect_layer( &configuration, cache.tree.as_ref().unwrap(), source, &[source_range(source)], &[source_range(source)], &mut None, 0, None, &mut QueryCursor::new(), &mut Vec::new(), &mut highlights, ) .unwrap(); let mut spans = vec![Vec::new(); 3]; let starts = [ 0, source.find("\nvalue;").unwrap() + 1, source.find("\nfunction").unwrap() + 1, ]; render_spans( source, &starts, &mut spans, &mut vec![Vec::new(); 3], &highlights, &mut Vec::new(), &[source_range(source)], ); let kind = |row: usize, byte: usize| { spans[row] .iter() .find_map(|span| (span.start_byte <= byte && byte < span.end_byte).then_some(span.kind)) }; assert_eq!(kind(0, 6), Some(SyntaxKind::Type)); assert_eq!(kind(0, 14), Some(SyntaxKind::Function)); assert_eq!(kind(1, 0), Some(SyntaxKind::Type)); assert_eq!(kind(2, 19), Some(SyntaxKind::Function)); let mut context = None; for row in [1, 2] { let window = [Range { start_byte: starts[row], end_byte: starts.get(row + 1).copied().unwrap_or(source.len()), start_point: Point::new(row, 0), end_point: point_at(source, starts.get(row + 1).copied().unwrap_or(source.len())), }]; let mut visible_highlights = Vec::new(); collect_layer( &configuration, cache.tree.as_ref().unwrap(), source, &[source_range(source)], &window, &mut context, 0, None, &mut QueryCursor::new(), &mut Vec::new(), &mut visible_highlights, ) .unwrap(); let mut visible_spans = vec![Vec::new(); 3]; render_spans( source, &starts, &mut visible_spans, &mut vec![Vec::new(); 3], &visible_highlights, &mut Vec::new(), &window, ); assert_eq!(visible_spans[row], spans[row]); assert!( visible_spans .iter() .enumerate() .all(|(other, spans)| other == row || spans.is_empty()) ); } } #[test] fn grammar_and_unsupported_predicate_errors_are_not_silent_empty_queries() { static INVALID: LazyLock<[LanguageSpec; 2]> = LazyLock::new(|| { let language = languages::by_name("rust").unwrap(); [ "(not_a_rust_node) @keyword", "((identifier) @function (#unsupported? @function))", ] .map(|query| LanguageSpec { id: language.id, name: "invalid test grammar query", language: language.language.clone(), highlights: query, injections: "", locals: "", }) }); for syntax in INVALID.iter() { assert!(compile_configuration(syntax).is_err()); } } #[test] fn combined_tagged_templates_and_nested_markdown_html_keep_real_injections() { let source = "const page = html`
${value}
`;"; let cache = self::cache(source, "a.js"); assert_token(&cache, source, 0, "div", SyntaxKind::Markup); assert_token(&cache, source, 0, "inside", SyntaxKind::Function); assert_token(&cache, source, 0, "42", SyntaxKind::Number); assert_fresh(&cache, source, "a.js"); let source = "```html\n\n```"; let cache = self::cache(source, "a.md"); assert_token(&cache, source, 1, "nested", SyntaxKind::Function); assert_token(&cache, source, 1, "7", SyntaxKind::Number); assert_fresh(&cache, source, "a.md"); let source = "fn main() { println!(\"{}\", 42); }"; let cache = self::cache(source, "a.rs"); assert_token(&cache, source, 0, "42", SyntaxKind::Number); assert!(cache.injections.len() <= 16); assert_fresh(&cache, source, "a.rs"); } #[test] fn parser_state_is_isolated_between_buffers() { let mut first = self::cache("/* open\ninside", "a.rs"); let second = self::cache("fn main() { let n = 42; }", "b.rs"); first.update( &lines("/* open */\nfn other() {}"), Some(Path::new("a.rs")), 1, ); assert_token( &second, "fn main() { let n = 42; }", 0, "fn", SyntaxKind::Keyword, ); assert_eq!(second.parses, 1); assert_eq!(first.incremental_parses, 1); }