use std::cell::OnceCell; use std::cmp::Reverse; use std::collections::{BTreeSet, HashMap, VecDeque}; use std::path::{Path, PathBuf}; use std::sync::LazyLock; use tree_sitter::{ InputEdit, Node, Parser, Point, Query, QueryCursor, Range, StreamingIterator, Tree, }; mod languages; use languages::LanguageSpec; pub(crate) const RUST_ID: &str = "rust"; pub(crate) const RUBY_ID: &str = "ruby"; pub(crate) const PYTHON_ID: &str = "python"; // Full trees and their indexes are optional; editing remains available beyond these budgets. const MAX_SYNTAX_BYTES: usize = 1024 * 1024; const MAX_SYNTAX_LINE_BYTES: usize = 64 * 1024; const SOURCE_BUDGET_NOTICE: &str = "source exceeds 1 MiB syntax budget"; const LINE_BUDGET_NOTICE: &str = "line exceeds 64 KiB syntax budget"; pub(crate) fn is_markdown_path(path: &Path) -> bool { languages::detect(Some(path), None).is_some_and(|syntax| syntax.name == "Markdown") } pub(crate) fn language_id(path: Option<&Path>, first_line: Option<&str>) -> Option<&'static str> { languages::detect(path, first_line).map(|syntax| syntax.id) } #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum SyntaxKind { Comment, String, Keyword, Number, Type, Function, Markup, } /// A syntax range in original line bytes; string coverage may include its newline. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct SyntaxSpan { pub start_byte: usize, pub end_byte: usize, pub kind: SyntaxKind, } /// Innermost named declaration, with its full enclosing path and end-exclusive source rows. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) struct ScopeContext<'a> { pub start_row: usize, pub end_row: usize, pub name: Point, pub label: &'a str, } struct CachedScope { start_row: usize, end_row: usize, name: Point, label: String, } struct ScopeIndex { declarations: Vec, rows: Vec>, } /// Frontend-owned, per-buffer parser state. Revision and path identify unchanged input. #[derive(Default)] pub struct SyntaxCache { revision: Option, path: Option, final_newline: bool, syntax: Option<&'static LanguageSpec>, budget_exceeded: Option<&'static str>, source: String, next_source: String, line_starts: Vec, lines: Vec>, highlighted: Vec, string_spans: Vec>, context: Option, parser: Parser, tree: Option, injection_parser: Parser, injections: Vec, scopes: OnceCell, cursor: QueryCursor, captures: Vec, highlights: Vec, events: Vec<(usize, bool, usize)>, error: Option, #[cfg(test)] parses: usize, #[cfg(test)] incremental_parses: usize, #[cfg(test)] injection_parses: usize, } impl SyntaxCache { pub fn new() -> Self { Self::default() } pub(crate) fn update_text(&mut self, text: &str, path: &Path) { // Check lengths before equality, splitting into owned lines, or copying the source. let budget_exceeded = if text.len() > MAX_SYNTAX_BYTES { Some(SOURCE_BUDGET_NOTICE) } else if text .split('\n') .any(|line| line.len() > MAX_SYNTAX_LINE_BYTES) { Some(LINE_BUDGET_NOTICE) } else { None }; if let Some(reason) = budget_exceeded { let syntax = languages::detect(Some(path), None) .or_else(|| languages::detect(None, Some(syntax_prefix(text)))); let language_changed = !same_language(self.syntax, syntax); self.revision = None; self.final_newline = false; if self.path.as_deref() != Some(path) { self.path = Some(path.to_path_buf()); } self.syntax = syntax; if self.budget_exceeded != Some(reason) || language_changed { self.use_plain_text(reason); } return; } if self.budget_exceeded.is_none() && self.source == text && self.path.as_deref() == Some(path) && self.highlighted.iter().all(|highlighted| *highlighted) { return; } let lines: Vec = text.split('\n').map(str::to_owned).collect(); self.update( &lines, Some(path), self.revision.unwrap_or(0).wrapping_add(1), ); } /// `revision` must change whenever line content changes; path changes are detected separately. pub fn update(&mut self, lines: &[String], path: Option<&Path>, revision: u64) { self.update_rows( lines, path, revision, std::slice::from_ref(&(0..lines.len())), false, ); } /// Parse the original full source, but collect colors only for these end-exclusive source rows. /// Unrequested rows may be empty; requesting more rows at the same revision reuses parsed trees. pub(crate) fn update_visible( &mut self, lines: &[String], path: Option<&Path>, revision: u64, visible: &[std::ops::Range], ) { self.update_rows(lines, path, revision, visible, false); } /// Diff sources store a final line break separately instead of a trailing empty row. pub(crate) fn update_diff( &mut self, lines: &[String], path: Option<&Path>, revision: u64, final_newline: bool, visible: Option<&[std::ops::Range]>, ) { let all_rows = 0..lines.len(); self.update_rows( lines, path, revision, visible.unwrap_or(std::slice::from_ref(&all_rows)), final_newline && !lines.is_empty(), ); let _ = self.scope_at(0); } fn update_rows( &mut self, lines: &[String], path: Option<&Path>, revision: u64, visible: &[std::ops::Range], final_newline: bool, ) { let path_changed = self.path.as_deref() != path; let revision_changed = self.revision != Some(revision) || self.final_newline != final_newline; if self.budget_exceeded.is_some() && !path_changed && !revision_changed { return; } let syntax = if path_changed || revision_changed { languages::detect(path, lines.first().map(|line| syntax_prefix(line))) } else { self.syntax }; let language_changed = !same_language(self.syntax, syntax); self.revision = Some(revision); self.final_newline = final_newline; if path_changed { self.path = path.map(Path::to_path_buf); } self.syntax = syntax; let length = if revision_changed { syntax_source_length(lines, final_newline) } else if let Some(reason) = self.budget_exceeded { Err(reason) } else { Ok(self.source.len()) }; let length = match length { Ok(length) => length, Err(reason) => { if self.budget_exceeded != Some(reason) || language_changed { self.use_plain_text(reason); } return; } }; if self.budget_exceeded.take().is_some() { self.error = None; } let mut edit = None; if revision_changed { self.next_source.clear(); self.next_source.reserve(length); for (row, line) in lines.iter().enumerate() { if row > 0 { self.next_source.push('\n'); } self.next_source.push_str(line); } if final_newline { self.next_source.push('\n'); } if self.source != self.next_source { edit = Some(minimal_edit(&self.source, &self.next_source)); std::mem::swap(&mut self.source, &mut self.next_source); } self.line_starts.clear(); let mut start = 0; for line in lines { self.line_starts.push(start); start += line.len() + 1; } if final_newline { self.line_starts.push(self.source.len()); } self.lines.resize_with(lines.len(), Vec::new); self.string_spans.resize_with(lines.len(), Vec::new); self.highlighted.resize(lines.len(), false); } if edit.is_some() || language_changed { for spans in self.lines.iter_mut().chain(&mut self.string_spans) { spans.clear(); } self.highlighted.fill(false); self.context = None; self.scopes.take(); } if language_changed { self.tree = None; self.injections.clear(); } let mut requested = Vec::new(); for rows in visible { let end = rows.end.min(lines.len()); let mut row = rows.start.min(end); while row < end { if self.highlighted[row] { row += 1; continue; } let start = row; while row < end && !self.highlighted[row] { row += 1; } requested.push(Range { start_byte: self.line_starts[start], end_byte: self .line_starts .get(row) .copied() .unwrap_or(self.source.len()), start_point: Point::new(start, 0), end_point: if row < lines.len() { Point::new(row, 0) } else if final_newline { Point::new(lines.len(), 0) } else { Point::new( lines.len().saturating_sub(1), lines.last().map_or(0, String::len), ) }, }); } } normalize_ranges(&mut requested); if requested.is_empty() && edit.is_none() && !language_changed { return; } self.error = None; let Some(syntax) = syntax else { self.tree = None; self.injections.clear(); self.highlighted.fill(true); return; }; if let Err(error) = self.highlight(syntax, edit.as_ref(), language_changed, &requested) { self.error = Some(format!("{} syntax: {error}", syntax.name)); self.tree = None; self.injections.clear(); self.context = None; self.scopes.take(); self.highlighted.fill(false); for spans in self.lines.iter_mut().chain(&mut self.string_spans) { spans.clear(); } } else { for rows in visible { let end = rows.end.min(lines.len()); self.highlighted[rows.start.min(end)..end].fill(true); } } } fn use_plain_text(&mut self, reason: &'static str) { // Drop parser arenas as well as source/span capacities when crossing the budget. *self = Self { revision: self.revision, path: self.path.take(), final_newline: self.final_newline, syntax: self.syntax, budget_exceeded: Some(reason), error: self .syntax .map(|syntax| format!("{}: plain text ({reason})", syntax.name)), #[cfg(test)] parses: self.parses, #[cfg(test)] incremental_parses: self.incremental_parses, #[cfg(test)] injection_parses: self.injection_parses, ..Self::default() }; } /// Sorted, non-overlapping, end-exclusive ranges. Unclassified text uses the normal foreground. pub fn line(&self, row: usize) -> &[SyntaxSpan] { self.lines.get(row).map_or(&[], Vec::as_slice) } /// Lazily index existing trees; subsequent lookups neither allocate nor reparse. pub(crate) fn scope_at(&self, row: usize) -> Option> { let syntax = self.syntax?; let tree = self.tree.as_ref()?; if row >= self.lines.len() || (!has_scope_language(syntax.id) && !self .injections .iter() .any(|layer| has_scope_language(layer.syntax.id))) { return None; } let index = self.scopes.get_or_init(|| { let mut index = ScopeIndex { declarations: Vec::new(), rows: vec![None; self.lines.len()], }; collect_scopes( tree.root_node(), syntax.id, &self.source, &[], &mut index.declarations, &mut index.rows, ); for layer in &self.injections { collect_scopes( layer.tree.root_node(), layer.syntax.id, &self.source, &layer.ranges, &mut index.declarations, &mut index.rows, ); } index }); let scope = &index.declarations[index.rows[row]?]; Some(ScopeContext { start_row: scope.start_row, end_row: scope.end_row, name: scope.name, label: &scope.label, }) } /// String coverage before foreground overrides, including embedded-language highlighting. /// An included line break has end_byte == line.len() + 1; foreground spans exclude it. pub(crate) fn string_spans(&self, row: usize) -> &[SyntaxSpan] { self.string_spans.get(row).map_or(&[], Vec::as_slice) } /// Bundled grammar name, or `None` for unknown languages and plain text. pub fn language(&self) -> Option<&str> { self.syntax.map(|syntax| syntax.name) } /// Why highlighting is unavailable: a deliberate plain-text fallback or grammar/query failure. pub fn error(&self) -> Option<&str> { self.error.as_deref() } #[cfg(test)] pub(crate) fn parse_counts(&self) -> (usize, usize) { (self.parses, self.injection_parses) } fn highlight( &mut self, syntax: &'static LanguageSpec, edit: Option<&InputEdit>, language_changed: bool, visible: &[Range], ) -> Result<(), String> { let primary_configuration = configuration(syntax)?; if language_changed || self.tree.is_none() { self.parser .set_language(&syntax.language) .map_err(|error| error.to_string())?; } if let Some(edit) = edit && let Some(tree) = &mut self.tree { tree.edit(edit); } if edit.is_some() || self.tree.is_none() { #[cfg(test)] { self.parses += 1; self.incremental_parses += usize::from(self.tree.is_some()); } self.tree = Some( self.parser .parse(self.source.as_bytes(), self.tree.as_ref()) .ok_or_else(|| "parser stopped without a tree".to_owned())?, ); } let full_range = Range { start_byte: 0, end_byte: self.source.len(), start_point: Point::new(0, 0), end_point: Point::new( self.line_starts.len().saturating_sub(1), self.source.len() - self.line_starts.last().copied().unwrap_or(0), ), }; self.highlights.clear(); let mut queue: VecDeque<_> = collect_layer( primary_configuration, self.tree.as_ref().unwrap(), &self.source, &[full_range], visible, &mut self.context, 0, None, &mut self.cursor, &mut self.captures, &mut self.highlights, )? .into(); let mut previous = std::mem::take(&mut self.injections); for layer in &mut previous { layer.dirty = edit.is_some_and(|edit| { layer.ranges.iter().any(|range| { edit.start_byte <= range.end_byte && edit.old_end_byte >= range.start_byte }) }); if let Some(edit) = edit { layer.tree.edit(edit); layer.ranges = layer.tree.included_ranges(); layer.context = None; } } while let Some(request) = queue.pop_front() { // An identical language/range ancestor is a genuine injection cycle. Keep that host // layer, but do not inject it into itself again (e.g. a grammar's injection.self). if request.ancestors.iter().any(|(spec, ranges)| { std::ptr::eq(*spec, request.syntax) && *ranges == request.ranges }) { continue; } if request.depth > 16 { return Err("injection nesting exceeds 16 language layers".to_owned()); } let configuration = configuration(request.syntax)?; let previous_index = previous .iter() .position(|layer| { std::ptr::eq(layer.syntax, request.syntax) && layer.depth == request.depth && layer.ranges == request.ranges }) .or_else(|| { previous.iter().position(|layer| { std::ptr::eq(layer.syntax, request.syntax) && layer.depth == request.depth }) }); let old = previous_index.map(|index| previous.swap_remove(index)); let unchanged = old .as_ref() .is_some_and(|layer| !layer.dirty && layer.ranges == request.ranges); let (tree, mut context) = if unchanged { let old = old.unwrap(); (old.tree, old.context) } else { self.injection_parser .set_language(&request.syntax.language) .map_err(|error| error.to_string())?; self.injection_parser .set_included_ranges(&request.ranges) .map_err(|error| error.to_string())?; #[cfg(test)] { self.injection_parses += 1; } let tree = self .injection_parser .parse( self.source.as_bytes(), old.as_ref().map(|layer| &layer.tree), ) .ok_or_else(|| { format!( "{} injection parser stopped without a tree", request.syntax.name ) })?; (tree, None) }; // Markdown's inline grammar refines the block grammar, so plain inline text keeps // its enclosing heading/link foreground. Other languages replace host strings/markup. let refines_host = languages::by_name("markdown_inline") .is_some_and(|inline| std::ptr::eq(inline, request.syntax)); if !refines_host { for range in request .ranges .iter() .filter(|range| intersects(visible, range.start_byte, range.end_byte)) { self.highlights.push(Highlight { start: range.start_byte, end: range.end_byte, kind: None, depth: request.depth, priority: i32::MIN, pattern: 0, }); } } let mut children = collect_layer( configuration, &tree, &self.source, &request.ranges, visible, &mut context, request.depth, Some(request.parent), &mut self.cursor, &mut self.captures, &mut self.highlights, )?; let mut ancestors = request.ancestors; ancestors.push((request.syntax, request.ranges.clone())); for child in &mut children { child.ancestors = ancestors.clone(); } queue.extend(children); self.injections.push(Layer { syntax: request.syntax, tree, ranges: request.ranges, depth: request.depth, dirty: false, context, }); } render_spans( &self.source, &self.line_starts, &mut self.lines, &mut self.string_spans, &self.highlights, &mut self.events, visible, ); Ok(()) } } fn syntax_prefix(text: &str) -> &str { let mut end = text.len().min(MAX_SYNTAX_LINE_BYTES); while !text.is_char_boundary(end) { end -= 1; } text[..end].split('\n').next().unwrap_or_default() } fn syntax_source_length(lines: &[String], final_newline: bool) -> Result { let mut length = lines .len() .saturating_sub(1) .saturating_add(usize::from(final_newline)); if length > MAX_SYNTAX_BYTES { return Err(SOURCE_BUDGET_NOTICE); } for line in lines { if line.len() > MAX_SYNTAX_LINE_BYTES { return Err(LINE_BUDGET_NOTICE); } length = length.saturating_add(line.len()); if length > MAX_SYNTAX_BYTES { return Err(SOURCE_BUDGET_NOTICE); } } Ok(length) } fn has_scope_language(language: &str) -> bool { matches!( language, "rust" | "python" | "ruby" | "javascript" | "typescript" | "typescriptreact" | "go" | "c" | "cpp" | "shellscript" | "zig" ) } fn collect_scopes( root: Node<'_>, language: &str, source: &str, ranges: &[Range], scopes: &mut Vec, rows: &mut [Option], ) { if !has_scope_language(language) { return; } let mut cursor = root.walk(); let mut ancestors: Vec<(usize, usize)> = Vec::new(); loop { let node = cursor.node(); while ancestors .last() .is_some_and(|(_, end)| *end <= node.start_byte()) { ancestors.pop(); } if let Some((extent, name, label)) = scope_declaration(node, language, source).filter(|(extent, _, _)| { ranges.is_empty() || ranges.iter().any(|range| { range.start_byte <= extent.start_byte() && extent.end_byte() <= range.end_byte }) }) { let parent = ancestors .last() .map_or("", |(index, _)| scopes[*index].label.as_str()); let label = if parent.is_empty() { label } else { format!("{parent} › {label}") }; let start_row = extent.start_position().row; let end = extent.end_position(); let end_row = end.row + usize::from(end.column > 0); let index = scopes.len(); scopes.push(CachedScope { start_row, end_row, name, label, }); let end_row = end_row.min(rows.len()); rows[start_row.min(end_row)..end_row].fill(Some(index)); ancestors.push((index, node.end_byte())); } if cursor.goto_first_child() { continue; } while !cursor.goto_next_sibling() { if !cursor.goto_parent() { return; } } } } fn scope_text<'a>(node: Node<'_>, source: &'a str) -> Option<&'a str> { (!node.has_error() && !node.is_missing() && node.kind() != "computed_property_name") .then(|| source.get(node.byte_range())) .flatten() .filter(|name| !name.trim().is_empty()) } fn scope_declaration<'a>( node: Node<'a>, language: &str, source: &str, ) -> Option<(Node<'a>, Point, String)> { if !node.is_named() || node.is_missing() { return None; } let field = |name| node.child_by_field_name(name); let mut extent = node; let name = match (language, node.kind()) { ("rust", "impl_item") => { let target = scope_text(field("type")?, source)?; let label = match field("trait").and_then(|node| scope_text(node, source)) { Some(trait_name) => scope_label(&["impl", trait_name, "for", target]), None => scope_label(&["impl", target]), }; return Some((node, field("type")?.start_position(), label)); } ( "rust", "function_item" | "function_signature_item" | "trait_item" | "mod_item" | "struct_item" | "enum_item", ) | ("python", "function_definition" | "class_definition") | ("ruby", "class" | "module" | "method" | "singleton_method") | ( "javascript" | "typescript" | "typescriptreact", "function_declaration" | "generator_function_declaration" | "method_definition" | "class_declaration" | "class" | "abstract_class_declaration" | "interface_declaration" | "internal_module", ) | ("go", "function_declaration") | ( "c" | "cpp", "class_specifier" | "struct_specifier" | "union_specifier" | "enum_specifier" | "namespace_definition", ) | ("shellscript", "function_definition") | ("zig", "function_declaration") => field("name")?, ("go", "method_declaration") => { let receiver = field("receiver")? .named_child(0)? .child_by_field_name("type")?; let receiver = scope_text(receiver, source)?; let name = field("name")?; return Some(( node, name.start_position(), scope_label(&[receiver, "›", scope_text(name, source)?]), )); } ("go", "type_spec") if field("type") .is_some_and(|node| matches!(node.kind(), "struct_type" | "interface_type")) => { field("name")? } ("c" | "cpp", "function_definition") => { let mut declarator = field("declarator")?; loop { if matches!( declarator.kind(), "identifier" | "field_identifier" | "qualified_identifier" | "operator_name" | "destructor_name" ) { break declarator; } declarator = declarator.child_by_field_name("declarator").or_else(|| { matches!( declarator.kind(), "reference_declarator" | "parenthesized_declarator" ) .then(|| declarator.named_child(0)) .flatten() })?; } } ( "javascript" | "typescript" | "typescriptreact", "arrow_function" | "function_expression" | "generator_function", ) => { if let Some(name) = field("name") { name } else { extent = node.parent()?; match extent.kind() { "variable_declarator" => { let name = extent.child_by_field_name("name")?; (name.kind() == "identifier").then_some(name)? } "pair" => extent.child_by_field_name("key")?, "field_definition" => extent.child_by_field_name("property")?, "public_field_definition" => extent.child_by_field_name("name")?, "assignment_expression" => { let name = extent.child_by_field_name("left")?; matches!(name.kind(), "identifier" | "member_expression").then_some(name)? } _ => return None, } } } ( "zig", "struct_declaration" | "union_declaration" | "enum_declaration" | "opaque_declaration", ) => { extent = node.parent()?; if extent.kind() != "variable_declaration" || node.prev_sibling().is_none_or(|node| node.kind() != "=") { return None; } let mut cursor = extent.walk(); extent .named_children(&mut cursor) .find(|node| node.kind() == "identifier")? } _ => return None, }; if let Some(parent) = node.parent() && parent.kind() == "decorated_definition" { extent = parent; } Some(( extent, name.start_position(), scope_label(&[scope_text(name, source)?]), )) } fn scope_label(parts: &[&str]) -> String { let mut label = String::new(); for word in parts.iter().flat_map(|part| part.split_whitespace()) { if !label.is_empty() { label.push(' '); } label.extend( word.chars() .map(|ch| if ch.is_control() { '�' } else { ch }), ); } label } fn same_language(left: Option<&LanguageSpec>, right: Option<&LanguageSpec>) -> bool { match (left, right) { (Some(left), Some(right)) => std::ptr::eq(left, right), (None, None) => true, _ => false, } } /// One aggregate edit, including byte columns (not character/display columns). fn minimal_edit(old: &str, new: &str) -> InputEdit { let mut start = old .bytes() .zip(new.bytes()) .take_while(|(a, b)| a == b) .count(); while !old.is_char_boundary(start) || !new.is_char_boundary(start) { start -= 1; } let mut suffix = old.as_bytes()[start..] .iter() .rev() .zip(new.as_bytes()[start..].iter().rev()) .take_while(|(a, b)| a == b) .count(); while !old.is_char_boundary(old.len() - suffix) || !new.is_char_boundary(new.len() - suffix) { suffix -= 1; } let old_end = old.len() - suffix; let new_end = new.len() - suffix; InputEdit { start_byte: start, old_end_byte: old_end, new_end_byte: new_end, start_position: point_at(old, start), old_end_position: point_at(old, old_end), new_end_position: point_at(new, new_end), } } fn point_at(source: &str, byte: usize) -> Point { let prefix = &source.as_bytes()[..byte]; Point::new( prefix.iter().filter(|byte| **byte == b'\n').count(), prefix .iter() .rposition(|byte| *byte == b'\n') .map_or(byte, |last| byte - last - 1), ) } struct Configuration { syntax: &'static LanguageSpec, query: Query, context_query: Query, highlights_query: Query, locals_start: usize, highlights_start: usize, } impl Configuration { fn new( syntax: &'static LanguageSpec, query: Query, locals_start: usize, highlights_start: usize, ) -> Self { let mut context_query = query.deep_clone(); let mut highlights_query = query.deep_clone(); for pattern in 0..query.pattern_count() { if pattern < highlights_start { highlights_query.disable_pattern(pattern); } else { context_query.disable_pattern(pattern); } } Self { syntax, query, context_query, highlights_query, locals_start, highlights_start, } } } fn configuration(syntax: &'static LanguageSpec) -> Result<&'static Configuration, String> { static CONFIGURATIONS: LazyLock)>> = LazyLock::new(|| { languages::all() .map(|syntax| (syntax, compile_configuration(syntax))) .collect() }); let (_, result) = CONFIGURATIONS .iter() .find(|(spec, _)| std::ptr::eq(*spec, syntax)) .ok_or_else(|| format!("missing {} query configuration", syntax.name))?; result.as_ref().map_err(Clone::clone) } fn compile_configuration(syntax: &'static LanguageSpec) -> Result { let source = format!( "{}\n{}\n{}", syntax.injections, syntax.locals, syntax.highlights ); let locals_offset = syntax.injections.len() + 1; let highlights_offset = locals_offset + syntax.locals.len() + 1; let query = Query::new(&syntax.language, &source).map_err(|error| error.to_string())?; let mut locals_start = 0; let mut highlights_start = 0; for pattern in 0..query.pattern_count() { let start = query.start_byte_for_pattern(pattern); locals_start += usize::from(start < locals_offset); highlights_start += usize::from(start < highlights_offset); if let Some(predicate) = query.general_predicates(pattern).first() { return Err(format!( "unsupported query predicate #{}", predicate.operator )); } if let Some((property, _)) = query .property_predicates(pattern) .iter() .find(|(property, _)| property.key.as_ref() != "local") { return Err(format!( "unsupported query property predicate {}", property.key )); } } Ok(Configuration::new( syntax, query, locals_start, highlights_start, )) } fn classify(capture: &str, numeric_literal: bool) -> Option { let mut parts = capture.split('.'); match parts.next()? { "comment" => Some(SyntaxKind::Comment), "string" | "character" => Some(SyntaxKind::String), "keyword" | "boolean" | "storage" | "conditional" | "repeat" | "include" | "exception" => { Some(SyntaxKind::Keyword) } "number" | "float" => Some(SyntaxKind::Number), "constant" if numeric_literal || parts.any(|part| part == "numeric") => { Some(SyntaxKind::Number) } "constant" => Some(SyntaxKind::Keyword), "type" | "constructor" => Some(SyntaxKind::Type), "function" | "method" => Some(SyntaxKind::Function), "markup" | "tag" | "text" => Some(SyntaxKind::Markup), _ => None, } } struct Layer { syntax: &'static LanguageSpec, tree: Tree, ranges: Vec, depth: usize, dirty: bool, context: Option, } #[derive(Clone)] struct Injection { syntax: &'static LanguageSpec, parent: &'static LanguageSpec, ranges: Vec, depth: usize, ancestors: Vec<(&'static LanguageSpec, Vec)>, } struct LayerContext { captures: Vec, injections: Vec, } #[derive(Clone, Copy, Eq, PartialEq)] struct Capture { node: usize, start: usize, end: usize, capture: u32, pattern: usize, value_end: usize, numeric_literal: bool, } impl Capture { fn new( capture: &tree_sitter::QueryCapture<'_>, pattern: usize, value_end: usize, name: &str, ) -> Self { Self { node: capture.node.id(), start: capture.node.start_byte(), end: capture.node.end_byte(), capture: capture.index, pattern, value_end, numeric_literal: name.starts_with("constant") && matches!( capture.node.kind(), "integer_literal" | "float_literal" | "number_literal" | "integer" | "float" | "number" ), } } } struct Highlight { start: usize, end: usize, kind: Option, depth: usize, priority: i32, pattern: usize, } struct Definition { value_end: usize, highlight: Option, } struct Scope<'a> { end: usize, inherits: bool, definitions: HashMap<&'a str, Vec>, } #[allow(clippy::too_many_arguments)] fn collect_layer( configuration: &Configuration, tree: &Tree, source: &str, ranges: &[Range], visible: &[Range], context: &mut Option, depth: usize, parent: Option<&'static LanguageSpec>, cursor: &mut QueryCursor, captures: &mut Vec, highlights: &mut Vec, ) -> Result, String> { let query = &configuration.query; let names = query.capture_names(); captures.clear(); let mut injections; if let Some(context) = context { captures.extend_from_slice(&context.captures); injections = context.injections.clone(); } else { injections = Vec::new(); let mut combined: HashMap<(usize, usize), usize> = HashMap::new(); cursor.set_byte_range(0..source.len()); let mut matches = cursor.matches( &configuration.context_query, tree.root_node(), source.as_bytes(), ); while let Some(found) = matches.next() { let pattern = found.pattern_index; if pattern < configuration.locals_start { let mut language = None; let mut nodes = Vec::new(); for capture in found.captures() { match names[capture.index as usize] { "injection.language" => { language = source[capture.node.byte_range()] .split_whitespace() .next() .and_then(languages::by_name); } "injection.content" => nodes.push(capture.node), _ => {} } } let mut include_children = false; let mut is_combined = false; for property in query.property_settings(pattern) { match property.key.as_ref() { "injection.language" => { language = property.value.as_deref().and_then(languages::by_name); } "injection.self" => language = Some(configuration.syntax), "injection.parent" => language = parent, "injection.include-children" => include_children = true, "injection.combined" => is_combined = true, _ => {} } } if let Some(language) = language { let mut content = injection_ranges(&nodes, ranges, include_children); if !content.is_empty() { let key = (pattern, language as *const LanguageSpec as usize); if is_combined && let Some(index) = combined.get(&key) { let injection: &mut Injection = &mut injections[*index]; injection.ranges.append(&mut content); } else { if is_combined { combined.insert(key, injections.len()); } injections.push(Injection { syntax: language, parent: configuration.syntax, ranges: content, depth: depth + 1, ancestors: vec![(configuration.syntax, ranges.to_vec())], }); } } } continue; } let value_end = found .captures() .iter() .find(|capture| names[capture.index as usize] == "local.definition-value") .map_or(0, |capture| capture.node.end_byte()); for capture in found.captures() { let name = names[capture.index as usize]; if name.starts_with('_') || name == "local.definition-value" { continue; } captures.push(Capture::new(capture, pattern, value_end, name)); } } if cursor.did_exceed_match_limit() { return Err(format!( "{} query exceeded its match limit", configuration.syntax.name )); } for injection in &mut injections { normalize_ranges(&mut injection.ranges); } // Locals must be resolved in source order, including definitions above the viewport. // Query only their nodes for the foreground that references inherit. let definitions: BTreeSet<_> = captures .iter() .filter(|capture| names[capture.capture as usize] == "local.definition") .map(|capture| capture.node) .collect(); let mut definition_ranges: Vec<_> = captures .iter() .filter(|capture| definitions.contains(&capture.node)) .map(|capture| Range { start_byte: capture.start, end_byte: capture.end, start_point: Point::new(0, 0), end_point: Point::new(0, 0), }) .collect(); normalize_ranges(&mut definition_ranges); collect_highlight_captures( configuration, tree, source, &definition_ranges, Some(&definitions), cursor, captures, )?; *context = Some(LayerContext { captures: captures.clone(), injections: injections.clone(), }); } collect_highlight_captures(configuration, tree, source, visible, None, cursor, captures)?; captures.sort_unstable_by_key(|capture| { ( capture.start, Reverse(capture.end), capture.node, capture.pattern, capture.capture, ) }); captures.dedup(); let mut scopes = vec![Scope { end: usize::MAX, inherits: false, definitions: HashMap::new(), }]; let mut start = 0; while start < captures.len() { let first = captures[start]; let mut end = start + 1; while end < captures.len() && captures[end].node == first.node { end += 1; } let group = &captures[start..end]; while scopes.len() > 1 && first.start >= scopes.last().unwrap().end { scopes.pop(); } let mut definition = None; let mut reference = None; let mut is_local = false; for capture in group .iter() .filter(|capture| capture.pattern < configuration.highlights_start) { match names[capture.capture as usize] { "local.scope" => { let inherits = query .property_settings(capture.pattern) .iter() .find(|property| property.key.as_ref() == "local.scope-inherits") .is_none_or(|property| { property .value .as_deref() .is_none_or(|value| value == "true") }); scopes.push(Scope { end: capture.end, inherits, definitions: HashMap::new(), }); } "local.definition" => { is_local = true; definition = Some(capture.value_end); } "local.reference" if definition.is_none() => { let name = &source[capture.start..capture.end]; for scope in scopes.iter().rev() { if let Some(found) = scope.definitions.get(name).and_then(|definitions| { definitions .iter() .rev() .find(|definition| capture.start >= definition.value_end) }) { is_local = true; reference = Some(found.highlight); break; } if !scope.inherits { break; } } } _ => {} } } let selected = group .iter() .filter(|capture| { capture.pattern >= configuration.highlights_start && query.property_predicates(capture.pattern).iter().all( |(property, positive)| { let local = property .capture_id .map_or(is_local, |id| id == capture.capture as usize && is_local); local == *positive }, ) }) .max_by_key(|capture| { ( capture_priority(query, capture), capture.pattern, capture.capture, ) }); let kind = reference.unwrap_or_else(|| { selected.and_then(|capture| { classify(names[capture.capture as usize], capture.numeric_literal) }) }); if let Some(value_end) = definition { let name = &source[first.start..first.end]; scopes .last_mut() .unwrap() .definitions .entry(name) .or_default() .push(Definition { value_end, highlight: kind, }); } if let Some(kind) = kind { let selected = selected.copied().unwrap_or(first); let range_start = ranges.partition_point(|range| range.end_byte <= first.start); for range in ranges[range_start..] .iter() .take_while(|range| range.start_byte < first.end) { let low = first.start.max(range.start_byte); let high = first.end.min(range.end_byte); if low < high && intersects(visible, low, high) { highlights.push(Highlight { start: low, end: high, kind: Some(kind), depth, priority: capture_priority(query, &selected), pattern: selected.pattern, }); } } } start = end; } Ok(injections) } fn intersects(ranges: &[Range], start: usize, end: usize) -> bool { ranges .get(ranges.partition_point(|range| range.end_byte <= start)) .is_some_and(|range| range.start_byte < end) } #[allow(clippy::too_many_arguments)] fn collect_highlight_captures( configuration: &Configuration, tree: &Tree, source: &str, ranges: &[Range], definitions: Option<&BTreeSet>, cursor: &mut QueryCursor, captures: &mut Vec, ) -> Result<(), String> { let names = configuration.query.capture_names(); for range in ranges { if range.start_byte == range.end_byte { continue; } cursor.set_byte_range(range.start_byte..range.end_byte); let mut matches = cursor.matches( &configuration.highlights_query, tree.root_node(), source.as_bytes(), ); while let Some(found) = matches.next() { for capture in found.captures() { let name = names[capture.index as usize]; if name.starts_with('_') || capture.node.end_byte() <= range.start_byte || capture.node.start_byte() >= range.end_byte || definitions .is_some_and(|definitions| !definitions.contains(&capture.node.id())) { continue; } captures.push(Capture::new(capture, found.pattern_index, 0, name)); } } if cursor.did_exceed_match_limit() { return Err(format!( "{} query exceeded its match limit", configuration.syntax.name )); } } Ok(()) } fn capture_priority(query: &Query, capture: &Capture) -> i32 { query .property_settings(capture.pattern) .iter() .rev() .find_map(|property| { if property.key.as_ref() == "priority" && property .capture_id .is_none_or(|id| id == capture.capture as usize) { property.value.as_deref()?.parse().ok() } else { None } }) .unwrap_or(0) } fn injection_ranges(nodes: &[Node<'_>], parents: &[Range], include_children: bool) -> Vec { let mut result = Vec::new(); for node in nodes { let mut start_byte = node.start_byte(); let mut start_point = node.start_position(); let mut cursor = node.walk(); let excluded: Vec<_> = if include_children { Vec::new() } else { // The bundled Markdown queries are Neovim-derived: anonymous punctuation is // injected source, while named children (e.g. block continuations) stay in the host. node.named_children(&mut cursor) .map(|child| child.range()) .collect() }; for excluded in excluded.into_iter().chain(std::iter::once(Range { start_byte: node.end_byte(), end_byte: node.end_byte(), start_point: node.end_position(), end_point: node.end_position(), })) { let parent_start = parents.partition_point(|range| range.end_byte <= start_byte); for parent in parents[parent_start..] .iter() .take_while(|range| range.start_byte < excluded.start_byte) { let low = start_byte.max(parent.start_byte); let high = excluded.start_byte.min(parent.end_byte); if low < high { result.push(Range { start_byte: low, end_byte: high, start_point: if low == start_byte { start_point } else { parent.start_point }, end_point: if high == excluded.start_byte { excluded.start_point } else { parent.end_point }, }); } } start_byte = excluded.end_byte; start_point = excluded.end_point; } } normalize_ranges(&mut result); result } fn normalize_ranges(ranges: &mut Vec) { ranges.sort_unstable_by_key(|range| (range.start_byte, range.end_byte)); let mut length = 0; for index in 0..ranges.len() { let range = ranges[index]; if length > 0 && ranges[length - 1].end_byte >= range.start_byte { let previous = &mut ranges[length - 1]; if previous.end_byte < range.end_byte { previous.end_byte = range.end_byte; previous.end_point = range.end_point; } } else { ranges[length] = range; length += 1; } } ranges.truncate(length); } fn render_spans( source: &str, starts: &[usize], lines: &mut [Vec], strings: &mut [Vec], highlights: &[Highlight], events: &mut Vec<(usize, bool, usize)>, visible: &[Range], ) { events.clear(); for (index, highlight) in highlights.iter().enumerate() { events.push((highlight.start, true, index)); events.push((highlight.end, false, index)); } events.sort_unstable(); let rank = |index: usize| { let highlight = &highlights[index]; ( highlight.depth, highlight.priority, Reverse(highlight.end - highlight.start), highlight.pattern, index, ) }; let mut active: BTreeSet<(_, usize)> = BTreeSet::new(); let mut previous = 0; let mut active_strings = 0; for &(byte, entering, index) in events.iter() { if previous < byte { let kind = active .last() .and_then(|(_, current)| highlights[*current].kind); if kind.is_some() || active_strings > 0 { let start = visible.partition_point(|range| range.end_byte <= previous); for range in visible[start..] .iter() .take_while(|range| range.start_byte < byte) { let low = previous.max(range.start_byte); let high = byte.min(range.end_byte); if let Some(kind) = kind { append_global(source, starts, lines, low, high, kind, false); } if active_strings > 0 { append_global(source, starts, strings, low, high, SyntaxKind::String, true); } } } } if highlights[index].kind == Some(SyntaxKind::String) { if entering { active_strings += 1; } else { active_strings -= 1; } } if entering { active.insert((rank(index), index)); } else { active.remove(&(rank(index), index)); } previous = byte; } } fn append_global( source: &str, starts: &[usize], lines: &mut [Vec], mut start: usize, end: usize, kind: SyntaxKind, include_newline: bool, ) { if starts.is_empty() { return; } let mut row = starts .partition_point(|offset| *offset <= start) .saturating_sub(1); while start < end && row < lines.len() { let line_end = starts.get(row + 1).map_or(source.len(), |next| next - 1); let high = end.min(line_end + usize::from(include_newline && row + 1 < starts.len())); if start < high { let spans = &mut lines[row]; let low = start - starts[row]; let high = high - starts[row]; if let Some(previous) = spans.last_mut() && previous.end_byte == low && previous.kind == kind { previous.end_byte = high; } else { spans.push(SyntaxSpan { start_byte: low, end_byte: high, kind, }); } } row += 1; start = line_end + 1; } } #[cfg(test)] #[path = "../tests/unit/syntax.rs"] mod tests;