1575 lines
56 KiB
Rust
1575 lines
56 KiB
Rust
use std::cmp::Reverse;
|
||
use std::collections::{BTreeSet, HashMap, VecDeque};
|
||
use std::path::{Path, PathBuf};
|
||
use std::sync::LazyLock;
|
||
use tree_sitter::{
|
||
InputEdit, Node, Parser, Point, Query, QueryCursor, Range, StreamingIterator, Tree,
|
||
};
|
||
|
||
mod languages;
|
||
use languages::LanguageSpec;
|
||
|
||
pub(crate) fn is_markdown_path(path: &Path) -> bool {
|
||
languages::detect(Some(path), None).is_some_and(|syntax| syntax.name == "Markdown")
|
||
}
|
||
|
||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||
pub enum SyntaxKind {
|
||
Comment,
|
||
String,
|
||
Keyword,
|
||
Number,
|
||
Type,
|
||
Function,
|
||
Markup,
|
||
}
|
||
|
||
/// A syntax foreground range in the original line's bytes, excluding its newline.
|
||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||
pub struct SyntaxSpan {
|
||
pub start_byte: usize,
|
||
pub end_byte: usize,
|
||
pub kind: SyntaxKind,
|
||
}
|
||
|
||
/// Frontend-owned, per-buffer parser state. Revision and path identify unchanged input.
|
||
#[derive(Default)]
|
||
pub struct SyntaxCache {
|
||
revision: Option<u64>,
|
||
path: Option<PathBuf>,
|
||
syntax: Option<&'static LanguageSpec>,
|
||
source: String,
|
||
next_source: String,
|
||
line_starts: Vec<usize>,
|
||
lines: Vec<Vec<SyntaxSpan>>,
|
||
parser: Parser,
|
||
tree: Option<Tree>,
|
||
injection_parser: Parser,
|
||
injections: Vec<Layer>,
|
||
cursor: QueryCursor,
|
||
captures: Vec<Capture>,
|
||
highlights: Vec<Highlight>,
|
||
events: Vec<(usize, bool, usize)>,
|
||
error: Option<String>,
|
||
#[cfg(test)]
|
||
parses: usize,
|
||
#[cfg(test)]
|
||
incremental_parses: usize,
|
||
#[cfg(test)]
|
||
injection_parses: usize,
|
||
}
|
||
|
||
impl SyntaxCache {
|
||
pub fn new() -> Self {
|
||
Self::default()
|
||
}
|
||
|
||
pub(crate) fn update_text(&mut self, text: &str, path: &Path) {
|
||
if self.source == text && self.path.as_deref() == Some(path) {
|
||
return;
|
||
}
|
||
let lines: Vec<String> = text.split('\n').map(str::to_owned).collect();
|
||
self.update(
|
||
&lines,
|
||
Some(path),
|
||
self.revision.unwrap_or(0).wrapping_add(1),
|
||
);
|
||
}
|
||
|
||
/// `revision` must change whenever line content changes; path changes are detected separately.
|
||
pub fn update(&mut self, lines: &[String], path: Option<&Path>, revision: u64) {
|
||
let path_changed = self.path.as_deref() != path;
|
||
if self.revision == Some(revision) && !path_changed {
|
||
return;
|
||
}
|
||
let syntax = languages::detect(path, lines.first().map(String::as_str));
|
||
let language_changed = !same_language(self.syntax, syntax);
|
||
let revision_changed = self.revision != Some(revision);
|
||
self.revision = Some(revision);
|
||
if path_changed {
|
||
self.path = path.map(Path::to_path_buf);
|
||
}
|
||
self.syntax = syntax;
|
||
|
||
let mut edit = None;
|
||
if revision_changed {
|
||
self.next_source.clear();
|
||
let length =
|
||
lines.iter().map(String::len).sum::<usize>() + lines.len().saturating_sub(1);
|
||
self.next_source.reserve(length);
|
||
for (row, line) in lines.iter().enumerate() {
|
||
if row > 0 {
|
||
self.next_source.push('\n');
|
||
}
|
||
self.next_source.push_str(line);
|
||
}
|
||
if self.source != self.next_source {
|
||
edit = Some(minimal_edit(&self.source, &self.next_source));
|
||
std::mem::swap(&mut self.source, &mut self.next_source);
|
||
}
|
||
self.line_starts.clear();
|
||
let mut start = 0;
|
||
for line in lines {
|
||
self.line_starts.push(start);
|
||
start += line.len() + 1;
|
||
}
|
||
self.lines.resize_with(lines.len(), Vec::new);
|
||
}
|
||
if edit.is_none() && !language_changed && self.error.is_none() {
|
||
return;
|
||
}
|
||
for spans in &mut self.lines {
|
||
spans.clear();
|
||
}
|
||
self.error = None;
|
||
if language_changed {
|
||
self.tree = None;
|
||
self.injections.clear();
|
||
}
|
||
let Some(syntax) = syntax else {
|
||
self.tree = None;
|
||
self.injections.clear();
|
||
return;
|
||
};
|
||
if let Err(error) = self.highlight(syntax, edit.as_ref(), language_changed) {
|
||
self.error = Some(format!("{} syntax: {error}", syntax.name));
|
||
self.tree = None;
|
||
self.injections.clear();
|
||
for spans in &mut self.lines {
|
||
spans.clear();
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Sorted, non-overlapping, end-exclusive ranges. Unclassified text uses the normal foreground.
|
||
pub fn line(&self, row: usize) -> &[SyntaxSpan] {
|
||
self.lines.get(row).map_or(&[], Vec::as_slice)
|
||
}
|
||
|
||
/// Bundled grammar name, or `None` for unknown languages and plain text.
|
||
pub fn language(&self) -> Option<&str> {
|
||
self.syntax.map(|syntax| syntax.name)
|
||
}
|
||
|
||
/// Genuine grammar/query failures disable highlighting; ERROR nodes in user text do not.
|
||
pub fn error(&self) -> Option<&str> {
|
||
self.error.as_deref()
|
||
}
|
||
|
||
fn highlight(
|
||
&mut self,
|
||
syntax: &'static LanguageSpec,
|
||
edit: Option<&InputEdit>,
|
||
language_changed: bool,
|
||
) -> Result<(), String> {
|
||
let primary_configuration = configuration(syntax)?;
|
||
if language_changed || self.tree.is_none() {
|
||
self.parser
|
||
.set_language(&syntax.language)
|
||
.map_err(|error| error.to_string())?;
|
||
}
|
||
if let Some(edit) = edit
|
||
&& let Some(tree) = &mut self.tree
|
||
{
|
||
tree.edit(edit);
|
||
}
|
||
if edit.is_some() || self.tree.is_none() {
|
||
#[cfg(test)]
|
||
{
|
||
self.parses += 1;
|
||
self.incremental_parses += usize::from(self.tree.is_some());
|
||
}
|
||
self.tree = Some(
|
||
self.parser
|
||
.parse(self.source.as_bytes(), self.tree.as_ref())
|
||
.ok_or_else(|| "parser stopped without a tree".to_owned())?,
|
||
);
|
||
}
|
||
let full_range = Range {
|
||
start_byte: 0,
|
||
end_byte: self.source.len(),
|
||
start_point: Point::new(0, 0),
|
||
end_point: point_at(&self.source, self.source.len()),
|
||
};
|
||
self.highlights.clear();
|
||
// ponytail: query the whole incremental tree; restrict changed ranges if query time dominates.
|
||
let mut queue: VecDeque<_> = collect_layer(
|
||
primary_configuration,
|
||
self.tree.as_ref().unwrap(),
|
||
&self.source,
|
||
&[full_range],
|
||
0,
|
||
None,
|
||
&mut self.cursor,
|
||
&mut self.captures,
|
||
&mut self.highlights,
|
||
)?
|
||
.into();
|
||
let mut previous = std::mem::take(&mut self.injections);
|
||
for layer in &mut previous {
|
||
layer.dirty = edit.is_some_and(|edit| {
|
||
layer.ranges.iter().any(|range| {
|
||
edit.start_byte <= range.end_byte && edit.old_end_byte >= range.start_byte
|
||
})
|
||
});
|
||
if let Some(edit) = edit {
|
||
layer.tree.edit(edit);
|
||
layer.ranges = layer.tree.included_ranges();
|
||
}
|
||
}
|
||
while let Some(request) = queue.pop_front() {
|
||
// An identical language/range ancestor is a genuine injection cycle. Keep that host
|
||
// layer, but do not inject it into itself again (e.g. a grammar's injection.self).
|
||
if request.ancestors.iter().any(|(spec, ranges)| {
|
||
std::ptr::eq(*spec, request.syntax) && *ranges == request.ranges
|
||
}) {
|
||
continue;
|
||
}
|
||
if request.depth > 16 {
|
||
return Err("injection nesting exceeds 16 language layers".to_owned());
|
||
}
|
||
let configuration = configuration(request.syntax)?;
|
||
let previous_index = previous
|
||
.iter()
|
||
.position(|layer| {
|
||
std::ptr::eq(layer.syntax, request.syntax)
|
||
&& layer.depth == request.depth
|
||
&& layer.ranges == request.ranges
|
||
})
|
||
.or_else(|| {
|
||
previous.iter().position(|layer| {
|
||
std::ptr::eq(layer.syntax, request.syntax) && layer.depth == request.depth
|
||
})
|
||
});
|
||
let old = previous_index.map(|index| previous.swap_remove(index));
|
||
let unchanged = old
|
||
.as_ref()
|
||
.is_some_and(|layer| !layer.dirty && layer.ranges == request.ranges);
|
||
let tree = if unchanged {
|
||
old.unwrap().tree
|
||
} else {
|
||
self.injection_parser
|
||
.set_language(&request.syntax.language)
|
||
.map_err(|error| error.to_string())?;
|
||
self.injection_parser
|
||
.set_included_ranges(&request.ranges)
|
||
.map_err(|error| error.to_string())?;
|
||
#[cfg(test)]
|
||
{
|
||
self.injection_parses += 1;
|
||
}
|
||
self.injection_parser
|
||
.parse(
|
||
self.source.as_bytes(),
|
||
old.as_ref().map(|layer| &layer.tree),
|
||
)
|
||
.ok_or_else(|| {
|
||
format!(
|
||
"{} injection parser stopped without a tree",
|
||
request.syntax.name
|
||
)
|
||
})?
|
||
};
|
||
// Markdown's inline grammar refines the block grammar, so plain inline text keeps
|
||
// its enclosing heading/link foreground. Other languages replace host strings/markup.
|
||
let refines_host = languages::by_name("markdown_inline")
|
||
.is_some_and(|inline| std::ptr::eq(inline, request.syntax));
|
||
if !refines_host {
|
||
for range in &request.ranges {
|
||
self.highlights.push(Highlight {
|
||
start: range.start_byte,
|
||
end: range.end_byte,
|
||
kind: None,
|
||
depth: request.depth,
|
||
priority: i32::MIN,
|
||
pattern: 0,
|
||
});
|
||
}
|
||
}
|
||
let mut children = collect_layer(
|
||
configuration,
|
||
&tree,
|
||
&self.source,
|
||
&request.ranges,
|
||
request.depth,
|
||
Some(request.parent),
|
||
&mut self.cursor,
|
||
&mut self.captures,
|
||
&mut self.highlights,
|
||
)?;
|
||
let mut ancestors = request.ancestors;
|
||
ancestors.push((request.syntax, request.ranges.clone()));
|
||
for child in &mut children {
|
||
child.ancestors = ancestors.clone();
|
||
}
|
||
queue.extend(children);
|
||
self.injections.push(Layer {
|
||
syntax: request.syntax,
|
||
tree,
|
||
ranges: request.ranges,
|
||
depth: request.depth,
|
||
dirty: false,
|
||
});
|
||
}
|
||
render_spans(
|
||
&self.source,
|
||
&self.line_starts,
|
||
&mut self.lines,
|
||
&self.highlights,
|
||
&mut self.events,
|
||
);
|
||
Ok(())
|
||
}
|
||
}
|
||
|
||
fn same_language(left: Option<&LanguageSpec>, right: Option<&LanguageSpec>) -> bool {
|
||
match (left, right) {
|
||
(Some(left), Some(right)) => std::ptr::eq(left, right),
|
||
(None, None) => true,
|
||
_ => false,
|
||
}
|
||
}
|
||
|
||
/// One aggregate edit, including byte columns (not character/display columns).
|
||
fn minimal_edit(old: &str, new: &str) -> InputEdit {
|
||
let mut start = old
|
||
.bytes()
|
||
.zip(new.bytes())
|
||
.take_while(|(a, b)| a == b)
|
||
.count();
|
||
while !old.is_char_boundary(start) || !new.is_char_boundary(start) {
|
||
start -= 1;
|
||
}
|
||
let mut suffix = old.as_bytes()[start..]
|
||
.iter()
|
||
.rev()
|
||
.zip(new.as_bytes()[start..].iter().rev())
|
||
.take_while(|(a, b)| a == b)
|
||
.count();
|
||
while !old.is_char_boundary(old.len() - suffix) || !new.is_char_boundary(new.len() - suffix) {
|
||
suffix -= 1;
|
||
}
|
||
let old_end = old.len() - suffix;
|
||
let new_end = new.len() - suffix;
|
||
InputEdit {
|
||
start_byte: start,
|
||
old_end_byte: old_end,
|
||
new_end_byte: new_end,
|
||
start_position: point_at(old, start),
|
||
old_end_position: point_at(old, old_end),
|
||
new_end_position: point_at(new, new_end),
|
||
}
|
||
}
|
||
|
||
fn point_at(source: &str, byte: usize) -> Point {
|
||
let prefix = &source.as_bytes()[..byte];
|
||
Point::new(
|
||
prefix.iter().filter(|byte| **byte == b'\n').count(),
|
||
prefix
|
||
.iter()
|
||
.rposition(|byte| *byte == b'\n')
|
||
.map_or(byte, |last| byte - last - 1),
|
||
)
|
||
}
|
||
|
||
struct Configuration {
|
||
syntax: &'static LanguageSpec,
|
||
query: Query,
|
||
locals_start: usize,
|
||
highlights_start: usize,
|
||
}
|
||
|
||
fn configuration(syntax: &'static LanguageSpec) -> Result<&'static Configuration, String> {
|
||
static CONFIGURATIONS: LazyLock<Vec<(&'static LanguageSpec, Result<Configuration, String>)>> =
|
||
LazyLock::new(|| {
|
||
languages::all()
|
||
.map(|syntax| (syntax, compile_configuration(syntax)))
|
||
.collect()
|
||
});
|
||
let (_, result) = CONFIGURATIONS
|
||
.iter()
|
||
.find(|(spec, _)| std::ptr::eq(*spec, syntax))
|
||
.ok_or_else(|| format!("missing {} query configuration", syntax.name))?;
|
||
result.as_ref().map_err(Clone::clone)
|
||
}
|
||
|
||
fn compile_configuration(syntax: &'static LanguageSpec) -> Result<Configuration, String> {
|
||
let source = format!(
|
||
"{}\n{}\n{}",
|
||
syntax.injections, syntax.locals, syntax.highlights
|
||
);
|
||
let locals_offset = syntax.injections.len() + 1;
|
||
let highlights_offset = locals_offset + syntax.locals.len() + 1;
|
||
let query = Query::new(&syntax.language, &source).map_err(|error| error.to_string())?;
|
||
let mut locals_start = 0;
|
||
let mut highlights_start = 0;
|
||
for pattern in 0..query.pattern_count() {
|
||
let start = query.start_byte_for_pattern(pattern);
|
||
locals_start += usize::from(start < locals_offset);
|
||
highlights_start += usize::from(start < highlights_offset);
|
||
if let Some(predicate) = query.general_predicates(pattern).first() {
|
||
return Err(format!(
|
||
"unsupported query predicate #{}",
|
||
predicate.operator
|
||
));
|
||
}
|
||
if let Some((property, _)) = query
|
||
.property_predicates(pattern)
|
||
.iter()
|
||
.find(|(property, _)| property.key.as_ref() != "local")
|
||
{
|
||
return Err(format!(
|
||
"unsupported query property predicate {}",
|
||
property.key
|
||
));
|
||
}
|
||
}
|
||
Ok(Configuration {
|
||
syntax,
|
||
query,
|
||
locals_start,
|
||
highlights_start,
|
||
})
|
||
}
|
||
|
||
fn classify(capture: &str, numeric_literal: bool) -> Option<SyntaxKind> {
|
||
let mut parts = capture.split('.');
|
||
match parts.next()? {
|
||
"comment" => Some(SyntaxKind::Comment),
|
||
"string" | "character" => Some(SyntaxKind::String),
|
||
"keyword" | "boolean" | "storage" | "conditional" | "repeat" | "include" | "exception" => {
|
||
Some(SyntaxKind::Keyword)
|
||
}
|
||
"number" | "float" => Some(SyntaxKind::Number),
|
||
"constant" if numeric_literal || parts.any(|part| part == "numeric") => {
|
||
Some(SyntaxKind::Number)
|
||
}
|
||
"constant" => Some(SyntaxKind::Keyword),
|
||
"type" | "constructor" => Some(SyntaxKind::Type),
|
||
"function" | "method" => Some(SyntaxKind::Function),
|
||
"markup" | "tag" | "text" => Some(SyntaxKind::Markup),
|
||
_ => None,
|
||
}
|
||
}
|
||
|
||
struct Layer {
|
||
syntax: &'static LanguageSpec,
|
||
tree: Tree,
|
||
ranges: Vec<Range>,
|
||
depth: usize,
|
||
dirty: bool,
|
||
}
|
||
|
||
struct Injection {
|
||
syntax: &'static LanguageSpec,
|
||
parent: &'static LanguageSpec,
|
||
ranges: Vec<Range>,
|
||
depth: usize,
|
||
ancestors: Vec<(&'static LanguageSpec, Vec<Range>)>,
|
||
}
|
||
|
||
#[derive(Clone, Copy)]
|
||
struct Capture {
|
||
node: usize,
|
||
start: usize,
|
||
end: usize,
|
||
capture: u32,
|
||
pattern: usize,
|
||
value_end: usize,
|
||
numeric_literal: bool,
|
||
}
|
||
|
||
struct Highlight {
|
||
start: usize,
|
||
end: usize,
|
||
kind: Option<SyntaxKind>,
|
||
depth: usize,
|
||
priority: i32,
|
||
pattern: usize,
|
||
}
|
||
|
||
struct Definition {
|
||
value_end: usize,
|
||
highlight: Option<SyntaxKind>,
|
||
}
|
||
|
||
struct Scope<'a> {
|
||
end: usize,
|
||
inherits: bool,
|
||
definitions: HashMap<&'a str, Vec<Definition>>,
|
||
}
|
||
|
||
#[allow(clippy::too_many_arguments)]
|
||
fn collect_layer(
|
||
configuration: &Configuration,
|
||
tree: &Tree,
|
||
source: &str,
|
||
ranges: &[Range],
|
||
depth: usize,
|
||
parent: Option<&'static LanguageSpec>,
|
||
cursor: &mut QueryCursor,
|
||
captures: &mut Vec<Capture>,
|
||
highlights: &mut Vec<Highlight>,
|
||
) -> Result<Vec<Injection>, String> {
|
||
let query = &configuration.query;
|
||
let names = query.capture_names();
|
||
captures.clear();
|
||
let mut injections = Vec::new();
|
||
let mut combined: HashMap<(usize, usize), usize> = HashMap::new();
|
||
let mut matches = cursor.matches(query, tree.root_node(), source.as_bytes());
|
||
while let Some(found) = matches.next() {
|
||
let pattern = found.pattern_index;
|
||
if pattern < configuration.locals_start {
|
||
let mut language = None;
|
||
let mut nodes = Vec::new();
|
||
for capture in found.captures() {
|
||
match names[capture.index as usize] {
|
||
"injection.language" => {
|
||
language = source[capture.node.byte_range()]
|
||
.split_whitespace()
|
||
.next()
|
||
.and_then(languages::by_name);
|
||
}
|
||
"injection.content" => nodes.push(capture.node),
|
||
_ => {}
|
||
}
|
||
}
|
||
let mut include_children = false;
|
||
let mut is_combined = false;
|
||
for property in query.property_settings(pattern) {
|
||
match property.key.as_ref() {
|
||
"injection.language" => {
|
||
language = property.value.as_deref().and_then(languages::by_name);
|
||
}
|
||
"injection.self" => language = Some(configuration.syntax),
|
||
"injection.parent" => language = parent,
|
||
"injection.include-children" => include_children = true,
|
||
"injection.combined" => is_combined = true,
|
||
_ => {}
|
||
}
|
||
}
|
||
if let Some(language) = language {
|
||
let mut content = injection_ranges(&nodes, ranges, include_children);
|
||
if !content.is_empty() {
|
||
let key = (pattern, language as *const LanguageSpec as usize);
|
||
if is_combined && let Some(index) = combined.get(&key) {
|
||
let injection: &mut Injection = &mut injections[*index];
|
||
injection.ranges.append(&mut content);
|
||
} else {
|
||
if is_combined {
|
||
combined.insert(key, injections.len());
|
||
}
|
||
injections.push(Injection {
|
||
syntax: language,
|
||
parent: configuration.syntax,
|
||
ranges: content,
|
||
depth: depth + 1,
|
||
ancestors: vec![(configuration.syntax, ranges.to_vec())],
|
||
});
|
||
}
|
||
}
|
||
}
|
||
continue;
|
||
}
|
||
let value_end = found
|
||
.captures()
|
||
.iter()
|
||
.find(|capture| names[capture.index as usize] == "local.definition-value")
|
||
.map_or(0, |capture| capture.node.end_byte());
|
||
for capture in found.captures() {
|
||
let name = names[capture.index as usize];
|
||
if name.starts_with('_') || name == "local.definition-value" {
|
||
continue;
|
||
}
|
||
captures.push(Capture {
|
||
node: capture.node.id(),
|
||
start: capture.node.start_byte(),
|
||
end: capture.node.end_byte(),
|
||
capture: capture.index,
|
||
pattern,
|
||
value_end,
|
||
numeric_literal: name.starts_with("constant")
|
||
&& matches!(
|
||
capture.node.kind(),
|
||
"integer_literal"
|
||
| "float_literal"
|
||
| "number_literal"
|
||
| "integer"
|
||
| "float"
|
||
| "number"
|
||
),
|
||
});
|
||
}
|
||
}
|
||
if cursor.did_exceed_match_limit() {
|
||
return Err(format!(
|
||
"{} query exceeded its match limit",
|
||
configuration.syntax.name
|
||
));
|
||
}
|
||
for injection in &mut injections {
|
||
normalize_ranges(&mut injection.ranges);
|
||
}
|
||
captures.sort_unstable_by_key(|capture| {
|
||
(
|
||
capture.start,
|
||
Reverse(capture.end),
|
||
capture.node,
|
||
capture.pattern,
|
||
capture.capture,
|
||
)
|
||
});
|
||
let mut scopes = vec![Scope {
|
||
end: usize::MAX,
|
||
inherits: false,
|
||
definitions: HashMap::new(),
|
||
}];
|
||
let mut start = 0;
|
||
while start < captures.len() {
|
||
let first = captures[start];
|
||
let mut end = start + 1;
|
||
while end < captures.len() && captures[end].node == first.node {
|
||
end += 1;
|
||
}
|
||
let group = &captures[start..end];
|
||
while scopes.len() > 1 && first.start >= scopes.last().unwrap().end {
|
||
scopes.pop();
|
||
}
|
||
let mut definition = None;
|
||
let mut reference = None;
|
||
let mut is_local = false;
|
||
for capture in group
|
||
.iter()
|
||
.filter(|capture| capture.pattern < configuration.highlights_start)
|
||
{
|
||
match names[capture.capture as usize] {
|
||
"local.scope" => {
|
||
let inherits = query
|
||
.property_settings(capture.pattern)
|
||
.iter()
|
||
.find(|property| property.key.as_ref() == "local.scope-inherits")
|
||
.is_none_or(|property| {
|
||
property
|
||
.value
|
||
.as_deref()
|
||
.is_none_or(|value| value == "true")
|
||
});
|
||
scopes.push(Scope {
|
||
end: capture.end,
|
||
inherits,
|
||
definitions: HashMap::new(),
|
||
});
|
||
}
|
||
"local.definition" => {
|
||
is_local = true;
|
||
definition = Some(capture.value_end);
|
||
}
|
||
"local.reference" if definition.is_none() => {
|
||
let name = &source[capture.start..capture.end];
|
||
for scope in scopes.iter().rev() {
|
||
if let Some(found) = scope.definitions.get(name).and_then(|definitions| {
|
||
definitions
|
||
.iter()
|
||
.rev()
|
||
.find(|definition| capture.start >= definition.value_end)
|
||
}) {
|
||
is_local = true;
|
||
reference = Some(found.highlight);
|
||
break;
|
||
}
|
||
if !scope.inherits {
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
_ => {}
|
||
}
|
||
}
|
||
let selected = group
|
||
.iter()
|
||
.filter(|capture| {
|
||
capture.pattern >= configuration.highlights_start
|
||
&& query.property_predicates(capture.pattern).iter().all(
|
||
|(property, positive)| {
|
||
let local = property
|
||
.capture_id
|
||
.map_or(is_local, |id| id == capture.capture as usize && is_local);
|
||
local == *positive
|
||
},
|
||
)
|
||
})
|
||
.max_by_key(|capture| {
|
||
(
|
||
capture_priority(query, capture),
|
||
capture.pattern,
|
||
capture.capture,
|
||
)
|
||
});
|
||
let kind = reference.unwrap_or_else(|| {
|
||
selected.and_then(|capture| {
|
||
classify(names[capture.capture as usize], capture.numeric_literal)
|
||
})
|
||
});
|
||
if let Some(value_end) = definition {
|
||
let name = &source[first.start..first.end];
|
||
scopes
|
||
.last_mut()
|
||
.unwrap()
|
||
.definitions
|
||
.entry(name)
|
||
.or_default()
|
||
.push(Definition {
|
||
value_end,
|
||
highlight: kind,
|
||
});
|
||
}
|
||
if let Some(kind) = kind {
|
||
let selected = selected.copied().unwrap_or(first);
|
||
let range_start = ranges.partition_point(|range| range.end_byte <= first.start);
|
||
for range in ranges[range_start..]
|
||
.iter()
|
||
.take_while(|range| range.start_byte < first.end)
|
||
{
|
||
let low = first.start.max(range.start_byte);
|
||
let high = first.end.min(range.end_byte);
|
||
if low < high {
|
||
highlights.push(Highlight {
|
||
start: low,
|
||
end: high,
|
||
kind: Some(kind),
|
||
depth,
|
||
priority: capture_priority(query, &selected),
|
||
pattern: selected.pattern,
|
||
});
|
||
}
|
||
}
|
||
}
|
||
start = end;
|
||
}
|
||
Ok(injections)
|
||
}
|
||
|
||
fn capture_priority(query: &Query, capture: &Capture) -> i32 {
|
||
query
|
||
.property_settings(capture.pattern)
|
||
.iter()
|
||
.rev()
|
||
.find_map(|property| {
|
||
if property.key.as_ref() == "priority"
|
||
&& property
|
||
.capture_id
|
||
.is_none_or(|id| id == capture.capture as usize)
|
||
{
|
||
property.value.as_deref()?.parse().ok()
|
||
} else {
|
||
None
|
||
}
|
||
})
|
||
.unwrap_or(0)
|
||
}
|
||
|
||
fn injection_ranges(nodes: &[Node<'_>], parents: &[Range], include_children: bool) -> Vec<Range> {
|
||
let mut result = Vec::new();
|
||
for node in nodes {
|
||
let mut start_byte = node.start_byte();
|
||
let mut start_point = node.start_position();
|
||
let mut cursor = node.walk();
|
||
let excluded: Vec<_> = if include_children {
|
||
Vec::new()
|
||
} else {
|
||
// The bundled Markdown queries are Neovim-derived: anonymous punctuation is
|
||
// injected source, while named children (e.g. block continuations) stay in the host.
|
||
node.named_children(&mut cursor)
|
||
.map(|child| child.range())
|
||
.collect()
|
||
};
|
||
for excluded in excluded.into_iter().chain(std::iter::once(Range {
|
||
start_byte: node.end_byte(),
|
||
end_byte: node.end_byte(),
|
||
start_point: node.end_position(),
|
||
end_point: node.end_position(),
|
||
})) {
|
||
let parent_start = parents.partition_point(|range| range.end_byte <= start_byte);
|
||
for parent in parents[parent_start..]
|
||
.iter()
|
||
.take_while(|range| range.start_byte < excluded.start_byte)
|
||
{
|
||
let low = start_byte.max(parent.start_byte);
|
||
let high = excluded.start_byte.min(parent.end_byte);
|
||
if low < high {
|
||
result.push(Range {
|
||
start_byte: low,
|
||
end_byte: high,
|
||
start_point: if low == start_byte {
|
||
start_point
|
||
} else {
|
||
parent.start_point
|
||
},
|
||
end_point: if high == excluded.start_byte {
|
||
excluded.start_point
|
||
} else {
|
||
parent.end_point
|
||
},
|
||
});
|
||
}
|
||
}
|
||
start_byte = excluded.end_byte;
|
||
start_point = excluded.end_point;
|
||
}
|
||
}
|
||
normalize_ranges(&mut result);
|
||
result
|
||
}
|
||
|
||
fn normalize_ranges(ranges: &mut Vec<Range>) {
|
||
ranges.sort_unstable_by_key(|range| (range.start_byte, range.end_byte));
|
||
let mut length = 0;
|
||
for index in 0..ranges.len() {
|
||
let range = ranges[index];
|
||
if length > 0 && ranges[length - 1].end_byte >= range.start_byte {
|
||
let previous = &mut ranges[length - 1];
|
||
if previous.end_byte < range.end_byte {
|
||
previous.end_byte = range.end_byte;
|
||
previous.end_point = range.end_point;
|
||
}
|
||
} else {
|
||
ranges[length] = range;
|
||
length += 1;
|
||
}
|
||
}
|
||
ranges.truncate(length);
|
||
}
|
||
|
||
fn render_spans(
|
||
source: &str,
|
||
starts: &[usize],
|
||
lines: &mut [Vec<SyntaxSpan>],
|
||
highlights: &[Highlight],
|
||
events: &mut Vec<(usize, bool, usize)>,
|
||
) {
|
||
events.clear();
|
||
for (index, highlight) in highlights.iter().enumerate() {
|
||
events.push((highlight.start, true, index));
|
||
events.push((highlight.end, false, index));
|
||
}
|
||
events.sort_unstable();
|
||
let rank = |index: usize| {
|
||
let highlight = &highlights[index];
|
||
(
|
||
highlight.depth,
|
||
highlight.priority,
|
||
Reverse(highlight.end - highlight.start),
|
||
highlight.pattern,
|
||
index,
|
||
)
|
||
};
|
||
let mut active: BTreeSet<(_, usize)> = BTreeSet::new();
|
||
let mut previous = 0;
|
||
for &(byte, entering, index) in events.iter() {
|
||
if previous < byte
|
||
&& let Some((_, current)) = active.last()
|
||
&& let Some(kind) = highlights[*current].kind
|
||
{
|
||
append_global(source, starts, lines, previous, byte, kind);
|
||
}
|
||
if entering {
|
||
active.insert((rank(index), index));
|
||
} else {
|
||
active.remove(&(rank(index), index));
|
||
}
|
||
previous = byte;
|
||
}
|
||
}
|
||
|
||
fn append_global(
|
||
source: &str,
|
||
starts: &[usize],
|
||
lines: &mut [Vec<SyntaxSpan>],
|
||
mut start: usize,
|
||
end: usize,
|
||
kind: SyntaxKind,
|
||
) {
|
||
if starts.is_empty() {
|
||
return;
|
||
}
|
||
let mut row = starts
|
||
.partition_point(|offset| *offset <= start)
|
||
.saturating_sub(1);
|
||
while start < end && row < lines.len() {
|
||
let line_end = starts.get(row + 1).map_or(source.len(), |next| next - 1);
|
||
let high = end.min(line_end);
|
||
if start < high {
|
||
let spans = &mut lines[row];
|
||
let low = start - starts[row];
|
||
let high = high - starts[row];
|
||
if let Some(previous) = spans.last_mut()
|
||
&& previous.end_byte == low
|
||
&& previous.kind == kind
|
||
{
|
||
previous.end_byte = high;
|
||
} else {
|
||
spans.push(SyntaxSpan {
|
||
start_byte: low,
|
||
end_byte: high,
|
||
kind,
|
||
});
|
||
}
|
||
}
|
||
row += 1;
|
||
start = line_end + 1;
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
fn lines(source: &str) -> Vec<String> {
|
||
source.split('\n').map(str::to_owned).collect()
|
||
}
|
||
|
||
fn cache(source: &str, path: &str) -> SyntaxCache {
|
||
let mut cache = SyntaxCache::new();
|
||
cache.update(&lines(source), Some(Path::new(path)), 0);
|
||
assert_eq!(cache.error(), None, "{path}");
|
||
cache
|
||
}
|
||
|
||
fn kind_at(cache: &SyntaxCache, row: usize, byte: usize) -> Option<SyntaxKind> {
|
||
cache
|
||
.line(row)
|
||
.iter()
|
||
.find_map(|span| (span.start_byte <= byte && byte < span.end_byte).then_some(span.kind))
|
||
}
|
||
|
||
fn assert_token(cache: &SyntaxCache, source: &str, row: usize, token: &str, kind: SyntaxKind) {
|
||
let line = source.split('\n').nth(row).unwrap();
|
||
let start = line.find(token).unwrap();
|
||
for (byte, _) in line[start..start + token.len()].char_indices() {
|
||
assert_eq!(
|
||
kind_at(cache, row, start + byte),
|
||
Some(kind),
|
||
"{token:?} in {}",
|
||
cache.language().unwrap()
|
||
);
|
||
}
|
||
}
|
||
|
||
fn assert_fresh(cache: &SyntaxCache, source: &str, path: &str) {
|
||
let fresh = self::cache(source, path);
|
||
assert_eq!(cache.language(), fresh.language());
|
||
assert_eq!(cache.error(), fresh.error());
|
||
assert_eq!(
|
||
cache.tree.as_ref().map(|tree| tree.root_node().to_sexp()),
|
||
fresh.tree.as_ref().map(|tree| tree.root_node().to_sexp())
|
||
);
|
||
let source = lines(source);
|
||
for (row, line) in source.iter().enumerate() {
|
||
assert_eq!(cache.line(row), fresh.line(row), "row {row}");
|
||
let mut previous = 0;
|
||
for span in cache.line(row) {
|
||
assert!(previous <= span.start_byte && span.start_byte < span.end_byte);
|
||
assert!(span.end_byte <= line.len());
|
||
assert!(
|
||
line.is_char_boundary(span.start_byte) && line.is_char_boundary(span.end_byte)
|
||
);
|
||
previous = span.end_byte;
|
||
}
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn every_bundled_query_compiles_and_common_languages_color_real_tokens() {
|
||
for language in languages::all() {
|
||
configuration(language).unwrap_or_else(|error| panic!("{}: {error}", language.name));
|
||
}
|
||
let cases = [
|
||
(
|
||
"a.rs",
|
||
"Rust",
|
||
"fn main() { let x = 42; let s = \"hi\"; /* note */ }",
|
||
"fn",
|
||
SyntaxKind::Keyword,
|
||
),
|
||
(
|
||
"a.rb",
|
||
"Ruby",
|
||
"def hello\n 42\nend",
|
||
"def",
|
||
SyntaxKind::Keyword,
|
||
),
|
||
(
|
||
"a.py",
|
||
"Python",
|
||
"def hello():\n return 42",
|
||
"def",
|
||
SyntaxKind::Keyword,
|
||
),
|
||
(
|
||
"a.json",
|
||
"JSON",
|
||
"{\"n\":42,\"yes\":true}",
|
||
"42",
|
||
SyntaxKind::Number,
|
||
),
|
||
(
|
||
"a.sh",
|
||
"Bash",
|
||
"echo \"hi\" # note",
|
||
"hi",
|
||
SyntaxKind::String,
|
||
),
|
||
(
|
||
"Makefile",
|
||
"Makefile",
|
||
"# note\nall:\n\techo hi",
|
||
"note",
|
||
SyntaxKind::Comment,
|
||
),
|
||
(
|
||
"a.md",
|
||
"Markdown",
|
||
"# Heading\n\n**bold**",
|
||
"Heading",
|
||
SyntaxKind::Markup,
|
||
),
|
||
(
|
||
"a.js",
|
||
"JavaScript",
|
||
"function hello() { return 42; }",
|
||
"function",
|
||
SyntaxKind::Keyword,
|
||
),
|
||
(
|
||
"a.ts",
|
||
"TypeScript",
|
||
"function hello(): number { return 42; }",
|
||
"function",
|
||
SyntaxKind::Keyword,
|
||
),
|
||
(
|
||
"a.html",
|
||
"HTML",
|
||
"<div title=\"hi\">text</div>",
|
||
"div",
|
||
SyntaxKind::Markup,
|
||
),
|
||
(
|
||
"a.css",
|
||
"CSS",
|
||
"body { width: 42px; }",
|
||
"42",
|
||
SyntaxKind::Number,
|
||
),
|
||
(
|
||
"a.yaml",
|
||
"YAML",
|
||
"message: \"hi\"\ncount: 42",
|
||
"hi",
|
||
SyntaxKind::String,
|
||
),
|
||
(
|
||
"Cargo.toml",
|
||
"TOML",
|
||
"message = \"hi\"\ncount = 42",
|
||
"hi",
|
||
SyntaxKind::String,
|
||
),
|
||
(
|
||
"a.c",
|
||
"C",
|
||
"int main(void) { return 42; }",
|
||
"return",
|
||
SyntaxKind::Keyword,
|
||
),
|
||
(
|
||
"a.cpp",
|
||
"C++",
|
||
"int main() { return 42; }",
|
||
"return",
|
||
SyntaxKind::Keyword,
|
||
),
|
||
(
|
||
"a.go",
|
||
"Go",
|
||
"package main\nfunc main() { return }",
|
||
"package",
|
||
SyntaxKind::Keyword,
|
||
),
|
||
];
|
||
for (path, language, source, token, kind) in cases {
|
||
let cache = self::cache(source, path);
|
||
assert_eq!(cache.language(), Some(language));
|
||
assert_token(&cache, source, 0, token, kind);
|
||
assert_fresh(&cache, source, path);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn semantic_colors_and_text_predicates_preserve_the_ui_contract() {
|
||
let source =
|
||
"fn main() { let n = 42; let f = 3.25; let b = true; let s = \"hello\"; /* note */ }";
|
||
let cache = self::cache(source, "a.rs");
|
||
for (token, kind) in [
|
||
("fn", SyntaxKind::Keyword),
|
||
("let", SyntaxKind::Keyword),
|
||
("main", SyntaxKind::Function),
|
||
("42", SyntaxKind::Number),
|
||
("3.25", SyntaxKind::Number),
|
||
("true", SyntaxKind::Keyword),
|
||
("hello", SyntaxKind::String),
|
||
("note", SyntaxKind::Comment),
|
||
] {
|
||
assert_token(&cache, source, 0, token, kind);
|
||
}
|
||
let source = "const foo = 42; const UPPER = 7; foo; UPPER;";
|
||
let cache = self::cache(source, "a.js");
|
||
assert_eq!(kind_at(&cache, 0, source.find("foo").unwrap()), None);
|
||
// JavaScript's bundled constant predicate must not color every identifier.
|
||
assert_ne!(
|
||
kind_at(&cache, 0, source.find("foo;").unwrap()),
|
||
Some(SyntaxKind::Keyword)
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn filenames_shebang_save_as_and_tsx_use_actual_grammar_identity() {
|
||
let source = lines("\u{feff}#!/usr/bin/env python3\nreturn 42");
|
||
let mut cache = SyntaxCache::new();
|
||
cache.update(&source, None, 0);
|
||
assert_eq!(cache.language(), Some("Python"));
|
||
cache.update(&source, Some(Path::new("a.RS")), 0);
|
||
assert_eq!(cache.language(), Some("Rust"));
|
||
assert_eq!(cache.parses, 2);
|
||
cache.update(&source, Some(Path::new("renamed.rs")), 0);
|
||
assert_eq!(cache.parses, 2);
|
||
cache.update(&lines("all:\n\techo hi"), Some(Path::new("GNUmakefile")), 1);
|
||
assert_eq!(cache.language(), Some("Makefile"));
|
||
let source = "const element = <div title=\"hi\" />;";
|
||
cache.update(&lines(source), Some(Path::new("a.ts")), 2);
|
||
let typescript = cache.syntax.unwrap();
|
||
let parses = cache.parses;
|
||
cache.update(&lines(source), Some(Path::new("a.tsx")), 2);
|
||
assert_eq!(cache.language(), Some("TypeScript"));
|
||
assert!(!std::ptr::eq(typescript, cache.syntax.unwrap()));
|
||
assert_eq!(cache.parses, parses + 1);
|
||
assert_token(&cache, source, 0, "div", SyntaxKind::Markup);
|
||
assert_fresh(&cache, source, "a.tsx");
|
||
let mut script = SyntaxCache::new();
|
||
script.update(&lines("#!/usr/bin/env python3\nreturn 42"), None, 0);
|
||
assert_eq!(script.language(), Some("Python"));
|
||
script.update(&lines("#!/usr/bin/env bash\necho \"hi\""), None, 1);
|
||
assert_eq!(script.language(), Some("Bash"));
|
||
assert_token(
|
||
&script,
|
||
"#!/usr/bin/env bash\necho \"hi\"",
|
||
1,
|
||
"hi",
|
||
SyntaxKind::String,
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn unknown_and_former_non_common_languages_are_honest_plain_text() {
|
||
for path in [
|
||
"a.unknown",
|
||
"a.txt",
|
||
"a.hs",
|
||
"a.java",
|
||
"a.php",
|
||
"a.swift",
|
||
"a.sql",
|
||
] {
|
||
let cache = self::cache("fn main() { \"hi\"; 42; }", path);
|
||
assert_eq!(cache.language(), None, "{path}");
|
||
assert!(cache.lines.iter().all(Vec::is_empty));
|
||
assert_eq!(cache.parses, 0);
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn unchanged_revision_and_same_source_do_not_allocate_or_parse() {
|
||
let mut cache = self::cache("fn main() {}\n", "a.rs");
|
||
let source = lines("fn main() {}\n");
|
||
let source_pointer = cache.source.as_ptr();
|
||
let scratch_pointer = cache.next_source.as_ptr();
|
||
let tree_id = cache.tree.as_ref().unwrap().root_node().id();
|
||
cache.update(&source, Some(Path::new("a.rs")), 0);
|
||
assert_eq!(cache.source.as_ptr(), source_pointer);
|
||
assert_eq!(cache.next_source.as_ptr(), scratch_pointer);
|
||
assert_eq!(cache.tree.as_ref().unwrap().root_node().id(), tree_id);
|
||
cache.update(&source, Some(Path::new("a.rs")), 1);
|
||
cache.update(&source, Some(Path::new("b.rs")), 1);
|
||
assert_eq!(cache.parses, 1);
|
||
assert_eq!(cache.incremental_parses, 0);
|
||
assert_eq!(cache.tree.as_ref().unwrap().root_node().id(), tree_id);
|
||
assert!(cache.line(1).is_empty());
|
||
cache.update(&source, Some(Path::new("b.txt")), 1);
|
||
assert_eq!(cache.language(), None);
|
||
assert!(cache.tree.is_none());
|
||
assert!(cache.lines.iter().all(Vec::is_empty));
|
||
cache.update(&source, Some(Path::new("b.rs")), 1);
|
||
assert_eq!(cache.parses, 2);
|
||
assert_eq!(cache.incremental_parses, 0);
|
||
}
|
||
|
||
#[test]
|
||
fn unicode_minimal_edits_use_byte_points_and_never_split_codepoints() {
|
||
let old = "x\n\t\"é 👩💻\"\n";
|
||
let new = "x\n\t\"ê 👩💻\"\n";
|
||
let edit = minimal_edit(old, new);
|
||
assert_eq!(edit.start_byte, 4);
|
||
assert_eq!(edit.start_position, Point::new(1, 2));
|
||
assert_eq!(edit.old_end_byte, 6);
|
||
assert_eq!(edit.new_end_byte, 6);
|
||
assert_eq!(edit.old_end_position, Point::new(1, 4));
|
||
let edit = minimal_edit("a\nβ\n", "a\n\nβ\n");
|
||
assert_eq!(edit.start_position, Point::new(1, 0));
|
||
assert_eq!(edit.old_end_position, Point::new(1, 0));
|
||
assert_eq!(edit.new_end_position, Point::new(2, 0));
|
||
let edit = minimal_edit("α", "");
|
||
assert_eq!(
|
||
(edit.start_byte, edit.old_end_byte, edit.new_end_byte),
|
||
(0, 2, 0)
|
||
);
|
||
let edit = minimal_edit("xα", "xѱ");
|
||
assert_eq!(
|
||
(edit.start_byte, edit.old_end_byte, edit.new_end_byte),
|
||
(1, 3, 3)
|
||
);
|
||
assert_eq!(edit.new_end_position, Point::new(0, 3));
|
||
}
|
||
|
||
#[test]
|
||
fn incremental_unicode_newline_insert_remove_and_undo_match_fresh_trees() {
|
||
let original =
|
||
"fn main() {\n\tlet text = \"é é 👩💻\";\n\t/* nested /* inner */ note */\n}\n";
|
||
let mut cache = self::cache(original, "a.rs");
|
||
let edits = [
|
||
original.replace("é", "ê"),
|
||
original.replace("let text", "\nlet text"),
|
||
original.replace("\n\t/* nested /* inner */ note */", ""),
|
||
original.replace("👩💻", "👩🚀"),
|
||
original.replace("\";", "\";\n\tlet n = 42;"),
|
||
original.to_owned(),
|
||
String::new(),
|
||
original.to_owned(),
|
||
];
|
||
for (revision, source) in edits.iter().enumerate() {
|
||
cache.update(&lines(source), Some(Path::new("a.rs")), revision as u64 + 1);
|
||
assert_fresh(&cache, source, "a.rs");
|
||
}
|
||
assert_eq!(cache.parses, edits.len() + 1);
|
||
assert_eq!(cache.incremental_parses, edits.len());
|
||
}
|
||
|
||
#[test]
|
||
fn unfinished_source_and_multiline_offscreen_state_remain_highlighted() {
|
||
let source =
|
||
"fn main() {\n/* start\ninside 👩💻\nend */\nlet n = 42;\nlet text = \"unfinished";
|
||
let cache = self::cache(source, "a.rs");
|
||
assert!(cache.tree.as_ref().unwrap().root_node().has_error());
|
||
assert_token(&cache, source, 0, "fn", SyntaxKind::Keyword);
|
||
assert_token(&cache, source, 2, "inside", SyntaxKind::Comment);
|
||
assert_token(&cache, source, 4, "42", SyntaxKind::Number);
|
||
assert_eq!(cache.error(), None);
|
||
let source = "text = '''first\ninside\nlast'''\nanswer = 42";
|
||
let cache = self::cache(source, "a.py");
|
||
assert_token(&cache, source, 1, "inside", SyntaxKind::String);
|
||
assert_token(&cache, source, 3, "42", SyntaxKind::Number);
|
||
}
|
||
|
||
#[test]
|
||
fn markdown_heading_markers_share_title_colors_without_coloring_plain_hashes() {
|
||
let source = "# One\n## Two\n### Three\n#### Four\n##### Five\n###### Six\n\nSetext\n======\n\nplain # text";
|
||
let cache = self::cache(source, "headings.md");
|
||
for row in 0..6 {
|
||
assert_token(
|
||
&cache,
|
||
source,
|
||
row,
|
||
&"#".repeat(row + 1),
|
||
SyntaxKind::Markup,
|
||
);
|
||
}
|
||
assert_token(&cache, source, 0, "One", SyntaxKind::Markup);
|
||
assert_token(&cache, source, 7, "Setext", SyntaxKind::Markup);
|
||
assert_token(&cache, source, 8, "======", SyntaxKind::Markup);
|
||
assert_eq!(kind_at(&cache, 10, 6), None);
|
||
}
|
||
|
||
#[test]
|
||
fn markdown_inline_and_fences_inject_known_languages_and_update_incrementally() {
|
||
let original = "A **bold** and `code` link.\n\n```rust\nfn main() { let n = 42; }\n```\n\n```python\ndef hello():\n return 7\n```\n\n```not-a-language\nfn not_rust() { 99; }\n```";
|
||
let mut cache = self::cache(original, "a.md");
|
||
assert_token(&cache, original, 0, "bold", SyntaxKind::Markup);
|
||
assert_token(&cache, original, 0, "code", SyntaxKind::Markup);
|
||
assert_token(&cache, original, 3, "fn", SyntaxKind::Keyword);
|
||
assert_token(&cache, original, 3, "main", SyntaxKind::Function);
|
||
assert_token(&cache, original, 3, "42", SyntaxKind::Number);
|
||
assert_token(&cache, original, 7, "def", SyntaxKind::Keyword);
|
||
assert_ne!(kind_at(&cache, 12, 0), Some(SyntaxKind::Keyword));
|
||
let parses = cache.injection_parses;
|
||
let source = original.replace("42", "123");
|
||
cache.update(&lines(&source), Some(Path::new("a.md")), 1);
|
||
assert_token(&cache, &source, 3, "123", SyntaxKind::Number);
|
||
assert_fresh(&cache, &source, "a.md");
|
||
assert!(cache.injection_parses > parses);
|
||
let source = source.replace("```rust", "```unknown");
|
||
cache.update(&lines(&source), Some(Path::new("a.md")), 2);
|
||
assert_ne!(kind_at(&cache, 3, 0), Some(SyntaxKind::Keyword));
|
||
assert_fresh(&cache, &source, "a.md");
|
||
}
|
||
|
||
#[test]
|
||
fn markdown_injections_keep_anonymous_syntax_but_not_named_block_continuations() {
|
||
let source =
|
||
"> # Heading **bold**\n>\n> ```rust\n> fn main() {\n> let n = 3.25;\n> }\n> ```";
|
||
let cache = self::cache(source, "a.md");
|
||
assert_token(&cache, source, 0, "Heading", SyntaxKind::Markup);
|
||
assert_token(&cache, source, 0, "bold", SyntaxKind::Markup);
|
||
assert_token(&cache, source, 3, "main", SyntaxKind::Function);
|
||
assert_token(&cache, source, 4, "3.25", SyntaxKind::Number);
|
||
let rust = languages::by_name("rust").unwrap();
|
||
let layer = cache
|
||
.injections
|
||
.iter()
|
||
.find(|layer| std::ptr::eq(layer.syntax, rust))
|
||
.unwrap();
|
||
for range in &layer.ranges {
|
||
assert!(!source[range.start_byte..range.end_byte].contains('>'));
|
||
}
|
||
let braces = [source.find('{').unwrap(), source.find('}').unwrap()];
|
||
for byte in braces {
|
||
assert!(
|
||
layer
|
||
.ranges
|
||
.iter()
|
||
.any(|range| range.start_byte <= byte && byte < range.end_byte)
|
||
);
|
||
}
|
||
assert_fresh(&cache, source, "a.md");
|
||
}
|
||
|
||
#[test]
|
||
fn html_script_style_injections_override_host_and_reuse_unchanged_trees() {
|
||
let original = "<div>hello</div>\n<script>function hello() { return 42; }</script>\n<style>body { color: red; width: 7px; }</style>";
|
||
let mut cache = self::cache(original, "a.html");
|
||
assert_token(&cache, original, 0, "div", SyntaxKind::Markup);
|
||
assert_token(&cache, original, 1, "function", SyntaxKind::Keyword);
|
||
assert_token(&cache, original, 1, "hello", SyntaxKind::Function);
|
||
assert_token(&cache, original, 1, "42", SyntaxKind::Number);
|
||
assert_token(&cache, original, 2, "7", SyntaxKind::Number);
|
||
let injection_parses = cache.injection_parses;
|
||
let source = original.replace("<div>hello", "<div>world");
|
||
cache.update(&lines(&source), Some(Path::new("a.html")), 1);
|
||
assert_eq!(cache.injection_parses, injection_parses);
|
||
assert_fresh(&cache, &source, "a.html");
|
||
let source = source.replace("42", "123");
|
||
cache.update(&lines(&source), Some(Path::new("a.html")), 2);
|
||
assert_token(&cache, &source, 1, "123", SyntaxKind::Number);
|
||
assert_fresh(&cache, &source, "a.html");
|
||
// Inserting at an included range's exact end must reparse, not merely edit its old tree.
|
||
let source = source.replace("return 123; }", "return 123; } function added() {}");
|
||
cache.update(&lines(&source), Some(Path::new("a.html")), 3);
|
||
assert_token(&cache, &source, 1, "added", SyntaxKind::Function);
|
||
assert_fresh(&cache, &source, "a.html");
|
||
}
|
||
|
||
#[test]
|
||
fn local_definitions_disable_builtin_captures_and_propagate_highlights() {
|
||
let source = "function hello(require) { require('local'); }\nrequire('global');";
|
||
let cache = self::cache(source, "a.js");
|
||
let local = source.find("require('local')").unwrap();
|
||
assert_eq!(kind_at(&cache, 0, local), None);
|
||
assert_token(&cache, source, 1, "require", SyntaxKind::Function);
|
||
let source = "const action = () => 42;\naction;";
|
||
let cache = self::cache(source, "a.js");
|
||
assert_token(&cache, source, 0, "action", SyntaxKind::Function);
|
||
assert_token(&cache, source, 1, "action", SyntaxKind::Function);
|
||
}
|
||
|
||
#[test]
|
||
fn overlapping_capture_specificity_priority_and_plain_injection_are_nonoverlapping() {
|
||
let source = "αβγ";
|
||
let mut spans = vec![Vec::new()];
|
||
let mut events = Vec::new();
|
||
let highlights = vec![
|
||
Highlight {
|
||
start: 0,
|
||
end: 6,
|
||
kind: Some(SyntaxKind::String),
|
||
depth: 0,
|
||
priority: 0,
|
||
pattern: 99,
|
||
},
|
||
Highlight {
|
||
start: 2,
|
||
end: 4,
|
||
kind: Some(SyntaxKind::Function),
|
||
depth: 0,
|
||
priority: 0,
|
||
pattern: 1,
|
||
},
|
||
Highlight {
|
||
start: 4,
|
||
end: 6,
|
||
kind: None,
|
||
depth: 1,
|
||
priority: i32::MIN,
|
||
pattern: 0,
|
||
},
|
||
Highlight {
|
||
start: 0,
|
||
end: 2,
|
||
kind: Some(SyntaxKind::Keyword),
|
||
depth: 0,
|
||
priority: 10,
|
||
pattern: 0,
|
||
},
|
||
];
|
||
render_spans(source, &[0], &mut spans, &highlights, &mut events);
|
||
assert_eq!(
|
||
spans[0],
|
||
vec![
|
||
SyntaxSpan {
|
||
start_byte: 0,
|
||
end_byte: 2,
|
||
kind: SyntaxKind::Keyword
|
||
},
|
||
SyntaxSpan {
|
||
start_byte: 2,
|
||
end_byte: 4,
|
||
kind: SyntaxKind::Function
|
||
},
|
||
]
|
||
);
|
||
}
|
||
|
||
#[test]
|
||
fn local_initializer_visibility_positive_predicates_and_noninheriting_scopes() {
|
||
let syntax = languages::by_name("javascript").unwrap();
|
||
let locals = r#"
|
||
(program) @local.scope
|
||
((function_declaration) @local.scope (#set! local.scope-inherits false))
|
||
(variable_declarator name: (identifier) @local.definition value: (_) @local.definition-value)
|
||
(identifier) @local.reference
|
||
"#;
|
||
let source_query = format!(
|
||
"{locals}\n{}",
|
||
r#"
|
||
((identifier) @function (#is-not? local))
|
||
((identifier) @string (#is? local))
|
||
(variable_declarator name: (identifier) @type)
|
||
"#
|
||
);
|
||
let query = Query::new(&syntax.language, &source_query).unwrap();
|
||
let highlights_start = (0..query.pattern_count())
|
||
.filter(|pattern| query.start_byte_for_pattern(*pattern) < locals.len() + 1)
|
||
.count();
|
||
let configuration = Configuration {
|
||
syntax,
|
||
query,
|
||
locals_start: 0,
|
||
highlights_start,
|
||
};
|
||
let source = "const value = value;\nvalue;\nfunction inner() { value; }";
|
||
let cache = self::cache(source, "a.js");
|
||
let mut highlights = Vec::new();
|
||
collect_layer(
|
||
&configuration,
|
||
cache.tree.as_ref().unwrap(),
|
||
source,
|
||
&[Range {
|
||
start_byte: 0,
|
||
end_byte: source.len(),
|
||
start_point: Point::new(0, 0),
|
||
end_point: point_at(source, source.len()),
|
||
}],
|
||
0,
|
||
None,
|
||
&mut QueryCursor::new(),
|
||
&mut Vec::new(),
|
||
&mut highlights,
|
||
)
|
||
.unwrap();
|
||
let mut spans = vec![Vec::new(); 3];
|
||
let starts = [
|
||
0,
|
||
source.find("\nvalue;").unwrap() + 1,
|
||
source.find("\nfunction").unwrap() + 1,
|
||
];
|
||
render_spans(source, &starts, &mut spans, &highlights, &mut Vec::new());
|
||
let kind = |row: usize, byte: usize| {
|
||
spans[row].iter().find_map(|span| {
|
||
(span.start_byte <= byte && byte < span.end_byte).then_some(span.kind)
|
||
})
|
||
};
|
||
assert_eq!(kind(0, 6), Some(SyntaxKind::Type));
|
||
assert_eq!(kind(0, 14), Some(SyntaxKind::Function));
|
||
assert_eq!(kind(1, 0), Some(SyntaxKind::Type));
|
||
assert_eq!(kind(2, 19), Some(SyntaxKind::Function));
|
||
}
|
||
|
||
#[test]
|
||
fn grammar_and_unsupported_predicate_errors_are_not_silent_empty_queries() {
|
||
static INVALID: LazyLock<[LanguageSpec; 2]> = LazyLock::new(|| {
|
||
let language = languages::by_name("rust").unwrap();
|
||
[
|
||
"(not_a_rust_node) @keyword",
|
||
"((identifier) @function (#unsupported? @function))",
|
||
]
|
||
.map(|query| LanguageSpec {
|
||
name: "invalid test grammar query",
|
||
language: language.language.clone(),
|
||
highlights: query,
|
||
injections: "",
|
||
locals: "",
|
||
})
|
||
});
|
||
for syntax in INVALID.iter() {
|
||
assert!(compile_configuration(syntax).is_err());
|
||
}
|
||
}
|
||
|
||
#[test]
|
||
fn combined_tagged_templates_and_nested_markdown_html_keep_real_injections() {
|
||
let source = "const page = html`<div>${value}</div><script>function inside() { return 42; }</script>`;";
|
||
let cache = self::cache(source, "a.js");
|
||
assert_token(&cache, source, 0, "div", SyntaxKind::Markup);
|
||
assert_token(&cache, source, 0, "inside", SyntaxKind::Function);
|
||
assert_token(&cache, source, 0, "42", SyntaxKind::Number);
|
||
assert_fresh(&cache, source, "a.js");
|
||
let source = "```html\n<script>function nested() { return 7; }</script>\n```";
|
||
let cache = self::cache(source, "a.md");
|
||
assert_token(&cache, source, 1, "nested", SyntaxKind::Function);
|
||
assert_token(&cache, source, 1, "7", SyntaxKind::Number);
|
||
assert_fresh(&cache, source, "a.md");
|
||
let source = "fn main() { println!(\"{}\", 42); }";
|
||
let cache = self::cache(source, "a.rs");
|
||
assert_token(&cache, source, 0, "42", SyntaxKind::Number);
|
||
assert!(cache.injections.len() <= 16);
|
||
assert_fresh(&cache, source, "a.rs");
|
||
}
|
||
|
||
#[test]
|
||
fn parser_state_is_isolated_between_buffers() {
|
||
let mut first = self::cache("/* open\ninside", "a.rs");
|
||
let second = self::cache("fn main() { let n = 42; }", "b.rs");
|
||
first.update(
|
||
&lines("/* open */\nfn other() {}"),
|
||
Some(Path::new("a.rs")),
|
||
1,
|
||
);
|
||
assert_token(
|
||
&second,
|
||
"fn main() { let n = 42; }",
|
||
0,
|
||
"fn",
|
||
SyntaxKind::Keyword,
|
||
);
|
||
assert_eq!(second.parses, 1);
|
||
assert_eq!(first.incremental_parses, 1);
|
||
}
|
||
}
|