From e5d545a4ca2b52fd00bace3ff6d78e587e6667a0 Mon Sep 17 00:00:00 2001 From: Valdemar Lorenzen Date: Fri, 9 Oct 2026 11:15:17 +0200 Subject: [PATCH] refactor: migrate syntax highlighting to Tree-sitter --- Cargo.lock | 355 ++++--- Cargo.toml | 18 +- README.md | 8 +- src/syntax.rs | 1948 ++++++++++++++++++++++++++++----------- src/syntax/languages.rs | 422 +++++++++ 5 files changed, 2079 insertions(+), 672 deletions(-) create mode 100644 src/syntax/languages.rs diff --git a/Cargo.lock b/Cargo.lock index bb711d7..f0e7d05 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,12 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "adler2" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" - [[package]] name = "aho-corasick" version = "1.1.5" @@ -38,30 +32,6 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - -[[package]] -name = "bit-set" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" - [[package]] name = "bitflags" version = "2.13.2" @@ -93,6 +63,16 @@ dependencies = [ "rustversion", ] +[[package]] +name = "cc" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" +dependencies = [ + "find-msvc-tools", + "shlex", +] + [[package]] name = "cfg-if" version = "1.0.5" @@ -122,15 +102,6 @@ dependencies = [ "unicode-segmentation", ] -[[package]] -name = "crc32fast" -version = "1.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" -dependencies = [ - "cfg-if", -] - [[package]] name = "crossbeam-deque" version = "0.8.8" @@ -277,32 +248,10 @@ dependencies = [ ] [[package]] -name = "fancy-regex" -version = "0.16.2" +name = "find-msvc-tools" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "998b056554fbe42e03ae0e152895cd1a7e1002aec800fdc6635d20270260c46f" -dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "flate2" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" -dependencies = [ - "crc32fast", - "miniz_oxide", - "zlib-rs", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "foldhash" @@ -373,6 +322,16 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + [[package]] name = "indoc" version = "2.0.7" @@ -484,16 +443,6 @@ version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" -[[package]] -name = "miniz_oxide" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" -dependencies = [ - "adler2", - "simd-adler32", -] - [[package]] name = "mio" version = "1.2.4" @@ -530,12 +479,6 @@ dependencies = [ "libc", ] -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - [[package]] name = "palette" version = "0.7.7" @@ -697,6 +640,18 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.18" @@ -805,6 +760,7 @@ version = "1.0.151" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" dependencies = [ + "indexmap", "itoa", "memchr", "serde", @@ -812,6 +768,12 @@ dependencies = [ "zmij", ] +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + [[package]] name = "signal-hook" version = "0.3.18" @@ -843,12 +805,6 @@ dependencies = [ "libc", ] -[[package]] -name = "simd-adler32" -version = "0.3.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" - [[package]] name = "smallvec" version = "1.16.2" @@ -861,6 +817,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + [[package]] name = "strsim" version = "0.11.1" @@ -910,24 +872,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "syntect" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "656b45c05d95a5704399aeef6bd0ddec7b2b3531b7c9e900abbf7c4d2190c925" -dependencies = [ - "bincode", - "fancy-regex", - "flate2", - "fnv", - "once_cell", - "regex-syntax", - "serde", - "serde_derive", - "thiserror", - "walkdir", -] - [[package]] name = "thiserror" version = "2.0.21" @@ -969,6 +913,185 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" +[[package]] +name = "tree-sitter" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5129de2ccc97808c4583d48f1fb44576d015cf241e432800572bda11ef30c3b" +dependencies = [ + "cc", + "regex", + "serde_json", + "streaming-iterator", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-bash" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5ec769279cc91b561d3df0d8a5deb26b0ad40d183127f409494d6d8fc53062" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-c" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9b2eb57a55fed6b00812912e730b7a275cf4fe98bfd6a5d76263d4438371728" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-cpp" +version = "0.23.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df2196ea9d47b4ab4a31b9297eaa5a5d19a0b121dceb9f118f6790ad0ab94743" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-css" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5cbc5e18f29a2c6d6435891f42569525cf95435a3e01c2f1947abcde178686f" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-go" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8560a4d2f835cc0d4d2c2e03cbd0dde2f6114b43bc491164238d333e28b16ea" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-html" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "261b708e5d92061ede329babaaa427b819329a9d427a1d710abb0f67bbef63ee" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-javascript" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68204f2abc0627a90bdf06e605f5c470aa26fdcb2081ea553a04bdad756693f5" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-json" +version = "0.24.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d727acca406c0020cffc6cf35516764f36c8e3dc4408e5ebe2cb35a947ec471" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-language" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0af592be68c579aa78a16846bd19422978c3c52e438523d45ff5d1bff1f9d4a" + +[[package]] +name = "tree-sitter-make" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5998dc7cbcbdab19fae8aefef982bf2d6544513d8d2e69cc44aec4c63810104" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-md" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2efd398be546456c814598ee56c0f51769a77241511b4a58077815d120afa882" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-python" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bf85fd39652e740bf60f46f4cda9492c3a9ad75880575bf14960f775cb74a1c" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-ruby" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be0484ea4ef6bb9c575b4fdabde7e31340a8d2dbc7d52b321ac83da703249f95" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-rust" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439e577dbe07423ec2582ac62c7531120dbfccfa6e5f92406f93dd271a120e45" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-toml-ng" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9adc2c898ae49730e857d75be403da3f92bb81d8e37a2f918a08dd10de5ebb1" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-typescript" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5f76ed8d947a75cc446d5fccd8b602ebf0cde64ccf2ffa434d873d7a575eff" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-yaml" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c223db85f05e34794f065454843b0668ebc15d240ada63e2b5939f43ce7c97" +dependencies = [ + "cc", + "tree-sitter-language", +] + [[package]] name = "unicode-ident" version = "1.0.26" @@ -1007,7 +1130,23 @@ dependencies = [ "ratatui", "serde", "serde_json", - "syntect", + "tree-sitter", + "tree-sitter-bash", + "tree-sitter-c", + "tree-sitter-cpp", + "tree-sitter-css", + "tree-sitter-go", + "tree-sitter-html", + "tree-sitter-javascript", + "tree-sitter-json", + "tree-sitter-make", + "tree-sitter-md", + "tree-sitter-python", + "tree-sitter-ruby", + "tree-sitter-rust", + "tree-sitter-toml-ng", + "tree-sitter-typescript", + "tree-sitter-yaml", "unicode-segmentation", "unicode-width", ] @@ -1074,12 +1213,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "zlib-rs" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" - [[package]] name = "zmij" version = "1.0.23" diff --git a/Cargo.toml b/Cargo.toml index 7d6c4bf..2d6e447 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,22 @@ ignore = "0.4" ratatui = { version = "0.30", default-features = false, features = ["crossterm_0_29"] } serde = { version = "1", features = ["derive"] } serde_json = "1" -syntect = { version = "5.3", default-features = false, features = ["default-syntaxes", "regex-fancy"] } +tree-sitter = "0.27.1" +tree-sitter-bash = "0.25.1" +tree-sitter-c = "0.24.2" +tree-sitter-cpp = "0.23.4" +tree-sitter-css = "0.25.0" +tree-sitter-go = "0.25.0" +tree-sitter-html = "0.23.2" +tree-sitter-javascript = "0.25.0" +tree-sitter-json = "0.24.8" +tree-sitter-make = "1.1.1" +tree-sitter-md = { version = "0.5.3", default-features = false } +tree-sitter-python = "0.25.0" +tree-sitter-ruby = "0.23.1" +tree-sitter-rust = "0.24.2" +tree-sitter-toml-ng = "0.7.0" +tree-sitter-typescript = "0.23.2" +tree-sitter-yaml = "0.7.2" unicode-segmentation = "1.12" unicode-width = "0.2" diff --git a/README.md b/README.md index f86e6e8..a49d8c0 100644 --- a/README.md +++ b/README.md @@ -205,7 +205,7 @@ One crate, seven modules, no workspace or terminal-coordinate types in the editi - `editor`: owns buffer lifecycle and per-buffer text, saved state, primary/selection set, modes, search, and undo/redo; session-wide clipboard policy, independent registers and owned-system fragment metadata, semantic native repeat, action metadata, command-palette and file-picker focus/dispatch, key notation, leader/count prefixes and binding resolution, modal paste policy, saving, command ownership, capabilities, and validated edit effects. Its low-level interface returns an `Invocation` when an external command is selected. - `files`: owns the gitignore-aware directory/file catalog, authoritative OS paths, compact tree projection, query filtering, browse/search focus, expansion/selection restoration, navigation, and scan/error state shared by floating and docked presentations. - `clipboard`: owns native plain-text provider selection and bounded subprocess I/O; it never computes selections or editing offsets. -- `syntax`: owns bundled grammar detection, semantic byte spans, multiline parser checkpoints, and changed-suffix highlighting; it has no terminal color/geometry or editing dependency. +- `syntax`: owns Tree-sitter grammar detection, per-buffer incremental trees, bundled highlight/local/injection queries, and semantic byte spans; it has no terminal color/geometry or editing dependency. - `session`: `EditorSession` owns the editor and its Ruby hosts together. It loads configuration once, registers eligible plugins before installing shared bindings, executes selected commands against the active buffer, and retires both host and command ownership on invocation or effect-validation failure. - `tui`: translates terminal events to session input, renders read-only active-buffer state, keeps frontend-owned viewports and syntax caches by stable buffer ID, and advertises `text` and `styled_text` capabilities. It maps semantic tokens to terminal colors and has no plugin lifecycle callback, host storage, or paste acceptance policy. - `ruby`: evaluates configuration and runs isolated plugin hosts. Protocol, deadlines, and process cleanup stay inside this module. @@ -228,11 +228,11 @@ The buffer viewport includes a fixed left gutter with right-aligned hybrid line ### Syntax highlighting -Buffers highlight automatically using [Syntect's bundled Sublime grammars](https://docs.rs/syntect/5.3.0/syntect/). Detection uses the supplied filename/extension and the buffer's own first line/shebang, without rereading the file from disk. Tested languages include Rust, Ruby, Python, JSON, shell, Makefile, and Markdown; other bundled grammars are available. Unknown and Plain Text buffers keep the terminal's normal foreground. The detected language appears in status, and saving under a different extension redetects it. +Buffers highlight automatically using [Tree-sitter](https://tree-sitter.github.io/tree-sitter/) and compiled-in grammar/query packages. The selected language set is Rust, Ruby, Python, JSON, Bash, Makefile, Markdown, JavaScript/JSX, TypeScript/TSX, HTML, CSS, YAML, TOML, C, C++, and Go. Detection uses filename/extension or the in-memory first-line shebang, including common special filenames such as Makefile, Gemfile, and Cargo.lock; no file reread occurs. Unsupported formats use the normal foreground. This migration intentionally replaces the broader former grammar bundle: languages outside this explicit set, such as Lua/Haskell/Scala/Perl, are plain text until added. The detected language appears in status, and save-as redetects it. -Colors use the terminal palette rather than a fixed background/theme: comments are dark gray, strings green, keywords magenta, numbers yellow, types/markup cyan, and functions light blue. Multiline/nested comments and strings retain parser scope across lines, including when their opening is above the viewport. Highlighting styles whole graphemes and expanded tab cells; partially clipped wide characters keep the existing blank-placeholder behavior. Search, Visual selections, and secondary carets paint afterward and retain their foreground/background precedence. +Colors retain the terminal palette: comments dark gray, strings green, keywords magenta, numbers yellow, types/markup cyan, and functions light blue. Grammar queries handle multiline/nested constructs, even when their opening is above the viewport. Markdown uses its inline grammar and named fenced-code injections; HTML script/style content uses JavaScript/CSS. Known tagged-template and raw-string injections use the same selected grammar registry; unknown injected languages retain host highlighting or plain text. Highlighting styles whole graphemes and expanded tab cells; partially clipped wide placeholders remain unchanged. Search, Visual selections, and secondary carets retain foreground/background precedence. -Each buffer has a frontend-owned cache. Text revisions distinguish edits/plugin effects/undo/redo from cursor and explorer changes; unchanged revision/path skips comparison and parsing. Changed text resumes at the first changed line's cached parser/scope state and reparses the suffix, preserving prefix work. Buffer switching restores its cache, and closing retires it. Real parser/scope errors clear syntax spans, retain readable/editable plain text, and report a status error. No custom lexer, runtime grammar downloads, or editor-offset changes are involved. +Each buffer has a frontend-owned parser/tree cache. Unchanged revision/path skips comparison and parsing; unchanged text at a later revision also avoids parsing. Actual text changes produce a minimal UTF-8-safe byte edit with newline-aware Tree-sitter Points, edit the prior tree, then parse using that tree for incremental reuse. Grammar changes reset the tree, including TypeScript versus TSX despite their shared display name. Native highlight/local/injection queries resolve capture precedence and scoped definitions into nonoverlapping original-line spans; unchanged injected trees are reused. Buffer switching restores its cache, and closing retires it. Incomplete text and grammar ERROR nodes remain highlightable; genuine language/query/parser failures expose a status error and readable plain text. Full-tree highlight queries still run after incremental parsingβ€”a deliberate ceiling to revisit if measured costly. Syntect, its scope checkpoints, and its dependencies are removed; there is no second baseline engine, runtime grammar download, or LSP scaffold. ### Core feature registry diff --git a/src/syntax.rs b/src/syntax.rs index 8e8741e..a065cd6 100644 --- a/src/syntax.rs +++ b/src/syntax.rs @@ -1,10 +1,14 @@ +use std::cmp::Reverse; +use std::collections::{BTreeSet, HashMap, VecDeque}; use std::path::{Path, PathBuf}; use std::sync::LazyLock; -use syntect::highlighting::ScopeSelectors; -use syntect::parsing::{ - ParseState, ScopeError, ScopeStack, ScopeStackOp, SyntaxReference, SyntaxSet, +use tree_sitter::{ + InputEdit, Node, Parser, Point, Query, QueryCursor, Range, StreamingIterator, Tree, }; +mod languages; +use languages::LanguageSpec; + #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum SyntaxKind { Comment, @@ -24,24 +28,31 @@ pub struct SyntaxSpan { pub kind: SyntaxKind, } -#[derive(Default)] -struct CachedLine { - source: String, - spans: Vec, - after: Option<(ParseState, ScopeStack)>, -} - /// Frontend-owned, per-buffer parser state. Revision and path identify unchanged input. #[derive(Default)] pub struct SyntaxCache { revision: Option, path: Option, - syntax: Option<&'static SyntaxReference>, - lines: Vec, - parse_line: String, + syntax: Option<&'static LanguageSpec>, + source: String, + next_source: String, + line_starts: Vec, + lines: Vec>, + parser: Parser, + tree: Option, + injection_parser: Parser, + injections: Vec, + cursor: QueryCursor, + captures: Vec, + highlights: Vec, + events: Vec<(usize, bool, usize)>, error: Option, #[cfg(test)] - parsed_lines: usize, + parses: usize, + #[cfg(test)] + incremental_parses: usize, + #[cfg(test)] + injection_parses: usize, } impl SyntaxCache { @@ -55,243 +66,859 @@ impl SyntaxCache { if self.revision == Some(revision) && !path_changed { return; } - - let prefix = self - .lines - .iter() - .zip(lines) - .take_while(|(cached, source)| cached.source == **source) - .count(); - let source_changed = prefix != self.lines.len() || prefix != lines.len(); - let syntaxes = syntax_set(); - let syntax = if self.revision.is_none() || path_changed || (source_changed && prefix == 0) { - detect_syntax(syntaxes, path, lines.first().map(String::as_str)) - } else { - self.syntax - }; - let syntax_changed = self.language() != syntax.map(|syntax| syntax.name.as_str()); - + let syntax = languages::detect(path, lines.first().map(String::as_str)); + let language_changed = !same_language(self.syntax, syntax); + let revision_changed = self.revision != Some(revision); self.revision = Some(revision); if path_changed { self.path = path.map(Path::to_path_buf); } self.syntax = syntax; - if !source_changed && !syntax_changed { - return; - } - self.lines.truncate(lines.len()); - for (row, source) in lines.iter().enumerate().skip(prefix) { - if let Some(cached) = self.lines.get_mut(row) { - if cached.source != *source { - cached.source.clone_from(source); + let mut edit = None; + if revision_changed { + self.next_source.clear(); + let length = + lines.iter().map(String::len).sum::() + lines.len().saturating_sub(1); + self.next_source.reserve(length); + for (row, line) in lines.iter().enumerate() { + if row > 0 { + self.next_source.push('\n'); } - } else { - self.lines.push(CachedLine { - source: source.clone(), - ..CachedLine::default() - }); + self.next_source.push_str(line); } + if self.source != self.next_source { + edit = Some(minimal_edit(&self.source, &self.next_source)); + std::mem::swap(&mut self.source, &mut self.next_source); + } + self.line_starts.clear(); + let mut start = 0; + for line in lines { + self.line_starts.push(start); + start += line.len() + 1; + } + self.lines.resize_with(lines.len(), Vec::new); + } + if edit.is_none() && !language_changed && self.error.is_none() { + return; + } + for spans in &mut self.lines { + spans.clear(); + } + self.error = None; + if language_changed { + self.tree = None; + self.injections.clear(); } - let Some(syntax) = syntax else { - self.clear_highlighting(); - self.error = None; + self.tree = None; + self.injections.clear(); return; }; - let start = if syntax_changed || self.error.is_some() { - 0 - } else { - prefix - }; - self.error = None; - if start < self.lines.len() { - self.parse_from(start, syntax, syntaxes); + if let Err(error) = self.highlight(syntax, edit.as_ref(), language_changed) { + self.error = Some(format!("{} syntax: {error}", syntax.name)); + self.tree = None; + self.injections.clear(); + for spans in &mut self.lines { + spans.clear(); + } } } /// Sorted, non-overlapping, end-exclusive ranges. Unclassified text uses the normal foreground. pub fn line(&self, row: usize) -> &[SyntaxSpan] { - self.lines.get(row).map_or(&[], |line| &line.spans) + self.lines.get(row).map_or(&[], Vec::as_slice) } - /// Bundled grammar name, or `None` for unknown languages and Plain Text. + /// Bundled grammar name, or `None` for unknown languages and plain text. pub fn language(&self) -> Option<&str> { - self.syntax.map(|syntax| syntax.name.as_str()) + self.syntax.map(|syntax| syntax.name) } - /// A genuine parser/scope error disables highlighting for the whole buffer until input changes. + /// Genuine grammar/query failures disable highlighting; ERROR nodes in user text do not. pub fn error(&self) -> Option<&str> { self.error.as_deref() } - fn clear_highlighting(&mut self) { - for line in &mut self.lines { - line.spans.clear(); - line.after = None; + fn highlight( + &mut self, + syntax: &'static LanguageSpec, + edit: Option<&InputEdit>, + language_changed: bool, + ) -> Result<(), String> { + let primary_configuration = configuration(syntax)?; + if language_changed || self.tree.is_none() { + self.parser + .set_language(&syntax.language) + .map_err(|error| error.to_string())?; } - } - - fn parse_from(&mut self, start: usize, syntax: &SyntaxReference, syntaxes: &SyntaxSet) { - let (mut parser, mut scopes) = if start == 0 { - (ParseState::new(syntax), ScopeStack::new()) - } else { - self.lines[start - 1] - .after - .as_ref() - .expect("unchanged prefix has a parser checkpoint") - .clone() - }; - - // ponytail: parse the dirty suffix to EOF; stop at converged checkpoints if this becomes costly. - for row in start..self.lines.len() { - let cached = &mut self.lines[row]; - self.parse_line.clear(); - self.parse_line.push_str(&cached.source); - self.parse_line.push('\n'); + if let Some(edit) = edit + && let Some(tree) = &mut self.tree + { + tree.edit(edit); + } + if edit.is_some() || self.tree.is_none() { #[cfg(test)] { - self.parsed_lines += 1; + self.parses += 1; + self.incremental_parses += usize::from(self.tree.is_some()); } - let result = parser - .parse_line(&self.parse_line, syntaxes) - .map_err(|error| error.to_string()) - .and_then(|changes| { - collect_spans( - cached.source.len(), - &changes, - &mut scopes, - &mut cached.spans, - ) - .map_err(|error| error.to_string()) - }); - if let Err(error) = result { - self.error = Some(format!("{} syntax, line {}: {error}", syntax.name, row + 1)); - self.clear_highlighting(); - return; - } - cached.after = Some((parser.clone(), scopes.clone())); + self.tree = Some( + self.parser + .parse(self.source.as_bytes(), self.tree.as_ref()) + .ok_or_else(|| "parser stopped without a tree".to_owned())?, + ); } + let full_range = Range { + start_byte: 0, + end_byte: self.source.len(), + start_point: Point::new(0, 0), + end_point: point_at(&self.source, self.source.len()), + }; + self.highlights.clear(); + // ponytail: query the whole incremental tree; restrict changed ranges if query time dominates. + let mut queue: VecDeque<_> = collect_layer( + primary_configuration, + self.tree.as_ref().unwrap(), + &self.source, + &[full_range], + 0, + None, + &mut self.cursor, + &mut self.captures, + &mut self.highlights, + )? + .into(); + let mut previous = std::mem::take(&mut self.injections); + for layer in &mut previous { + layer.dirty = edit.is_some_and(|edit| { + layer.ranges.iter().any(|range| { + edit.start_byte <= range.end_byte && edit.old_end_byte >= range.start_byte + }) + }); + if let Some(edit) = edit { + layer.tree.edit(edit); + layer.ranges = layer.tree.included_ranges(); + } + } + while let Some(request) = queue.pop_front() { + // An identical language/range ancestor is a genuine injection cycle. Keep that host + // layer, but do not inject it into itself again (e.g. a grammar's injection.self). + if request.ancestors.iter().any(|(spec, ranges)| { + std::ptr::eq(*spec, request.syntax) && *ranges == request.ranges + }) { + continue; + } + if request.depth > 16 { + return Err("injection nesting exceeds 16 language layers".to_owned()); + } + let configuration = configuration(request.syntax)?; + let previous_index = previous + .iter() + .position(|layer| { + std::ptr::eq(layer.syntax, request.syntax) + && layer.depth == request.depth + && layer.ranges == request.ranges + }) + .or_else(|| { + previous.iter().position(|layer| { + std::ptr::eq(layer.syntax, request.syntax) && layer.depth == request.depth + }) + }); + let old = previous_index.map(|index| previous.swap_remove(index)); + let unchanged = old + .as_ref() + .is_some_and(|layer| !layer.dirty && layer.ranges == request.ranges); + let tree = if unchanged { + old.unwrap().tree + } else { + self.injection_parser + .set_language(&request.syntax.language) + .map_err(|error| error.to_string())?; + self.injection_parser + .set_included_ranges(&request.ranges) + .map_err(|error| error.to_string())?; + #[cfg(test)] + { + self.injection_parses += 1; + } + self.injection_parser + .parse( + self.source.as_bytes(), + old.as_ref().map(|layer| &layer.tree), + ) + .ok_or_else(|| { + format!( + "{} injection parser stopped without a tree", + request.syntax.name + ) + })? + }; + // Markdown's inline grammar refines the block grammar, so plain inline text keeps + // its enclosing heading/link foreground. Other languages replace host strings/markup. + let refines_host = languages::by_name("markdown_inline") + .is_some_and(|inline| std::ptr::eq(inline, request.syntax)); + if !refines_host { + for range in &request.ranges { + self.highlights.push(Highlight { + start: range.start_byte, + end: range.end_byte, + kind: None, + depth: request.depth, + priority: i32::MIN, + pattern: 0, + }); + } + } + let mut children = collect_layer( + configuration, + &tree, + &self.source, + &request.ranges, + request.depth, + Some(request.parent), + &mut self.cursor, + &mut self.captures, + &mut self.highlights, + )?; + let mut ancestors = request.ancestors; + ancestors.push((request.syntax, request.ranges.clone())); + for child in &mut children { + child.ancestors = ancestors.clone(); + } + queue.extend(children); + self.injections.push(Layer { + syntax: request.syntax, + tree, + ranges: request.ranges, + depth: request.depth, + dirty: false, + }); + } + render_spans( + &self.source, + &self.line_starts, + &mut self.lines, + &self.highlights, + &mut self.events, + ); + Ok(()) } } -fn syntax_set() -> &'static SyntaxSet { - static SYNTAXES: LazyLock = LazyLock::new(SyntaxSet::load_defaults_newlines); - &SYNTAXES +fn same_language(left: Option<&LanguageSpec>, right: Option<&LanguageSpec>) -> bool { + match (left, right) { + (Some(left), Some(right)) => std::ptr::eq(left, right), + (None, None) => true, + _ => false, + } } -fn detect_syntax<'a>( - syntaxes: &'a SyntaxSet, - path: Option<&Path>, - first_line: Option<&str>, -) -> Option<&'a SyntaxReference> { - let by_path = path.and_then(|path| { - path.file_name() - .and_then(|name| name.to_str()) - .and_then(|name| syntaxes.find_syntax_by_extension(name)) - .or_else(|| { - path.extension() - .and_then(|extension| extension.to_str()) - .and_then(|extension| syntaxes.find_syntax_by_extension(extension)) - }) - }); - by_path - .or_else(|| first_line.and_then(|line| syntaxes.find_syntax_by_first_line(line))) - .filter(|syntax| syntax.name != "Plain Text") -} - -fn classify(scopes: &ScopeStack) -> Option { - static SELECTORS: LazyLock<[(ScopeSelectors, SyntaxKind); 7]> = LazyLock::new(|| { - [ - ("comment", SyntaxKind::Comment), - ("string", SyntaxKind::String), - ("keyword, storage, constant.language", SyntaxKind::Keyword), - ("constant.numeric", SyntaxKind::Number), - ( - "entity.name.type, entity.name.class, entity.name.struct, entity.name.enum, \ - entity.name.trait, support.type, support.class", - SyntaxKind::Type, - ), - ( - "entity.name.function, support.function", - SyntaxKind::Function, - ), - ("markup, entity.name.tag", SyntaxKind::Markup), - ] - .map(|(selector, kind)| { - ( - selector.parse().expect("fixed semantic scope selector"), - kind, - ) - }) - }); - SELECTORS +/// One aggregate edit, including byte columns (not character/display columns). +fn minimal_edit(old: &str, new: &str) -> InputEdit { + let mut start = old + .bytes() + .zip(new.bytes()) + .take_while(|(a, b)| a == b) + .count(); + while !old.is_char_boundary(start) || !new.is_char_boundary(start) { + start -= 1; + } + let mut suffix = old.as_bytes()[start..] .iter() - .filter_map(|(selector, kind)| { - selector - .does_match(scopes.as_slice()) - .map(|power| (power, *kind)) - }) - .max_by_key(|(power, _)| *power) - .map(|(_, kind)| kind) + .rev() + .zip(new.as_bytes()[start..].iter().rev()) + .take_while(|(a, b)| a == b) + .count(); + while !old.is_char_boundary(old.len() - suffix) || !new.is_char_boundary(new.len() - suffix) { + suffix -= 1; + } + let old_end = old.len() - suffix; + let new_end = new.len() - suffix; + InputEdit { + start_byte: start, + old_end_byte: old_end, + new_end_byte: new_end, + start_position: point_at(old, start), + old_end_position: point_at(old, old_end), + new_end_position: point_at(new, new_end), + } } -fn collect_spans( - line_len: usize, - changes: &[(usize, ScopeStackOp)], - scopes: &mut ScopeStack, - spans: &mut Vec, -) -> Result<(), ScopeError> { - spans.clear(); - let mut start = 0; - let mut changes = changes.iter().peekable(); - while let Some((offset, operation)) = changes.next() { - let end = (*offset).min(line_len); - if start < end { - append_span(spans, start, end, classify(scopes)); +fn point_at(source: &str, byte: usize) -> Point { + let prefix = &source.as_bytes()[..byte]; + Point::new( + prefix.iter().filter(|byte| **byte == b'\n').count(), + prefix + .iter() + .rposition(|byte| *byte == b'\n') + .map_or(byte, |last| byte - last - 1), + ) +} + +struct Configuration { + syntax: &'static LanguageSpec, + query: Query, + locals_start: usize, + highlights_start: usize, +} + +fn configuration(syntax: &'static LanguageSpec) -> Result<&'static Configuration, String> { + static CONFIGURATIONS: LazyLock)>> = + LazyLock::new(|| { + languages::all() + .map(|syntax| (syntax, compile_configuration(syntax))) + .collect() + }); + let (_, result) = CONFIGURATIONS + .iter() + .find(|(spec, _)| std::ptr::eq(*spec, syntax)) + .ok_or_else(|| format!("missing {} query configuration", syntax.name))?; + result.as_ref().map_err(Clone::clone) +} + +fn compile_configuration(syntax: &'static LanguageSpec) -> Result { + let source = format!( + "{}\n{}\n{}", + syntax.injections, syntax.locals, syntax.highlights + ); + let locals_offset = syntax.injections.len() + 1; + let highlights_offset = locals_offset + syntax.locals.len() + 1; + let query = Query::new(&syntax.language, &source).map_err(|error| error.to_string())?; + let mut locals_start = 0; + let mut highlights_start = 0; + for pattern in 0..query.pattern_count() { + let start = query.start_byte_for_pattern(pattern); + locals_start += usize::from(start < locals_offset); + highlights_start += usize::from(start < highlights_offset); + if let Some(predicate) = query.general_predicates(pattern).first() { + return Err(format!( + "unsupported query predicate #{}", + predicate.operator + )); } - scopes.apply(operation)?; - while changes.peek().is_some_and(|(next, _)| next == offset) { - scopes.apply(&changes.next().unwrap().1)?; + if let Some((property, _)) = query + .property_predicates(pattern) + .iter() + .find(|(property, _)| property.key.as_ref() != "local") + { + return Err(format!( + "unsupported query property predicate {}", + property.key + )); + } + } + Ok(Configuration { + syntax, + query, + locals_start, + highlights_start, + }) +} + +fn classify(capture: &str, numeric_literal: bool) -> Option { + let mut parts = capture.split('.'); + match parts.next()? { + "comment" => Some(SyntaxKind::Comment), + "string" | "character" => Some(SyntaxKind::String), + "keyword" | "boolean" | "storage" | "conditional" | "repeat" | "include" | "exception" => { + Some(SyntaxKind::Keyword) + } + "number" | "float" => Some(SyntaxKind::Number), + "constant" if numeric_literal || parts.any(|part| part == "numeric") => { + Some(SyntaxKind::Number) + } + "constant" => Some(SyntaxKind::Keyword), + "type" | "constructor" => Some(SyntaxKind::Type), + "function" | "method" => Some(SyntaxKind::Function), + "markup" | "tag" | "text" => Some(SyntaxKind::Markup), + _ => None, + } +} + +struct Layer { + syntax: &'static LanguageSpec, + tree: Tree, + ranges: Vec, + depth: usize, + dirty: bool, +} + +struct Injection { + syntax: &'static LanguageSpec, + parent: &'static LanguageSpec, + ranges: Vec, + depth: usize, + ancestors: Vec<(&'static LanguageSpec, Vec)>, +} + +#[derive(Clone, Copy)] +struct Capture { + node: usize, + start: usize, + end: usize, + capture: u32, + pattern: usize, + value_end: usize, + numeric_literal: bool, +} + +struct Highlight { + start: usize, + end: usize, + kind: Option, + depth: usize, + priority: i32, + pattern: usize, +} + +struct Definition { + value_end: usize, + highlight: Option, +} + +struct Scope<'a> { + end: usize, + inherits: bool, + definitions: HashMap<&'a str, Vec>, +} + +#[allow(clippy::too_many_arguments)] +fn collect_layer( + configuration: &Configuration, + tree: &Tree, + source: &str, + ranges: &[Range], + depth: usize, + parent: Option<&'static LanguageSpec>, + cursor: &mut QueryCursor, + captures: &mut Vec, + highlights: &mut Vec, +) -> Result, String> { + let query = &configuration.query; + let names = query.capture_names(); + captures.clear(); + let mut injections = Vec::new(); + let mut combined: HashMap<(usize, usize), usize> = HashMap::new(); + let mut matches = cursor.matches(query, tree.root_node(), source.as_bytes()); + while let Some(found) = matches.next() { + let pattern = found.pattern_index; + if pattern < configuration.locals_start { + let mut language = None; + let mut nodes = Vec::new(); + for capture in found.captures() { + match names[capture.index as usize] { + "injection.language" => { + language = source[capture.node.byte_range()] + .split_whitespace() + .next() + .and_then(languages::by_name); + } + "injection.content" => nodes.push(capture.node), + _ => {} + } + } + let mut include_children = false; + let mut is_combined = false; + for property in query.property_settings(pattern) { + match property.key.as_ref() { + "injection.language" => { + language = property.value.as_deref().and_then(languages::by_name); + } + "injection.self" => language = Some(configuration.syntax), + "injection.parent" => language = parent, + "injection.include-children" => include_children = true, + "injection.combined" => is_combined = true, + _ => {} + } + } + if let Some(language) = language { + let mut content = injection_ranges(&nodes, ranges, include_children); + if !content.is_empty() { + let key = (pattern, language as *const LanguageSpec as usize); + if is_combined && let Some(index) = combined.get(&key) { + let injection: &mut Injection = &mut injections[*index]; + injection.ranges.append(&mut content); + } else { + if is_combined { + combined.insert(key, injections.len()); + } + injections.push(Injection { + syntax: language, + parent: configuration.syntax, + ranges: content, + depth: depth + 1, + ancestors: vec![(configuration.syntax, ranges.to_vec())], + }); + } + } + } + continue; + } + let value_end = found + .captures() + .iter() + .find(|capture| names[capture.index as usize] == "local.definition-value") + .map_or(0, |capture| capture.node.end_byte()); + for capture in found.captures() { + let name = names[capture.index as usize]; + if name.starts_with('_') || name == "local.definition-value" { + continue; + } + captures.push(Capture { + node: capture.node.id(), + start: capture.node.start_byte(), + end: capture.node.end_byte(), + capture: capture.index, + pattern, + value_end, + numeric_literal: name.starts_with("constant") + && matches!( + capture.node.kind(), + "integer_literal" + | "float_literal" + | "number_literal" + | "integer" + | "float" + | "number" + ), + }); + } + } + if cursor.did_exceed_match_limit() { + return Err(format!( + "{} query exceeded its match limit", + configuration.syntax.name + )); + } + for injection in &mut injections { + normalize_ranges(&mut injection.ranges); + } + captures.sort_unstable_by_key(|capture| { + ( + capture.start, + Reverse(capture.end), + capture.node, + capture.pattern, + capture.capture, + ) + }); + let mut scopes = vec![Scope { + end: usize::MAX, + inherits: false, + definitions: HashMap::new(), + }]; + let mut start = 0; + while start < captures.len() { + let first = captures[start]; + let mut end = start + 1; + while end < captures.len() && captures[end].node == first.node { + end += 1; + } + let group = &captures[start..end]; + while scopes.len() > 1 && first.start >= scopes.last().unwrap().end { + scopes.pop(); + } + let mut definition = None; + let mut reference = None; + let mut is_local = false; + for capture in group + .iter() + .filter(|capture| capture.pattern < configuration.highlights_start) + { + match names[capture.capture as usize] { + "local.scope" => { + let inherits = query + .property_settings(capture.pattern) + .iter() + .find(|property| property.key.as_ref() == "local.scope-inherits") + .is_none_or(|property| { + property + .value + .as_deref() + .is_none_or(|value| value == "true") + }); + scopes.push(Scope { + end: capture.end, + inherits, + definitions: HashMap::new(), + }); + } + "local.definition" => { + is_local = true; + definition = Some(capture.value_end); + } + "local.reference" if definition.is_none() => { + let name = &source[capture.start..capture.end]; + for scope in scopes.iter().rev() { + if let Some(found) = scope.definitions.get(name).and_then(|definitions| { + definitions + .iter() + .rev() + .find(|definition| capture.start >= definition.value_end) + }) { + is_local = true; + reference = Some(found.highlight); + break; + } + if !scope.inherits { + break; + } + } + } + _ => {} + } + } + let selected = group + .iter() + .filter(|capture| { + capture.pattern >= configuration.highlights_start + && query.property_predicates(capture.pattern).iter().all( + |(property, positive)| { + let local = property + .capture_id + .map_or(is_local, |id| id == capture.capture as usize && is_local); + local == *positive + }, + ) + }) + .max_by_key(|capture| { + ( + capture_priority(query, capture), + capture.pattern, + capture.capture, + ) + }); + let kind = reference.unwrap_or_else(|| { + selected.and_then(|capture| { + classify(names[capture.capture as usize], capture.numeric_literal) + }) + }); + if let Some(value_end) = definition { + let name = &source[first.start..first.end]; + scopes + .last_mut() + .unwrap() + .definitions + .entry(name) + .or_default() + .push(Definition { + value_end, + highlight: kind, + }); + } + if let Some(kind) = kind { + let selected = selected.copied().unwrap_or(first); + let range_start = ranges.partition_point(|range| range.end_byte <= first.start); + for range in ranges[range_start..] + .iter() + .take_while(|range| range.start_byte < first.end) + { + let low = first.start.max(range.start_byte); + let high = first.end.min(range.end_byte); + if low < high { + highlights.push(Highlight { + start: low, + end: high, + kind: Some(kind), + depth, + priority: capture_priority(query, &selected), + pattern: selected.pattern, + }); + } + } } start = end; } - if start < line_len { - append_span(spans, start, line_len, classify(scopes)); - } - Ok(()) + Ok(injections) } -fn append_span(spans: &mut Vec, start: usize, end: usize, kind: Option) { - if start >= end { +fn capture_priority(query: &Query, capture: &Capture) -> i32 { + query + .property_settings(capture.pattern) + .iter() + .rev() + .find_map(|property| { + if property.key.as_ref() == "priority" + && property + .capture_id + .is_none_or(|id| id == capture.capture as usize) + { + property.value.as_deref()?.parse().ok() + } else { + None + } + }) + .unwrap_or(0) +} + +fn injection_ranges(nodes: &[Node<'_>], parents: &[Range], include_children: bool) -> Vec { + let mut result = Vec::new(); + for node in nodes { + let mut start_byte = node.start_byte(); + let mut start_point = node.start_position(); + let mut cursor = node.walk(); + let excluded: Vec<_> = if include_children { + Vec::new() + } else { + // The bundled Markdown queries are Neovim-derived: anonymous punctuation is + // injected source, while named children (e.g. block continuations) stay in the host. + node.named_children(&mut cursor) + .map(|child| child.range()) + .collect() + }; + for excluded in excluded.into_iter().chain(std::iter::once(Range { + start_byte: node.end_byte(), + end_byte: node.end_byte(), + start_point: node.end_position(), + end_point: node.end_position(), + })) { + let parent_start = parents.partition_point(|range| range.end_byte <= start_byte); + for parent in parents[parent_start..] + .iter() + .take_while(|range| range.start_byte < excluded.start_byte) + { + let low = start_byte.max(parent.start_byte); + let high = excluded.start_byte.min(parent.end_byte); + if low < high { + result.push(Range { + start_byte: low, + end_byte: high, + start_point: if low == start_byte { + start_point + } else { + parent.start_point + }, + end_point: if high == excluded.start_byte { + excluded.start_point + } else { + parent.end_point + }, + }); + } + } + start_byte = excluded.end_byte; + start_point = excluded.end_point; + } + } + normalize_ranges(&mut result); + result +} + +fn normalize_ranges(ranges: &mut Vec) { + ranges.sort_unstable_by_key(|range| (range.start_byte, range.end_byte)); + let mut length = 0; + for index in 0..ranges.len() { + let range = ranges[index]; + if length > 0 && ranges[length - 1].end_byte >= range.start_byte { + let previous = &mut ranges[length - 1]; + if previous.end_byte < range.end_byte { + previous.end_byte = range.end_byte; + previous.end_point = range.end_point; + } + } else { + ranges[length] = range; + length += 1; + } + } + ranges.truncate(length); +} + +fn render_spans( + source: &str, + starts: &[usize], + lines: &mut [Vec], + highlights: &[Highlight], + events: &mut Vec<(usize, bool, usize)>, +) { + events.clear(); + for (index, highlight) in highlights.iter().enumerate() { + events.push((highlight.start, true, index)); + events.push((highlight.end, false, index)); + } + events.sort_unstable(); + let rank = |index: usize| { + let highlight = &highlights[index]; + ( + highlight.depth, + highlight.priority, + Reverse(highlight.end - highlight.start), + highlight.pattern, + index, + ) + }; + let mut active: BTreeSet<(_, usize)> = BTreeSet::new(); + let mut previous = 0; + for &(byte, entering, index) in events.iter() { + if previous < byte + && let Some((_, current)) = active.last() + && let Some(kind) = highlights[*current].kind + { + append_global(source, starts, lines, previous, byte, kind); + } + if entering { + active.insert((rank(index), index)); + } else { + active.remove(&(rank(index), index)); + } + previous = byte; + } +} + +fn append_global( + source: &str, + starts: &[usize], + lines: &mut [Vec], + mut start: usize, + end: usize, + kind: SyntaxKind, +) { + if starts.is_empty() { return; } - let Some(kind) = kind else { - return; - }; - if let Some(previous) = spans.last_mut() - && previous.end_byte == start - && previous.kind == kind - { - previous.end_byte = end; - } else { - spans.push(SyntaxSpan { - start_byte: start, - end_byte: end, - kind, - }); + let mut row = starts + .partition_point(|offset| *offset <= start) + .saturating_sub(1); + while start < end && row < lines.len() { + let line_end = starts.get(row + 1).map_or(source.len(), |next| next - 1); + let high = end.min(line_end); + if start < high { + let spans = &mut lines[row]; + let low = start - starts[row]; + let high = high - starts[row]; + if let Some(previous) = spans.last_mut() + && previous.end_byte == low + && previous.kind == kind + { + previous.end_byte = high; + } else { + spans.push(SyntaxSpan { + start_byte: low, + end_byte: high, + kind, + }); + } + } + row += 1; + start = line_end + 1; } } #[cfg(test)] mod tests { use super::*; - use syntect::parsing::{ClearAmount, Scope}; fn lines(source: &str) -> Vec { source.split('\n').map(str::to_owned).collect() } + fn cache(source: &str, path: &str) -> SyntaxCache { + let mut cache = SyntaxCache::new(); + cache.update(&lines(source), Some(Path::new(path)), 0); + assert_eq!(cache.error(), None, "{path}"); + cache + } + fn kind_at(cache: &SyntaxCache, row: usize, byte: usize) -> Option { cache .line(row) @@ -299,405 +926,614 @@ mod tests { .find_map(|span| (span.start_byte <= byte && byte < span.end_byte).then_some(span.kind)) } - fn assert_token( - cache: &SyntaxCache, - source: &[String], - row: usize, - token: &str, - kind: SyntaxKind, - ) { - let start = source[row].find(token).unwrap(); - for (byte, _) in source[row][start..start + token.len()].char_indices() { + fn assert_token(cache: &SyntaxCache, source: &str, row: usize, token: &str, kind: SyntaxKind) { + let line = source.split('\n').nth(row).unwrap(); + let start = line.find(token).unwrap(); + for (byte, _) in line[start..start + token.len()].char_indices() { assert_eq!( kind_at(cache, row, start + byte), Some(kind), - "{token:?} on row {row}" + "{token:?} in {}", + cache.language().unwrap() ); } } - fn assert_fresh(cache: &SyntaxCache, source: &[String], path: Option<&Path>) { - let mut fresh = SyntaxCache::new(); - fresh.update(source, path, 0); + fn assert_fresh(cache: &SyntaxCache, source: &str, path: &str) { + let fresh = self::cache(source, path); assert_eq!(cache.language(), fresh.language()); assert_eq!(cache.error(), fresh.error()); - for row in 0..source.len() { - assert_eq!(cache.line(row), fresh.line(row), "row {row}"); - } - } - - #[test] - fn bundled_languages_supply_semantic_scopes() { - let cases = [ - ( - "sample.rs", - "Rust", - "fn main() { let value = 42; let text = \"hello\"; /* note */ }", - "fn", - "main", - ), - ( - "sample.rb", - "Ruby", - "def hello\n value = 42\n text = \"hello\" # note\nend", - "def", - "hello", - ), - ( - "sample.py", - "Python", - "def hello():\n value = 42\n text = \"hello\" # note", - "def", - "hello", - ), - ]; - for (path, language, text, keyword, function) in cases { - let source = lines(text); - let mut cache = SyntaxCache::new(); - cache.update(&source, Some(Path::new(path)), 0); - assert_eq!(cache.language(), Some(language)); - assert_eq!(cache.error(), None); - assert_token(&cache, &source, 0, keyword, SyntaxKind::Keyword); - assert_token(&cache, &source, 0, function, SyntaxKind::Function); - let number_row = if language == "Rust" { 0 } else { 1 }; - let string_row = if language == "Rust" { 0 } else { 2 }; - assert_token(&cache, &source, number_row, "42", SyntaxKind::Number); - // Select the quoted occurrence, not a function with the same name. - let quoted = source[string_row].find("\"hello\"").unwrap(); - assert_eq!( - kind_at(&cache, string_row, quoted + 1), - Some(SyntaxKind::String) - ); - assert_token(&cache, &source, string_row, "note", SyntaxKind::Comment); - } - - let source = lines("{\"message\":\"hello\",\"n\":42,\"enabled\":true}"); - let mut cache = SyntaxCache::new(); - cache.update(&source, Some(Path::new("sample.json")), 0); - assert_eq!(cache.language(), Some("JSON")); - assert_token(&cache, &source, 0, "hello", SyntaxKind::String); - assert_token(&cache, &source, 0, "42", SyntaxKind::Number); - assert_token(&cache, &source, 0, "true", SyntaxKind::Keyword); - assert_eq!(cache.error(), None); - } - - #[test] - fn detects_bundled_special_filenames_extensions_and_actual_shebangs() { - let source = lines("all:\n\techo hello"); - let mut cache = SyntaxCache::new(); - cache.update(&source, Some(Path::new("/not/read/from/disk/Makefile")), 0); - assert_eq!(cache.language(), Some("Makefile")); - - let source = lines("#!/usr/bin/env bash\necho \"hello\" # note"); - cache.update(&source, Some(Path::new("/not/read/from/disk/script")), 1); - assert_eq!(cache.language(), Some("Bourne Again Shell (bash)")); - assert_token(&cache, &source, 1, "hello", SyntaxKind::String); - assert_token(&cache, &source, 1, "note", SyntaxKind::Comment); - assert_eq!(cache.error(), None); - - let source = lines("echo \"hello\" # note"); - cache.update(&source, Some(Path::new(".bashrc")), 1); - assert_eq!(cache.language(), Some("Bourne Again Shell (bash)")); - assert_token(&cache, &source, 0, "hello", SyntaxKind::String); - - let source = lines("\u{feff}#!/usr/bin/env python3\nreturn 42"); - cache.update(&source, None, 2); - assert_eq!(cache.language(), Some("Python")); - assert_token(&cache, &source, 1, "return", SyntaxKind::Keyword); - cache.update(&source, Some(Path::new("sample.RS")), 2); - assert_eq!(cache.language(), Some("Rust")); - } - - #[test] - fn unknown_and_plain_text_have_no_spans_or_errors() { - let source = lines("fn main() { \"hello\"; 42; }\n/* note"); - let mut cache = SyntaxCache::new(); - for path in [ - None, - Some(Path::new("sample.unknown-extension")), - Some(Path::new("sample.txt")), - ] { - cache.update(&source, path, 0); - assert_eq!(cache.language(), None); - assert_eq!(cache.error(), None); - assert!(cache.line(0).is_empty()); - assert!(cache.line(1).is_empty()); - assert!(cache.line(500).is_empty()); - } - cache.update(&[], Some(Path::new("empty.rs")), 1); - assert_eq!(cache.language(), Some("Rust")); - assert!(cache.line(0).is_empty()); - } - - #[test] - fn nested_multiline_comments_and_edits_before_visible_rows() { - let mut source = lines("/* outer\n/* inner */ still outer\nlast */ fn main() {}\n// tail"); - let path = Some(Path::new("sample.rs")); - let mut cache = SyntaxCache::new(); - cache.update(&source, path, 0); - assert_token(&cache, &source, 1, "inner", SyntaxKind::Comment); - assert_token(&cache, &source, 1, "still outer", SyntaxKind::Comment); - assert_token(&cache, &source, 2, "last", SyntaxKind::Comment); - assert_token(&cache, &source, 2, "fn", SyntaxKind::Keyword); - - source[0] = "fn earlier() {}".into(); - cache.update(&source, path, 1); assert_eq!( - kind_at(&cache, 1, source[1].find("still outer").unwrap()), - None + cache.tree.as_ref().map(|tree| tree.root_node().to_sexp()), + fresh.tree.as_ref().map(|tree| tree.root_node().to_sexp()) ); - assert_eq!(kind_at(&cache, 2, 0), None); - assert_fresh(&cache, &source, path); - - source[0] = "/* outer".into(); - cache.update(&source, path, 2); - assert_token(&cache, &source, 2, "last", SyntaxKind::Comment); - assert_fresh(&cache, &source, path); - } - - #[test] - fn multiline_strings_preserve_unicode_byte_ranges_and_tolerate_unfinished_source() { - let mut source = lines( - "text = \"\"\"start\n\tη•Œe\u{301}πŸ‘©β€πŸ’» middle\nfinish\"\"\"\nreturn 42 # η•Œe\u{301}πŸ‘©β€πŸ’»", - ); - let path = Some(Path::new("sample.py")); - let mut cache = SyntaxCache::new(); - cache.update(&source, path, 0); - assert_token( - &cache, - &source, - 1, - "\tη•Œe\u{301}πŸ‘©β€πŸ’» middle", - SyntaxKind::String, - ); - assert_token(&cache, &source, 2, "finish", SyntaxKind::String); - assert_token(&cache, &source, 3, "return", SyntaxKind::Keyword); - assert_token(&cache, &source, 3, "η•Œe\u{301}πŸ‘©β€πŸ’»", SyntaxKind::Comment); + let source = lines(source); for (row, line) in source.iter().enumerate() { - let mut end = 0; + assert_eq!(cache.line(row), fresh.line(row), "row {row}"); + let mut previous = 0; for span in cache.line(row) { - assert!(end <= span.start_byte && span.start_byte < span.end_byte); + assert!(previous <= span.start_byte && span.start_byte < span.end_byte); assert!(span.end_byte <= line.len()); - assert!(line.is_char_boundary(span.start_byte)); - assert!(line.is_char_boundary(span.end_byte)); - end = span.end_byte; + assert!( + line.is_char_boundary(span.start_byte) && line.is_char_boundary(span.end_byte) + ); + previous = span.end_byte; } } - source[2] = "unfinished".into(); - cache.update(&source, path, 1); - assert_token(&cache, &source, 3, "return", SyntaxKind::String); - assert_eq!(cache.error(), None); - assert_fresh(&cache, &source, path); } #[test] - fn blank_lines_keep_scopes_and_unicode_before_tokens_keeps_byte_offsets() { - let source = lines("/* first\n\n\tη•Œe\u{301}πŸ‘©β€πŸ’» last */ fn main() { let value = 42; }"); - let mut cache = SyntaxCache::new(); - cache.update(&source, Some(Path::new("sample.rs")), 0); - assert!(cache.line(1).is_empty()); - assert_token(&cache, &source, 2, "η•Œe\u{301}πŸ‘©β€πŸ’» last", SyntaxKind::Comment); - assert_token(&cache, &source, 2, "fn", SyntaxKind::Keyword); - assert_token(&cache, &source, 2, "main", SyntaxKind::Function); - assert_token(&cache, &source, 2, "42", SyntaxKind::Number); - for span in cache.line(2) { - assert!(source[2].is_char_boundary(span.start_byte)); - assert!(source[2].is_char_boundary(span.end_byte)); - assert!(span.end_byte <= source[2].len()); + fn every_bundled_query_compiles_and_common_languages_color_real_tokens() { + for language in languages::all() { + configuration(language).unwrap_or_else(|error| panic!("{}: {error}", language.name)); } - assert_eq!(cache.error(), None); - } - - #[test] - fn revisions_reuse_prefix_checkpoints_and_unchanged_source_snapshots() { - let mut source = lines("fn main() {\n /* note\n continued */\n let value = 42;\n}"); - let path = Some(Path::new("sample.rs")); - let mut cache = SyntaxCache::new(); - cache.update(&source, path, 10); - assert_eq!(cache.parsed_lines, 5); - let prefix_source = cache.lines[0].source.as_ptr(); - let prefix_spans = cache.lines[0].spans.as_ptr(); - cache.update(&source, path, 10); - cache.update(&source, path, 11); - assert_eq!(cache.parsed_lines, 5); - - source[2] = " changed */".into(); - cache.update(&source, path, 12); - assert_eq!(cache.parsed_lines, 8); - assert_eq!(cache.lines[0].source.as_ptr(), prefix_source); - assert_eq!(cache.lines[0].spans.as_ptr(), prefix_spans); - assert_token(&cache, &source, 2, "changed", SyntaxKind::Comment); - assert_fresh(&cache, &source, path); - - source[2] = " continued */".into(); - cache.update(&source, path, 13); - assert_eq!(cache.parsed_lines, 11); - assert_fresh(&cache, &source, path); - cache.update(&source, path, 14); - assert_eq!(cache.parsed_lines, 11); - } - - #[test] - fn insertion_removal_and_appending_preserve_multiline_state() { - let path = Some(Path::new("sample.py")); - let mut source = lines("text = \"\"\"start\nmiddle\nend\"\"\"\nreturn 42"); - let mut cache = SyntaxCache::new(); - cache.update(&source, path, 0); - assert_eq!(cache.parsed_lines, 4); - - source.insert(1, "inserted η•Œ".into()); - cache.update(&source, path, 1); - assert_eq!(cache.parsed_lines, 8); - assert_token(&cache, &source, 2, "middle", SyntaxKind::String); - assert_fresh(&cache, &source, path); - - source.remove(1); - cache.update(&source, path, 2); - assert_eq!(cache.parsed_lines, 11); - assert_fresh(&cache, &source, path); - - source.push("# appended".into()); - cache.update(&source, path, 3); - assert_eq!(cache.parsed_lines, 12); - assert_token(&cache, &source, 4, "appended", SyntaxKind::Comment); - source.pop(); - cache.update(&source, path, 4); - assert_eq!(cache.parsed_lines, 12); - assert!(cache.line(4).is_empty()); - assert_fresh(&cache, &source, path); - } - - #[test] - fn rename_at_the_same_revision_and_shebang_edits_redetect_language() { - let mut source = lines("fn main() { let value = 42; }"); - let mut cache = SyntaxCache::new(); - cache.update(&source, Some(Path::new("old.rs")), 7); - assert_eq!(cache.parsed_lines, 1); - cache.update(&source, Some(Path::new("new.RS")), 7); - assert_eq!(cache.parsed_lines, 1); - cache.update(&source, Some(Path::new("new.txt")), 7); - assert_eq!(cache.language(), None); - assert!(cache.line(0).is_empty()); - cache.update(&source, Some(Path::new("new.rs")), 7); - assert_eq!(cache.parsed_lines, 2); - assert_token(&cache, &source, 0, "fn", SyntaxKind::Keyword); - - source = lines("#!/usr/bin/env bash\necho \"hello\""); - cache.update(&source, None, 8); - assert_eq!(cache.language(), Some("Bourne Again Shell (bash)")); - source[0] = "#!/usr/bin/env python3".into(); - source[1] = "return \"hello\"".into(); - cache.update(&source, None, 9); - assert_eq!(cache.language(), Some("Python")); - assert_token(&cache, &source, 1, "return", SyntaxKind::Keyword); - assert_fresh(&cache, &source, None); - } - - #[test] - fn type_function_and_markup_categories_come_from_scopes() { - let mut cache = SyntaxCache::new(); - let source = lines("struct Widget;\nfn build() {}"); - cache.update(&source, Some(Path::new("sample.rs")), 0); - assert_token(&cache, &source, 0, "Widget", SyntaxKind::Type); - assert_token(&cache, &source, 1, "build", SyntaxKind::Function); - let source = lines("# Heading"); - cache.update(&source, Some(Path::new("sample.md")), 1); - assert_token(&cache, &source, 0, "Heading", SyntaxKind::Markup); - } - - #[test] - fn deepest_scope_match_wins_and_clear_restore_survives_checkpoints() { - let mut scopes: ScopeStack = "source.python string.quoted.double.python".parse().unwrap(); - let mut spans = Vec::new(); - let changes = [ - (2, ScopeStackOp::Clear(ClearAmount::All)), - (2, ScopeStackOp::Push(Scope::new("source.rust").unwrap())), + let cases = [ ( - 2, - ScopeStackOp::Push(Scope::new("comment.block.rust").unwrap()), + "a.rs", + "Rust", + "fn main() { let x = 42; let s = \"hi\"; /* note */ }", + "fn", + SyntaxKind::Keyword, + ), + ( + "a.rb", + "Ruby", + "def hello\n 42\nend", + "def", + SyntaxKind::Keyword, + ), + ( + "a.py", + "Python", + "def hello():\n return 42", + "def", + SyntaxKind::Keyword, + ), + ( + "a.json", + "JSON", + "{\"n\":42,\"yes\":true}", + "42", + SyntaxKind::Number, + ), + ( + "a.sh", + "Bash", + "echo \"hi\" # note", + "hi", + SyntaxKind::String, + ), + ( + "Makefile", + "Makefile", + "# note\nall:\n\techo hi", + "note", + SyntaxKind::Comment, + ), + ( + "a.md", + "Markdown", + "# Heading\n\n**bold**", + "Heading", + SyntaxKind::Markup, + ), + ( + "a.js", + "JavaScript", + "function hello() { return 42; }", + "function", + SyntaxKind::Keyword, + ), + ( + "a.ts", + "TypeScript", + "function hello(): number { return 42; }", + "function", + SyntaxKind::Keyword, + ), + ( + "a.html", + "HTML", + "
text
", + "div", + SyntaxKind::Markup, + ), + ( + "a.css", + "CSS", + "body { width: 42px; }", + "42", + SyntaxKind::Number, + ), + ( + "a.yaml", + "YAML", + "message: \"hi\"\ncount: 42", + "hi", + SyntaxKind::String, + ), + ( + "Cargo.toml", + "TOML", + "message = \"hi\"\ncount = 42", + "hi", + SyntaxKind::String, + ), + ( + "a.c", + "C", + "int main(void) { return 42; }", + "return", + SyntaxKind::Keyword, + ), + ( + "a.cpp", + "C++", + "int main() { return 42; }", + "return", + SyntaxKind::Keyword, + ), + ( + "a.go", + "Go", + "package main\nfunc main() { return }", + "package", + SyntaxKind::Keyword, ), ]; - collect_spans(4, &changes, &mut scopes, &mut spans).unwrap(); + for (path, language, source, token, kind) in cases { + let cache = self::cache(source, path); + assert_eq!(cache.language(), Some(language)); + assert_token(&cache, source, 0, token, kind); + assert_fresh(&cache, source, path); + } + } + + #[test] + fn semantic_colors_and_text_predicates_preserve_the_ui_contract() { + let source = + "fn main() { let n = 42; let f = 3.25; let b = true; let s = \"hello\"; /* note */ }"; + let cache = self::cache(source, "a.rs"); + for (token, kind) in [ + ("fn", SyntaxKind::Keyword), + ("let", SyntaxKind::Keyword), + ("main", SyntaxKind::Function), + ("42", SyntaxKind::Number), + ("3.25", SyntaxKind::Number), + ("true", SyntaxKind::Keyword), + ("hello", SyntaxKind::String), + ("note", SyntaxKind::Comment), + ] { + assert_token(&cache, source, 0, token, kind); + } + let source = "const foo = 42; const UPPER = 7; foo; UPPER;"; + let cache = self::cache(source, "a.js"); + assert_eq!(kind_at(&cache, 0, source.find("foo").unwrap()), None); + // JavaScript's bundled constant predicate must not color every identifier. + assert_ne!( + kind_at(&cache, 0, source.find("foo;").unwrap()), + Some(SyntaxKind::Keyword) + ); + } + + #[test] + fn filenames_shebang_save_as_and_tsx_use_actual_grammar_identity() { + let source = lines("\u{feff}#!/usr/bin/env python3\nreturn 42"); + let mut cache = SyntaxCache::new(); + cache.update(&source, None, 0); + assert_eq!(cache.language(), Some("Python")); + cache.update(&source, Some(Path::new("a.RS")), 0); + assert_eq!(cache.language(), Some("Rust")); + assert_eq!(cache.parses, 2); + cache.update(&source, Some(Path::new("renamed.rs")), 0); + assert_eq!(cache.parses, 2); + cache.update(&lines("all:\n\techo hi"), Some(Path::new("GNUmakefile")), 1); + assert_eq!(cache.language(), Some("Makefile")); + let source = "const element =
;"; + cache.update(&lines(source), Some(Path::new("a.ts")), 2); + let typescript = cache.syntax.unwrap(); + let parses = cache.parses; + cache.update(&lines(source), Some(Path::new("a.tsx")), 2); + assert_eq!(cache.language(), Some("TypeScript")); + assert!(!std::ptr::eq(typescript, cache.syntax.unwrap())); + assert_eq!(cache.parses, parses + 1); + assert_token(&cache, source, 0, "div", SyntaxKind::Markup); + assert_fresh(&cache, source, "a.tsx"); + let mut script = SyntaxCache::new(); + script.update(&lines("#!/usr/bin/env python3\nreturn 42"), None, 0); + assert_eq!(script.language(), Some("Python")); + script.update(&lines("#!/usr/bin/env bash\necho \"hi\""), None, 1); + assert_eq!(script.language(), Some("Bash")); + assert_token( + &script, + "#!/usr/bin/env bash\necho \"hi\"", + 1, + "hi", + SyntaxKind::String, + ); + } + + #[test] + fn unknown_and_former_non_common_languages_are_honest_plain_text() { + for path in [ + "a.unknown", + "a.txt", + "a.hs", + "a.java", + "a.php", + "a.swift", + "a.sql", + ] { + let cache = self::cache("fn main() { \"hi\"; 42; }", path); + assert_eq!(cache.language(), None, "{path}"); + assert!(cache.lines.iter().all(Vec::is_empty)); + assert_eq!(cache.parses, 0); + } + } + + #[test] + fn unchanged_revision_and_same_source_do_not_allocate_or_parse() { + let mut cache = self::cache("fn main() {}\n", "a.rs"); + let source = lines("fn main() {}\n"); + let source_pointer = cache.source.as_ptr(); + let scratch_pointer = cache.next_source.as_ptr(); + let tree_id = cache.tree.as_ref().unwrap().root_node().id(); + cache.update(&source, Some(Path::new("a.rs")), 0); + assert_eq!(cache.source.as_ptr(), source_pointer); + assert_eq!(cache.next_source.as_ptr(), scratch_pointer); + assert_eq!(cache.tree.as_ref().unwrap().root_node().id(), tree_id); + cache.update(&source, Some(Path::new("a.rs")), 1); + cache.update(&source, Some(Path::new("b.rs")), 1); + assert_eq!(cache.parses, 1); + assert_eq!(cache.incremental_parses, 0); + assert_eq!(cache.tree.as_ref().unwrap().root_node().id(), tree_id); + assert!(cache.line(1).is_empty()); + cache.update(&source, Some(Path::new("b.txt")), 1); + assert_eq!(cache.language(), None); + assert!(cache.tree.is_none()); + assert!(cache.lines.iter().all(Vec::is_empty)); + cache.update(&source, Some(Path::new("b.rs")), 1); + assert_eq!(cache.parses, 2); + assert_eq!(cache.incremental_parses, 0); + } + + #[test] + fn unicode_minimal_edits_use_byte_points_and_never_split_codepoints() { + let old = "x\n\t\"Γ© πŸ‘©β€πŸ’»\"\n"; + let new = "x\n\t\"Γͺ πŸ‘©β€πŸ’»\"\n"; + let edit = minimal_edit(old, new); + assert_eq!(edit.start_byte, 4); + assert_eq!(edit.start_position, Point::new(1, 2)); + assert_eq!(edit.old_end_byte, 6); + assert_eq!(edit.new_end_byte, 6); + assert_eq!(edit.old_end_position, Point::new(1, 4)); + let edit = minimal_edit("a\nΞ²\n", "a\n\nΞ²\n"); + assert_eq!(edit.start_position, Point::new(1, 0)); + assert_eq!(edit.old_end_position, Point::new(1, 0)); + assert_eq!(edit.new_end_position, Point::new(2, 0)); + let edit = minimal_edit("Ξ±", ""); assert_eq!( - spans, - [ + (edit.start_byte, edit.old_end_byte, edit.new_end_byte), + (0, 2, 0) + ); + let edit = minimal_edit("xΞ±", "xΡ±"); + assert_eq!( + (edit.start_byte, edit.old_end_byte, edit.new_end_byte), + (1, 3, 3) + ); + assert_eq!(edit.new_end_position, Point::new(0, 3)); + } + + #[test] + fn incremental_unicode_newline_insert_remove_and_undo_match_fresh_trees() { + let original = + "fn main() {\n\tlet text = \"Γ© é πŸ‘©β€πŸ’»\";\n\t/* nested /* inner */ note */\n}\n"; + let mut cache = self::cache(original, "a.rs"); + let edits = [ + original.replace("Γ©", "Γͺ"), + original.replace("let text", "\nlet text"), + original.replace("\n\t/* nested /* inner */ note */", ""), + original.replace("πŸ‘©β€πŸ’»", "πŸ‘©β€πŸš€"), + original.replace("\";", "\";\n\tlet n = 42;"), + original.to_owned(), + String::new(), + original.to_owned(), + ]; + for (revision, source) in edits.iter().enumerate() { + cache.update(&lines(source), Some(Path::new("a.rs")), revision as u64 + 1); + assert_fresh(&cache, source, "a.rs"); + } + assert_eq!(cache.parses, edits.len() + 1); + assert_eq!(cache.incremental_parses, edits.len()); + } + + #[test] + fn unfinished_source_and_multiline_offscreen_state_remain_highlighted() { + let source = + "fn main() {\n/* start\ninside πŸ‘©β€πŸ’»\nend */\nlet n = 42;\nlet text = \"unfinished"; + let cache = self::cache(source, "a.rs"); + assert!(cache.tree.as_ref().unwrap().root_node().has_error()); + assert_token(&cache, source, 0, "fn", SyntaxKind::Keyword); + assert_token(&cache, source, 2, "inside", SyntaxKind::Comment); + assert_token(&cache, source, 4, "42", SyntaxKind::Number); + assert_eq!(cache.error(), None); + let source = "text = '''first\ninside\nlast'''\nanswer = 42"; + let cache = self::cache(source, "a.py"); + assert_token(&cache, source, 1, "inside", SyntaxKind::String); + assert_token(&cache, source, 3, "42", SyntaxKind::Number); + } + + #[test] + fn markdown_inline_and_fences_inject_known_languages_and_update_incrementally() { + let original = "A **bold** and `code` link.\n\n```rust\nfn main() { let n = 42; }\n```\n\n```python\ndef hello():\n return 7\n```\n\n```not-a-language\nfn not_rust() { 99; }\n```"; + let mut cache = self::cache(original, "a.md"); + assert_token(&cache, original, 0, "bold", SyntaxKind::Markup); + assert_token(&cache, original, 0, "code", SyntaxKind::Markup); + assert_token(&cache, original, 3, "fn", SyntaxKind::Keyword); + assert_token(&cache, original, 3, "main", SyntaxKind::Function); + assert_token(&cache, original, 3, "42", SyntaxKind::Number); + assert_token(&cache, original, 7, "def", SyntaxKind::Keyword); + assert_ne!(kind_at(&cache, 12, 0), Some(SyntaxKind::Keyword)); + let parses = cache.injection_parses; + let source = original.replace("42", "123"); + cache.update(&lines(&source), Some(Path::new("a.md")), 1); + assert_token(&cache, &source, 3, "123", SyntaxKind::Number); + assert_fresh(&cache, &source, "a.md"); + assert!(cache.injection_parses > parses); + let source = source.replace("```rust", "```unknown"); + cache.update(&lines(&source), Some(Path::new("a.md")), 2); + assert_ne!(kind_at(&cache, 3, 0), Some(SyntaxKind::Keyword)); + assert_fresh(&cache, &source, "a.md"); + } + + #[test] + fn markdown_injections_keep_anonymous_syntax_but_not_named_block_continuations() { + let source = + "> # Heading **bold**\n>\n> ```rust\n> fn main() {\n> let n = 3.25;\n> }\n> ```"; + let cache = self::cache(source, "a.md"); + assert_token(&cache, source, 0, "Heading", SyntaxKind::Markup); + assert_token(&cache, source, 0, "bold", SyntaxKind::Markup); + assert_token(&cache, source, 3, "main", SyntaxKind::Function); + assert_token(&cache, source, 4, "3.25", SyntaxKind::Number); + let rust = languages::by_name("rust").unwrap(); + let layer = cache + .injections + .iter() + .find(|layer| std::ptr::eq(layer.syntax, rust)) + .unwrap(); + for range in &layer.ranges { + assert!(!source[range.start_byte..range.end_byte].contains('>')); + } + let braces = [source.find('{').unwrap(), source.find('}').unwrap()]; + for byte in braces { + assert!( + layer + .ranges + .iter() + .any(|range| range.start_byte <= byte && byte < range.end_byte) + ); + } + assert_fresh(&cache, source, "a.md"); + } + + #[test] + fn html_script_style_injections_override_host_and_reuse_unchanged_trees() { + let original = "
hello
\n\n"; + let mut cache = self::cache(original, "a.html"); + assert_token(&cache, original, 0, "div", SyntaxKind::Markup); + assert_token(&cache, original, 1, "function", SyntaxKind::Keyword); + assert_token(&cache, original, 1, "hello", SyntaxKind::Function); + assert_token(&cache, original, 1, "42", SyntaxKind::Number); + assert_token(&cache, original, 2, "7", SyntaxKind::Number); + let injection_parses = cache.injection_parses; + let source = original.replace("
hello", "
world"); + cache.update(&lines(&source), Some(Path::new("a.html")), 1); + assert_eq!(cache.injection_parses, injection_parses); + assert_fresh(&cache, &source, "a.html"); + let source = source.replace("42", "123"); + cache.update(&lines(&source), Some(Path::new("a.html")), 2); + assert_token(&cache, &source, 1, "123", SyntaxKind::Number); + assert_fresh(&cache, &source, "a.html"); + // Inserting at an included range's exact end must reparse, not merely edit its old tree. + let source = source.replace("return 123; }", "return 123; } function added() {}"); + cache.update(&lines(&source), Some(Path::new("a.html")), 3); + assert_token(&cache, &source, 1, "added", SyntaxKind::Function); + assert_fresh(&cache, &source, "a.html"); + } + + #[test] + fn local_definitions_disable_builtin_captures_and_propagate_highlights() { + let source = "function hello(require) { require('local'); }\nrequire('global');"; + let cache = self::cache(source, "a.js"); + let local = source.find("require('local')").unwrap(); + assert_eq!(kind_at(&cache, 0, local), None); + assert_token(&cache, source, 1, "require", SyntaxKind::Function); + let source = "const action = () => 42;\naction;"; + let cache = self::cache(source, "a.js"); + assert_token(&cache, source, 0, "action", SyntaxKind::Function); + assert_token(&cache, source, 1, "action", SyntaxKind::Function); + } + + #[test] + fn overlapping_capture_specificity_priority_and_plain_injection_are_nonoverlapping() { + let source = "Ξ±Ξ²Ξ³"; + let mut spans = vec![Vec::new()]; + let mut events = Vec::new(); + let highlights = vec![ + Highlight { + start: 0, + end: 6, + kind: Some(SyntaxKind::String), + depth: 0, + priority: 0, + pattern: 99, + }, + Highlight { + start: 2, + end: 4, + kind: Some(SyntaxKind::Function), + depth: 0, + priority: 0, + pattern: 1, + }, + Highlight { + start: 4, + end: 6, + kind: None, + depth: 1, + priority: i32::MIN, + pattern: 0, + }, + Highlight { + start: 0, + end: 2, + kind: Some(SyntaxKind::Keyword), + depth: 0, + priority: 10, + pattern: 0, + }, + ]; + render_spans(source, &[0], &mut spans, &highlights, &mut events); + assert_eq!( + spans[0], + vec![ SyntaxSpan { start_byte: 0, end_byte: 2, - kind: SyntaxKind::String + kind: SyntaxKind::Keyword }, SyntaxSpan { start_byte: 2, end_byte: 4, - kind: SyntaxKind::Comment + kind: SyntaxKind::Function }, ] ); - let mut restored = scopes.clone(); - collect_spans( - 4, - &[(1, ScopeStackOp::Pop(2)), (1, ScopeStackOp::Restore)], - &mut restored, - &mut spans, - ) - .unwrap(); - assert_eq!( - spans, - [ - SyntaxSpan { - start_byte: 0, - end_byte: 1, - kind: SyntaxKind::Comment - }, - SyntaxSpan { - start_byte: 1, - end_byte: 4, - kind: SyntaxKind::String - }, - ] - ); - let before = restored.clone(); - collect_spans( - 3, - &[ - (1, ScopeStackOp::Clear(ClearAmount::TopN(1))), - ( - 1, - ScopeStackOp::Push(Scope::new("constant.numeric.python").unwrap()), - ), - (2, ScopeStackOp::Pop(1)), - (2, ScopeStackOp::Restore), - ], - &mut restored, - &mut spans, - ) - .unwrap(); - assert_eq!(restored, before); - assert_eq!(spans[1].kind, SyntaxKind::Number); - restored.push(Scope::new("keyword.operator.python").unwrap()); - assert_eq!(classify(&restored), Some(SyntaxKind::Keyword)); } #[test] - fn real_parser_errors_clear_the_entire_buffer_and_recover_on_an_edit() { - let path = Some(Path::new("sample.rs")); - let mut source = lines("fn main() {}\nlet value = 42;"); - let mut cache = SyntaxCache::new(); - cache.update(&source, path, 0); - assert!(!cache.line(0).is_empty()); - // Missing grammar contexts make Syntect return a genuine ParsingError, not a mock failure. - cache.parse_from(0, cache.syntax.unwrap(), &SyntaxSet::new()); - assert!(cache.error().unwrap().contains("Missing context")); - assert!(cache.error().unwrap().contains("line 1")); - assert!(cache.line(0).is_empty()); - assert!(cache.line(1).is_empty()); - cache.update(&source, path, 1); - assert!(cache.error().is_some()); - source[1] = "let value = 43;".into(); - cache.update(&source, path, 2); - assert_eq!(cache.error(), None); - assert_token(&cache, &source, 0, "fn", SyntaxKind::Keyword); - assert_token(&cache, &source, 1, "43", SyntaxKind::Number); - assert_fresh(&cache, &source, path); + fn local_initializer_visibility_positive_predicates_and_noninheriting_scopes() { + let syntax = languages::by_name("javascript").unwrap(); + let locals = r#" + (program) @local.scope + ((function_declaration) @local.scope (#set! local.scope-inherits false)) + (variable_declarator name: (identifier) @local.definition value: (_) @local.definition-value) + (identifier) @local.reference + "#; + let source_query = format!( + "{locals}\n{}", + r#" + ((identifier) @function (#is-not? local)) + ((identifier) @string (#is? local)) + (variable_declarator name: (identifier) @type) + "# + ); + let query = Query::new(&syntax.language, &source_query).unwrap(); + let highlights_start = (0..query.pattern_count()) + .filter(|pattern| query.start_byte_for_pattern(*pattern) < locals.len() + 1) + .count(); + let configuration = Configuration { + syntax, + query, + locals_start: 0, + highlights_start, + }; + let source = "const value = value;\nvalue;\nfunction inner() { value; }"; + let cache = self::cache(source, "a.js"); + let mut highlights = Vec::new(); + collect_layer( + &configuration, + cache.tree.as_ref().unwrap(), + source, + &[Range { + start_byte: 0, + end_byte: source.len(), + start_point: Point::new(0, 0), + end_point: point_at(source, source.len()), + }], + 0, + None, + &mut QueryCursor::new(), + &mut Vec::new(), + &mut highlights, + ) + .unwrap(); + let mut spans = vec![Vec::new(); 3]; + let starts = [ + 0, + source.find("\nvalue;").unwrap() + 1, + source.find("\nfunction").unwrap() + 1, + ]; + render_spans(source, &starts, &mut spans, &highlights, &mut Vec::new()); + let kind = |row: usize, byte: usize| { + spans[row].iter().find_map(|span| { + (span.start_byte <= byte && byte < span.end_byte).then_some(span.kind) + }) + }; + assert_eq!(kind(0, 6), Some(SyntaxKind::Type)); + assert_eq!(kind(0, 14), Some(SyntaxKind::Function)); + assert_eq!(kind(1, 0), Some(SyntaxKind::Type)); + assert_eq!(kind(2, 19), Some(SyntaxKind::Function)); + } + + #[test] + fn grammar_and_unsupported_predicate_errors_are_not_silent_empty_queries() { + static INVALID: LazyLock<[LanguageSpec; 2]> = LazyLock::new(|| { + let language = languages::by_name("rust").unwrap(); + [ + "(not_a_rust_node) @keyword", + "((identifier) @function (#unsupported? @function))", + ] + .map(|query| LanguageSpec { + name: "invalid test grammar query", + language: language.language.clone(), + highlights: query, + injections: "", + locals: "", + }) + }); + for syntax in INVALID.iter() { + assert!(compile_configuration(syntax).is_err()); + } + } + + #[test] + fn combined_tagged_templates_and_nested_markdown_html_keep_real_injections() { + let source = "const page = html`
${value}
`;"; + let cache = self::cache(source, "a.js"); + assert_token(&cache, source, 0, "div", SyntaxKind::Markup); + assert_token(&cache, source, 0, "inside", SyntaxKind::Function); + assert_token(&cache, source, 0, "42", SyntaxKind::Number); + assert_fresh(&cache, source, "a.js"); + let source = "```html\n\n```"; + let cache = self::cache(source, "a.md"); + assert_token(&cache, source, 1, "nested", SyntaxKind::Function); + assert_token(&cache, source, 1, "7", SyntaxKind::Number); + assert_fresh(&cache, source, "a.md"); + let source = "fn main() { println!(\"{}\", 42); }"; + let cache = self::cache(source, "a.rs"); + assert_token(&cache, source, 0, "42", SyntaxKind::Number); + assert!(cache.injections.len() <= 16); + assert_fresh(&cache, source, "a.rs"); + } + + #[test] + fn parser_state_is_isolated_between_buffers() { + let mut first = self::cache("/* open\ninside", "a.rs"); + let second = self::cache("fn main() { let n = 42; }", "b.rs"); + first.update( + &lines("/* open */\nfn other() {}"), + Some(Path::new("a.rs")), + 1, + ); + assert_token( + &second, + "fn main() { let n = 42; }", + 0, + "fn", + SyntaxKind::Keyword, + ); + assert_eq!(second.parses, 1); + assert_eq!(first.incremental_parses, 1); } } diff --git a/src/syntax/languages.rs b/src/syntax/languages.rs new file mode 100644 index 0000000..076a3f7 --- /dev/null +++ b/src/syntax/languages.rs @@ -0,0 +1,422 @@ +use std::borrow::Cow; +use std::path::Path; +use std::sync::LazyLock; + +pub(super) struct LanguageSpec { + pub name: &'static str, + pub language: tree_sitter::Language, + pub highlights: &'static str, + pub injections: &'static str, + pub locals: &'static str, +} + +// These grammars extend the upstream base queries, rather than replacing them. +static JS_HIGHLIGHTS: LazyLock = LazyLock::new(|| { + [ + tree_sitter_javascript::HIGHLIGHT_QUERY, + tree_sitter_javascript::JSX_HIGHLIGHT_QUERY, + ] + .join("\n") +}); +static TS_HIGHLIGHTS: LazyLock = LazyLock::new(|| { + [ + tree_sitter_javascript::HIGHLIGHT_QUERY, + tree_sitter_typescript::HIGHLIGHTS_QUERY, + ] + .join("\n") +}); +static TSX_HIGHLIGHTS: LazyLock = LazyLock::new(|| { + [ + JS_HIGHLIGHTS.as_str(), + tree_sitter_typescript::HIGHLIGHTS_QUERY, + ] + .join("\n") +}); +static TS_LOCALS: LazyLock = LazyLock::new(|| { + [ + tree_sitter_javascript::LOCALS_QUERY, + tree_sitter_typescript::LOCALS_QUERY, + ] + .join("\n") +}); +static CPP_HIGHLIGHTS: LazyLock = LazyLock::new(|| { + [ + tree_sitter_c::HIGHLIGHT_QUERY, + tree_sitter_cpp::HIGHLIGHT_QUERY, + ] + .join("\n") +}); + +// The MIT-licensed upstream C++ crate bundles, but does not export, injections.scm. +// This minimal AST query uses its raw-string delimiter/content nodes: +// https://github.com/tree-sitter/tree-sitter-cpp/blob/v0.23.4/queries/injections.scm +const CPP_INJECTIONS: &str = " +(raw_string_literal + delimiter: (raw_string_delimiter) @injection.language + (raw_string_content) @injection.content) +"; + +fn javascript_injections() -> &'static str { + // The trailing upstream hbs/@glimmer rule uses a nonstandard #offset! directive, + // not injection.* captures. Glimmer is outside the selected language set; + // retain all standard tagged-template, regex and JSDoc injection queries. + let query = tree_sitter_javascript::INJECTIONS_QUERY; + query + .split_once("\n; Parse Ember/") + .map_or(query, |(query, _)| query) +} + +fn spec( + name: &'static str, + language: tree_sitter::Language, + highlights: &'static str, + injections: &'static str, + locals: &'static str, +) -> LanguageSpec { + LanguageSpec { + name, + language, + highlights, + injections, + locals, + } +} + +static LANGUAGES: LazyLock<[LanguageSpec; 18]> = LazyLock::new(|| { + [ + spec( + "Rust", + tree_sitter_rust::LANGUAGE.into(), + tree_sitter_rust::HIGHLIGHTS_QUERY, + tree_sitter_rust::INJECTIONS_QUERY, + "", + ), + spec( + "Ruby", + tree_sitter_ruby::LANGUAGE.into(), + tree_sitter_ruby::HIGHLIGHTS_QUERY, + "", + tree_sitter_ruby::LOCALS_QUERY, + ), + spec( + "Python", + tree_sitter_python::LANGUAGE.into(), + tree_sitter_python::HIGHLIGHTS_QUERY, + "", + "", + ), + spec( + "JSON", + tree_sitter_json::LANGUAGE.into(), + tree_sitter_json::HIGHLIGHTS_QUERY, + "", + "", + ), + spec( + "Bash", + tree_sitter_bash::LANGUAGE.into(), + tree_sitter_bash::HIGHLIGHT_QUERY, + "", + "", + ), + spec( + "Makefile", + tree_sitter_make::LANGUAGE.into(), + tree_sitter_make::HIGHLIGHTS_QUERY, + "", + "", + ), + spec( + "Markdown", + tree_sitter_md::LANGUAGE.into(), + tree_sitter_md::HIGHLIGHT_QUERY_BLOCK, + tree_sitter_md::INJECTION_QUERY_BLOCK, + "", + ), + spec( + "JavaScript", + tree_sitter_javascript::LANGUAGE.into(), + JS_HIGHLIGHTS.as_str(), + javascript_injections(), + tree_sitter_javascript::LOCALS_QUERY, + ), + spec( + "TypeScript", + tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(), + TS_HIGHLIGHTS.as_str(), + javascript_injections(), + TS_LOCALS.as_str(), + ), + spec( + "HTML", + tree_sitter_html::LANGUAGE.into(), + tree_sitter_html::HIGHLIGHTS_QUERY, + tree_sitter_html::INJECTIONS_QUERY, + "", + ), + spec( + "CSS", + tree_sitter_css::LANGUAGE.into(), + tree_sitter_css::HIGHLIGHTS_QUERY, + "", + "", + ), + spec( + "YAML", + tree_sitter_yaml::LANGUAGE.into(), + tree_sitter_yaml::HIGHLIGHTS_QUERY, + "", + "", + ), + spec( + "TOML", + tree_sitter_toml_ng::LANGUAGE.into(), + tree_sitter_toml_ng::HIGHLIGHTS_QUERY, + "", + "", + ), + spec( + "C", + tree_sitter_c::LANGUAGE.into(), + tree_sitter_c::HIGHLIGHT_QUERY, + "", + "", + ), + spec( + "C++", + tree_sitter_cpp::LANGUAGE.into(), + CPP_HIGHLIGHTS.as_str(), + CPP_INJECTIONS, + "", + ), + spec( + "Go", + tree_sitter_go::LANGUAGE.into(), + tree_sitter_go::HIGHLIGHTS_QUERY, + "", + "", + ), + spec( + "TypeScript", + tree_sitter_typescript::LANGUAGE_TSX.into(), + TSX_HIGHLIGHTS.as_str(), + javascript_injections(), + TS_LOCALS.as_str(), + ), + spec( + "Markdown inline", + tree_sitter_md::INLINE_LANGUAGE.into(), + tree_sitter_md::HIGHLIGHT_QUERY_INLINE, + tree_sitter_md::INJECTION_QUERY_INLINE, + "", + ), + ] +}); + +pub(super) fn all() -> impl Iterator { + LANGUAGES.iter() +} + +pub(super) fn by_name(name: &str) -> Option<&'static LanguageSpec> { + let name = name.trim(); + if name.eq_ignore_ascii_case("tsx") { + return Some(&LANGUAGES[16]); + } + if name.eq_ignore_ascii_case("markdown_inline") || name.eq_ignore_ascii_case("markdown-inline") + { + return Some(&LANGUAGES[17]); + } + const ALIASES: &[(&str, &str)] = &[ + ("rs", "Rust"), + ("rb", "Ruby"), + ("py", "Python"), + ("sh", "Bash"), + ("shell", "Bash"), + ("shellscript", "Bash"), + ("make", "Makefile"), + ("mk", "Makefile"), + ("md", "Markdown"), + ("js", "JavaScript"), + ("jsx", "JavaScript"), + ("node", "JavaScript"), + ("ecmascript", "JavaScript"), + ("ts", "TypeScript"), + ("htm", "HTML"), + ("xhtml", "HTML"), + ("yml", "YAML"), + ("cc", "C++"), + ("cpp", "C++"), + ("cxx", "C++"), + ("golang", "Go"), + ]; + let name = ALIASES + .iter() + .find(|(alias, _)| alias.eq_ignore_ascii_case(name)) + .map_or(name, |(_, name)| *name); + all().find(|spec| spec.name.eq_ignore_ascii_case(name)) +} + +pub(super) fn detect( + path: Option<&Path>, + first_line: Option<&str>, +) -> Option<&'static LanguageSpec> { + path.and_then(detect_path) + .or_else(|| first_line.and_then(detect_shebang)) +} + +fn detect_path(path: &Path) -> Option<&'static LanguageSpec> { + let filename = path.file_name()?.to_str()?; + let language = match filename { + "Makefile" | "makefile" | "GNUmakefile" | "BSDmakefile" => "Makefile", + "Gemfile" | "Rakefile" | "Guardfile" | "Vagrantfile" | "Brewfile" | "Podfile" + | "Fastfile" | "Appfile" | "Berksfile" | ".irbrc" => "Ruby", + "SConstruct" | "SConscript" => "Python", + ".bashrc" | ".bash_profile" | ".bash_login" | ".bash_logout" | ".profile" => "Bash", + "Cargo.lock" | "Pipfile" | "poetry.lock" | "uv.lock" => "TOML", + _ => { + let extension = path.extension()?.to_str()?; + if matches!(extension, "C" | "H") { + return by_name("C++"); + } + let extension = if extension.bytes().any(|byte| byte.is_ascii_uppercase()) { + Cow::Owned(extension.to_ascii_lowercase()) + } else { + Cow::Borrowed(extension) + }; + match extension.as_ref() { + "rs" => "Rust", + "rb" | "rake" | "gemspec" => "Ruby", + "py" | "pyw" | "pyi" => "Python", + "json" => "JSON", + "sh" | "bash" => "Bash", + "mk" | "mak" | "make" => "Makefile", + "md" | "markdown" | "mdown" | "mkd" | "mdwn" => "Markdown", + "js" | "jsx" | "mjs" | "cjs" => "JavaScript", + "ts" | "mts" | "cts" => "TypeScript", + "tsx" => "tsx", + "html" | "htm" | "xhtml" => "HTML", + "css" => "CSS", + "yaml" | "yml" => "YAML", + "toml" => "TOML", + "c" | "h" => "C", + "cc" | "cpp" | "cxx" | "c++" | "hh" | "hpp" | "hxx" | "h++" | "ipp" | "inl" + | "tpp" => "C++", + "go" => "Go", + _ => return None, + } + } + }; + by_name(language) +} + +fn detect_shebang(line: &str) -> Option<&'static LanguageSpec> { + let line = line + .strip_prefix('\u{feff}') + .unwrap_or(line) + .strip_prefix("#!")?; + let mut words = line.split_whitespace(); + let mut program = words.next()?; + if Path::new(program).file_name()?.to_str()? == "env" { + program = loop { + let word = words.next()?; + if matches!(word, "-u" | "--unset" | "-C" | "--chdir" | "--argv0") { + words.next()?; + } else if !word.starts_with('-') && !word.contains('=') { + break word; + } + }; + } + let program = Path::new(program).file_name()?.to_str()?; + for (prefix, language) in [("python", "Python"), ("ruby", "Ruby")] { + if let Some(suffix) = program.strip_prefix(prefix) + && (suffix.is_empty() + || (suffix.starts_with(|c: char| c.is_ascii_digit()) + && suffix.chars().all(|c| c.is_ascii_digit() || c == '.'))) + { + return by_name(language); + } + } + match program { + "sh" | "bash" => by_name("Bash"), + "node" | "nodejs" => by_name("JavaScript"), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn selected_files_and_auxiliary_grammars_are_registered() { + assert_eq!(all().count(), 18); + assert!(all().all(|spec| !spec.highlights.is_empty())); + for (filename, language) in [ + ("main.rs", "Rust"), + ("Gemfile", "Ruby"), + ("main.py", "Python"), + ("package.json", "JSON"), + (".bashrc", "Bash"), + ("GNUmakefile", "Makefile"), + ("README.md", "Markdown"), + ("main.jsx", "JavaScript"), + ("main.ts", "TypeScript"), + ("index.html", "HTML"), + ("style.css", "CSS"), + ("config.yml", "YAML"), + ("Cargo.toml", "TOML"), + ("Cargo.lock", "TOML"), + ("main.c", "C"), + ("main.cpp", "C++"), + ("main.C", "C++"), + ("main.go", "Go"), + ] { + assert_eq!( + detect(Some(Path::new(filename)), None).map(|spec| spec.name), + Some(language) + ); + } + let ts = detect(Some(Path::new("main.ts")), None).unwrap(); + let tsx = detect(Some(Path::new("main.tsx")), None).unwrap(); + assert_eq!(ts.name, tsx.name); + assert!(!std::ptr::eq(ts, tsx)); + assert_ne!(ts.language, tsx.language); + assert!(std::ptr::eq(tsx, by_name("TSX").unwrap())); + assert!(by_name("markdown_inline").is_some()); + for filename in [ + "notes.txt", + "main.hs", + "main.scala", + "main.lua", + "Dockerfile", + ] { + assert!(detect(Some(Path::new(filename)), None).is_none()); + } + } + + #[test] + fn shebangs_support_bom_env_options_and_filename_precedence() { + for (line, language) in [ + ("#!/usr/bin/python3", "Python"), + ("\u{feff}#!/usr/bin/env -S python3.12 -u", "Python"), + ("#!/usr/bin/env -u PYTHONHOME FOO=bar python3", "Python"), + ("#!/usr/bin/ruby3.3", "Ruby"), + ("#!/bin/bash", "Bash"), + ("#! /usr/bin/env node", "JavaScript"), + ] { + assert_eq!( + detect(None, Some(line)).map(|spec| spec.name), + Some(language) + ); + } + assert_eq!( + detect(Some(Path::new("main.rs")), Some("#!/bin/bash")) + .unwrap() + .name, + "Rust" + ); + assert!(detect(None, Some("#!/usr/bin/python-not-real")).is_none()); + assert!(detect(None, Some("#!/usr/bin/perl")).is_none()); + assert!(detect(None, Some("plain text")).is_none()); + } +}