refactor: migrate syntax highlighting to Tree-sitter
This commit is contained in:
Valdemar Lorenzen
@@ -0,0 +1,422 @@
|
||||
use std::borrow::Cow;
|
||||
use std::path::Path;
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub(super) struct LanguageSpec {
|
||||
pub name: &'static str,
|
||||
pub language: tree_sitter::Language,
|
||||
pub highlights: &'static str,
|
||||
pub injections: &'static str,
|
||||
pub locals: &'static str,
|
||||
}
|
||||
|
||||
// These grammars extend the upstream base queries, rather than replacing them.
|
||||
static JS_HIGHLIGHTS: LazyLock<String> = LazyLock::new(|| {
|
||||
[
|
||||
tree_sitter_javascript::HIGHLIGHT_QUERY,
|
||||
tree_sitter_javascript::JSX_HIGHLIGHT_QUERY,
|
||||
]
|
||||
.join("\n")
|
||||
});
|
||||
static TS_HIGHLIGHTS: LazyLock<String> = LazyLock::new(|| {
|
||||
[
|
||||
tree_sitter_javascript::HIGHLIGHT_QUERY,
|
||||
tree_sitter_typescript::HIGHLIGHTS_QUERY,
|
||||
]
|
||||
.join("\n")
|
||||
});
|
||||
static TSX_HIGHLIGHTS: LazyLock<String> = LazyLock::new(|| {
|
||||
[
|
||||
JS_HIGHLIGHTS.as_str(),
|
||||
tree_sitter_typescript::HIGHLIGHTS_QUERY,
|
||||
]
|
||||
.join("\n")
|
||||
});
|
||||
static TS_LOCALS: LazyLock<String> = LazyLock::new(|| {
|
||||
[
|
||||
tree_sitter_javascript::LOCALS_QUERY,
|
||||
tree_sitter_typescript::LOCALS_QUERY,
|
||||
]
|
||||
.join("\n")
|
||||
});
|
||||
static CPP_HIGHLIGHTS: LazyLock<String> = LazyLock::new(|| {
|
||||
[
|
||||
tree_sitter_c::HIGHLIGHT_QUERY,
|
||||
tree_sitter_cpp::HIGHLIGHT_QUERY,
|
||||
]
|
||||
.join("\n")
|
||||
});
|
||||
|
||||
// The MIT-licensed upstream C++ crate bundles, but does not export, injections.scm.
|
||||
// This minimal AST query uses its raw-string delimiter/content nodes:
|
||||
// https://github.com/tree-sitter/tree-sitter-cpp/blob/v0.23.4/queries/injections.scm
|
||||
const CPP_INJECTIONS: &str = "
|
||||
(raw_string_literal
|
||||
delimiter: (raw_string_delimiter) @injection.language
|
||||
(raw_string_content) @injection.content)
|
||||
";
|
||||
|
||||
fn javascript_injections() -> &'static str {
|
||||
// The trailing upstream hbs/@glimmer rule uses a nonstandard #offset! directive,
|
||||
// not injection.* captures. Glimmer is outside the selected language set;
|
||||
// retain all standard tagged-template, regex and JSDoc injection queries.
|
||||
let query = tree_sitter_javascript::INJECTIONS_QUERY;
|
||||
query
|
||||
.split_once("\n; Parse Ember/")
|
||||
.map_or(query, |(query, _)| query)
|
||||
}
|
||||
|
||||
fn spec(
|
||||
name: &'static str,
|
||||
language: tree_sitter::Language,
|
||||
highlights: &'static str,
|
||||
injections: &'static str,
|
||||
locals: &'static str,
|
||||
) -> LanguageSpec {
|
||||
LanguageSpec {
|
||||
name,
|
||||
language,
|
||||
highlights,
|
||||
injections,
|
||||
locals,
|
||||
}
|
||||
}
|
||||
|
||||
static LANGUAGES: LazyLock<[LanguageSpec; 18]> = LazyLock::new(|| {
|
||||
[
|
||||
spec(
|
||||
"Rust",
|
||||
tree_sitter_rust::LANGUAGE.into(),
|
||||
tree_sitter_rust::HIGHLIGHTS_QUERY,
|
||||
tree_sitter_rust::INJECTIONS_QUERY,
|
||||
"",
|
||||
),
|
||||
spec(
|
||||
"Ruby",
|
||||
tree_sitter_ruby::LANGUAGE.into(),
|
||||
tree_sitter_ruby::HIGHLIGHTS_QUERY,
|
||||
"",
|
||||
tree_sitter_ruby::LOCALS_QUERY,
|
||||
),
|
||||
spec(
|
||||
"Python",
|
||||
tree_sitter_python::LANGUAGE.into(),
|
||||
tree_sitter_python::HIGHLIGHTS_QUERY,
|
||||
"",
|
||||
"",
|
||||
),
|
||||
spec(
|
||||
"JSON",
|
||||
tree_sitter_json::LANGUAGE.into(),
|
||||
tree_sitter_json::HIGHLIGHTS_QUERY,
|
||||
"",
|
||||
"",
|
||||
),
|
||||
spec(
|
||||
"Bash",
|
||||
tree_sitter_bash::LANGUAGE.into(),
|
||||
tree_sitter_bash::HIGHLIGHT_QUERY,
|
||||
"",
|
||||
"",
|
||||
),
|
||||
spec(
|
||||
"Makefile",
|
||||
tree_sitter_make::LANGUAGE.into(),
|
||||
tree_sitter_make::HIGHLIGHTS_QUERY,
|
||||
"",
|
||||
"",
|
||||
),
|
||||
spec(
|
||||
"Markdown",
|
||||
tree_sitter_md::LANGUAGE.into(),
|
||||
tree_sitter_md::HIGHLIGHT_QUERY_BLOCK,
|
||||
tree_sitter_md::INJECTION_QUERY_BLOCK,
|
||||
"",
|
||||
),
|
||||
spec(
|
||||
"JavaScript",
|
||||
tree_sitter_javascript::LANGUAGE.into(),
|
||||
JS_HIGHLIGHTS.as_str(),
|
||||
javascript_injections(),
|
||||
tree_sitter_javascript::LOCALS_QUERY,
|
||||
),
|
||||
spec(
|
||||
"TypeScript",
|
||||
tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
|
||||
TS_HIGHLIGHTS.as_str(),
|
||||
javascript_injections(),
|
||||
TS_LOCALS.as_str(),
|
||||
),
|
||||
spec(
|
||||
"HTML",
|
||||
tree_sitter_html::LANGUAGE.into(),
|
||||
tree_sitter_html::HIGHLIGHTS_QUERY,
|
||||
tree_sitter_html::INJECTIONS_QUERY,
|
||||
"",
|
||||
),
|
||||
spec(
|
||||
"CSS",
|
||||
tree_sitter_css::LANGUAGE.into(),
|
||||
tree_sitter_css::HIGHLIGHTS_QUERY,
|
||||
"",
|
||||
"",
|
||||
),
|
||||
spec(
|
||||
"YAML",
|
||||
tree_sitter_yaml::LANGUAGE.into(),
|
||||
tree_sitter_yaml::HIGHLIGHTS_QUERY,
|
||||
"",
|
||||
"",
|
||||
),
|
||||
spec(
|
||||
"TOML",
|
||||
tree_sitter_toml_ng::LANGUAGE.into(),
|
||||
tree_sitter_toml_ng::HIGHLIGHTS_QUERY,
|
||||
"",
|
||||
"",
|
||||
),
|
||||
spec(
|
||||
"C",
|
||||
tree_sitter_c::LANGUAGE.into(),
|
||||
tree_sitter_c::HIGHLIGHT_QUERY,
|
||||
"",
|
||||
"",
|
||||
),
|
||||
spec(
|
||||
"C++",
|
||||
tree_sitter_cpp::LANGUAGE.into(),
|
||||
CPP_HIGHLIGHTS.as_str(),
|
||||
CPP_INJECTIONS,
|
||||
"",
|
||||
),
|
||||
spec(
|
||||
"Go",
|
||||
tree_sitter_go::LANGUAGE.into(),
|
||||
tree_sitter_go::HIGHLIGHTS_QUERY,
|
||||
"",
|
||||
"",
|
||||
),
|
||||
spec(
|
||||
"TypeScript",
|
||||
tree_sitter_typescript::LANGUAGE_TSX.into(),
|
||||
TSX_HIGHLIGHTS.as_str(),
|
||||
javascript_injections(),
|
||||
TS_LOCALS.as_str(),
|
||||
),
|
||||
spec(
|
||||
"Markdown inline",
|
||||
tree_sitter_md::INLINE_LANGUAGE.into(),
|
||||
tree_sitter_md::HIGHLIGHT_QUERY_INLINE,
|
||||
tree_sitter_md::INJECTION_QUERY_INLINE,
|
||||
"",
|
||||
),
|
||||
]
|
||||
});
|
||||
|
||||
pub(super) fn all() -> impl Iterator<Item = &'static LanguageSpec> {
|
||||
LANGUAGES.iter()
|
||||
}
|
||||
|
||||
pub(super) fn by_name(name: &str) -> Option<&'static LanguageSpec> {
|
||||
let name = name.trim();
|
||||
if name.eq_ignore_ascii_case("tsx") {
|
||||
return Some(&LANGUAGES[16]);
|
||||
}
|
||||
if name.eq_ignore_ascii_case("markdown_inline") || name.eq_ignore_ascii_case("markdown-inline")
|
||||
{
|
||||
return Some(&LANGUAGES[17]);
|
||||
}
|
||||
const ALIASES: &[(&str, &str)] = &[
|
||||
("rs", "Rust"),
|
||||
("rb", "Ruby"),
|
||||
("py", "Python"),
|
||||
("sh", "Bash"),
|
||||
("shell", "Bash"),
|
||||
("shellscript", "Bash"),
|
||||
("make", "Makefile"),
|
||||
("mk", "Makefile"),
|
||||
("md", "Markdown"),
|
||||
("js", "JavaScript"),
|
||||
("jsx", "JavaScript"),
|
||||
("node", "JavaScript"),
|
||||
("ecmascript", "JavaScript"),
|
||||
("ts", "TypeScript"),
|
||||
("htm", "HTML"),
|
||||
("xhtml", "HTML"),
|
||||
("yml", "YAML"),
|
||||
("cc", "C++"),
|
||||
("cpp", "C++"),
|
||||
("cxx", "C++"),
|
||||
("golang", "Go"),
|
||||
];
|
||||
let name = ALIASES
|
||||
.iter()
|
||||
.find(|(alias, _)| alias.eq_ignore_ascii_case(name))
|
||||
.map_or(name, |(_, name)| *name);
|
||||
all().find(|spec| spec.name.eq_ignore_ascii_case(name))
|
||||
}
|
||||
|
||||
pub(super) fn detect(
|
||||
path: Option<&Path>,
|
||||
first_line: Option<&str>,
|
||||
) -> Option<&'static LanguageSpec> {
|
||||
path.and_then(detect_path)
|
||||
.or_else(|| first_line.and_then(detect_shebang))
|
||||
}
|
||||
|
||||
fn detect_path(path: &Path) -> Option<&'static LanguageSpec> {
|
||||
let filename = path.file_name()?.to_str()?;
|
||||
let language = match filename {
|
||||
"Makefile" | "makefile" | "GNUmakefile" | "BSDmakefile" => "Makefile",
|
||||
"Gemfile" | "Rakefile" | "Guardfile" | "Vagrantfile" | "Brewfile" | "Podfile"
|
||||
| "Fastfile" | "Appfile" | "Berksfile" | ".irbrc" => "Ruby",
|
||||
"SConstruct" | "SConscript" => "Python",
|
||||
".bashrc" | ".bash_profile" | ".bash_login" | ".bash_logout" | ".profile" => "Bash",
|
||||
"Cargo.lock" | "Pipfile" | "poetry.lock" | "uv.lock" => "TOML",
|
||||
_ => {
|
||||
let extension = path.extension()?.to_str()?;
|
||||
if matches!(extension, "C" | "H") {
|
||||
return by_name("C++");
|
||||
}
|
||||
let extension = if extension.bytes().any(|byte| byte.is_ascii_uppercase()) {
|
||||
Cow::Owned(extension.to_ascii_lowercase())
|
||||
} else {
|
||||
Cow::Borrowed(extension)
|
||||
};
|
||||
match extension.as_ref() {
|
||||
"rs" => "Rust",
|
||||
"rb" | "rake" | "gemspec" => "Ruby",
|
||||
"py" | "pyw" | "pyi" => "Python",
|
||||
"json" => "JSON",
|
||||
"sh" | "bash" => "Bash",
|
||||
"mk" | "mak" | "make" => "Makefile",
|
||||
"md" | "markdown" | "mdown" | "mkd" | "mdwn" => "Markdown",
|
||||
"js" | "jsx" | "mjs" | "cjs" => "JavaScript",
|
||||
"ts" | "mts" | "cts" => "TypeScript",
|
||||
"tsx" => "tsx",
|
||||
"html" | "htm" | "xhtml" => "HTML",
|
||||
"css" => "CSS",
|
||||
"yaml" | "yml" => "YAML",
|
||||
"toml" => "TOML",
|
||||
"c" | "h" => "C",
|
||||
"cc" | "cpp" | "cxx" | "c++" | "hh" | "hpp" | "hxx" | "h++" | "ipp" | "inl"
|
||||
| "tpp" => "C++",
|
||||
"go" => "Go",
|
||||
_ => return None,
|
||||
}
|
||||
}
|
||||
};
|
||||
by_name(language)
|
||||
}
|
||||
|
||||
fn detect_shebang(line: &str) -> Option<&'static LanguageSpec> {
|
||||
let line = line
|
||||
.strip_prefix('\u{feff}')
|
||||
.unwrap_or(line)
|
||||
.strip_prefix("#!")?;
|
||||
let mut words = line.split_whitespace();
|
||||
let mut program = words.next()?;
|
||||
if Path::new(program).file_name()?.to_str()? == "env" {
|
||||
program = loop {
|
||||
let word = words.next()?;
|
||||
if matches!(word, "-u" | "--unset" | "-C" | "--chdir" | "--argv0") {
|
||||
words.next()?;
|
||||
} else if !word.starts_with('-') && !word.contains('=') {
|
||||
break word;
|
||||
}
|
||||
};
|
||||
}
|
||||
let program = Path::new(program).file_name()?.to_str()?;
|
||||
for (prefix, language) in [("python", "Python"), ("ruby", "Ruby")] {
|
||||
if let Some(suffix) = program.strip_prefix(prefix)
|
||||
&& (suffix.is_empty()
|
||||
|| (suffix.starts_with(|c: char| c.is_ascii_digit())
|
||||
&& suffix.chars().all(|c| c.is_ascii_digit() || c == '.')))
|
||||
{
|
||||
return by_name(language);
|
||||
}
|
||||
}
|
||||
match program {
|
||||
"sh" | "bash" => by_name("Bash"),
|
||||
"node" | "nodejs" => by_name("JavaScript"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn selected_files_and_auxiliary_grammars_are_registered() {
|
||||
assert_eq!(all().count(), 18);
|
||||
assert!(all().all(|spec| !spec.highlights.is_empty()));
|
||||
for (filename, language) in [
|
||||
("main.rs", "Rust"),
|
||||
("Gemfile", "Ruby"),
|
||||
("main.py", "Python"),
|
||||
("package.json", "JSON"),
|
||||
(".bashrc", "Bash"),
|
||||
("GNUmakefile", "Makefile"),
|
||||
("README.md", "Markdown"),
|
||||
("main.jsx", "JavaScript"),
|
||||
("main.ts", "TypeScript"),
|
||||
("index.html", "HTML"),
|
||||
("style.css", "CSS"),
|
||||
("config.yml", "YAML"),
|
||||
("Cargo.toml", "TOML"),
|
||||
("Cargo.lock", "TOML"),
|
||||
("main.c", "C"),
|
||||
("main.cpp", "C++"),
|
||||
("main.C", "C++"),
|
||||
("main.go", "Go"),
|
||||
] {
|
||||
assert_eq!(
|
||||
detect(Some(Path::new(filename)), None).map(|spec| spec.name),
|
||||
Some(language)
|
||||
);
|
||||
}
|
||||
let ts = detect(Some(Path::new("main.ts")), None).unwrap();
|
||||
let tsx = detect(Some(Path::new("main.tsx")), None).unwrap();
|
||||
assert_eq!(ts.name, tsx.name);
|
||||
assert!(!std::ptr::eq(ts, tsx));
|
||||
assert_ne!(ts.language, tsx.language);
|
||||
assert!(std::ptr::eq(tsx, by_name("TSX").unwrap()));
|
||||
assert!(by_name("markdown_inline").is_some());
|
||||
for filename in [
|
||||
"notes.txt",
|
||||
"main.hs",
|
||||
"main.scala",
|
||||
"main.lua",
|
||||
"Dockerfile",
|
||||
] {
|
||||
assert!(detect(Some(Path::new(filename)), None).is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn shebangs_support_bom_env_options_and_filename_precedence() {
|
||||
for (line, language) in [
|
||||
("#!/usr/bin/python3", "Python"),
|
||||
("\u{feff}#!/usr/bin/env -S python3.12 -u", "Python"),
|
||||
("#!/usr/bin/env -u PYTHONHOME FOO=bar python3", "Python"),
|
||||
("#!/usr/bin/ruby3.3", "Ruby"),
|
||||
("#!/bin/bash", "Bash"),
|
||||
("#! /usr/bin/env node", "JavaScript"),
|
||||
] {
|
||||
assert_eq!(
|
||||
detect(None, Some(line)).map(|spec| spec.name),
|
||||
Some(language)
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
detect(Some(Path::new("main.rs")), Some("#!/bin/bash"))
|
||||
.unwrap()
|
||||
.name,
|
||||
"Rust"
|
||||
);
|
||||
assert!(detect(None, Some("#!/usr/bin/python-not-real")).is_none());
|
||||
assert!(detect(None, Some("#!/usr/bin/perl")).is_none());
|
||||
assert!(detect(None, Some("plain text")).is_none());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user