hl-valdemar/vamp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

feat: enable LSP semantic token highlighting by default

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-10-09 22:17:30 +02:00
parent 548711980c
commit d418d670ee
13 changed files with 1714 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/editor.rs
+252 -1
View File
@@ -1,6 +1,6 @@
use crate::clipboard::{ClipboardMode, SystemClipboard};
use crate::files::{DockSide, FilePicker, FilePresentation, FileView};
use crate::plugin::{Diagnostic, TextPosition};
use crate::plugin::{Diagnostic, SemanticToken, TextPosition};
use serde::Serialize;
use std::borrow::Cow;
use std::collections::{BTreeMap, HashMap, HashSet};
@@ -10,6 +10,7 @@ use std::io::{self, Write};
#[cfg(unix)]
use std::os::unix::fs::OpenOptionsExt;
use std::path::{Component, Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use unicode_segmentation::{GraphemeCursor, UnicodeSegmentation};
use unicode_width::UnicodeWidthStr;
@@ -773,6 +774,7 @@ struct Buffer {
text_revision: u64,
save_revision: u64,
diagnostics: BTreeMap<String, AnnotationSet>,
semantic_tokens: Option<Arc<[SemanticToken]>>,
targets: Vec<Target>,
primary: usize,
mode: Mode,
@@ -814,6 +816,7 @@ impl Buffer {
text_revision: 0,
save_revision: 0,
diagnostics: BTreeMap::new(),
semantic_tokens: None,
targets: vec![Target::caret(Cursor::default())],
primary: 0,
mode: Mode::Normal,
@@ -855,6 +858,7 @@ impl Buffer {
fn text_changed(&mut self) {
self.text_revision = self.text_revision.wrapping_add(1);
self.diagnostics.clear();
self.semantic_tokens = None;
}
}
@@ -1074,6 +1078,101 @@ impl Editor {
}
}
pub fn semantic_tokens(&self) -> &[SemanticToken] {
self.active.semantic_tokens.as_deref().unwrap_or_default()
}
pub(crate) fn validate_semantic_tokens(
&self,
buffer_id: usize,
revision: u64,
tokens: Option<&[SemanticToken]>,
) -> Result<bool, String> {
let Some(buffer) = self.buffer(buffer_id) else {
return Ok(false);
};
if buffer.text_revision != revision {
return Ok(false);
}
let Some(tokens) = tokens else {
return Ok(true);
};
if tokens.len() > MAX_EDIT_BYTES / std::mem::size_of::<SemanticToken>() {
return Err("Semantic tokens exceed 4 MiB".into());
}
let mut previous = None;
let mut row_length = None;
for token in tokens {
if previous.is_some_and(|start| token.start < start) {
return Err("Semantic token starts must be sorted".into());
}
if token.start.row != token.end.row || token.start >= token.end {
return Err("Semantic tokens must have nonempty single-line ranges".into());
}
if token.modifiers & !1023 != 0 {
return Err("Semantic token modifiers are not canonical".into());
}
if row_length.is_none_or(|(row, _)| row != token.start.row) {
let line = buffer
.lines
.get(token.start.row)
.ok_or("Semantic token position is outside the document")?;
row_length = Some((token.start.row, grapheme_count(line)));
}
if token.end.column > row_length.unwrap().1 {
return Err("Semantic token position is outside the document".into());
}
previous = Some(token.start);
}
Ok(true)
}
pub fn replace_semantic_tokens(
&mut self,
buffer_id: usize,
revision: u64,
tokens: Option<Arc<[SemanticToken]>>,
) -> Result<bool, String> {
if !self.validate_semantic_tokens(buffer_id, revision, tokens.as_deref())? {
return Ok(false);
}
Ok(self.replace_validated_semantic_tokens(buffer_id, revision, tokens))
}
/// Only session's fully prevalidated batch may use this storage path.
pub(crate) fn replace_validated_semantic_tokens(
&mut self,
buffer_id: usize,
revision: u64,
tokens: Option<Arc<[SemanticToken]>>,
) -> bool {
let buffer = if buffer_id == self.active.id {
&mut self.active
} else if let Some(buffer) = self.parked.get_mut(&buffer_id) {
buffer
} else {
return false;
};
if buffer.text_revision != revision {
return false;
}
let tokens = tokens.filter(|tokens| !tokens.is_empty());
match (&buffer.semantic_tokens, &tokens) {
(None, None) => return false,
(Some(current), Some(next)) if Arc::ptr_eq(current, next) => return false,
_ => {}
}
buffer.semantic_tokens = tokens;
true
}
pub fn clear_semantic_tokens(&mut self) {
self.active.semantic_tokens = None;
for buffer in self.parked.values_mut() {
buffer.semantic_tokens = None;
}
}
pub fn diagnostics(&self) -> impl Iterator<Item = &Diagnostic> {
self.active
.diagnostics
@@ -2754,6 +2853,9 @@ impl Editor {
Ok(()) => {
self.active.saved_lines.clone_from(&self.active.lines);
self.active.save_revision = self.active.save_revision.wrapping_add(1);
if self.active.path.as_ref() != Some(&path) {
self.active.semantic_tokens = None;
}
self.active.path = Some(path);
self.active.identity = Some(identity);
self.set_message("Written");
@@ -4316,6 +4418,155 @@ mod tests {
editor
}
#[test]
fn semantic_snapshots_are_borrowed_buffer_local_and_invalidated_by_text_and_path() {
use crate::plugin::SemanticTokenKind;
let directory = Directory::new();
let a = directory.path("a.py");
let b = directory.path("b.py");
fs::write(&a, "e\u{301}界").unwrap();
fs::write(&b, "other").unwrap();
let mut editor = Editor::open(Some(a.clone()), vec![]).unwrap();
editor.set_clipboard_mode("separate").unwrap();
let tokens: Arc<[SemanticToken]> = Arc::from([SemanticToken {
start: TextPosition { row: 0, column: 0 },
end: TextPosition { row: 0, column: 2 },
kind: SemanticTokenKind::Variable,
modifiers: 1023,
}]);
assert!(
editor
.replace_semantic_tokens(1, 0, Some(tokens.clone()))
.unwrap()
);
assert!(std::ptr::eq(editor.semantic_tokens(), tokens.as_ref()));
assert!(!editor.dirty());
assert!(editor.active.undo.is_empty());
editor.execute(&format!("edit {}", b.display()));
assert!(editor.semantic_tokens().is_empty());
assert!(
editor
.replace_semantic_tokens(2, 0, Some(tokens.clone()))
.unwrap()
);
editor.execute("buffer 1");
assert!(std::ptr::eq(editor.semantic_tokens(), tokens.as_ref()));
assert!(
!editor
.replace_semantic_tokens(1, 0, Some(tokens.clone()))
.unwrap()
);
editor.execute("w");
assert_eq!(editor.semantic_tokens(), tokens.as_ref());
editor.execute(&format!("w {}", directory.path("renamed.py").display()));
assert_eq!(editor.text_revision(), 0);
assert!(editor.semantic_tokens().is_empty());
assert!(
editor
.replace_semantic_tokens(1, 0, Some(tokens.clone()))
.unwrap()
);
editor.handle(Input::Char('x'));
assert!(editor.semantic_tokens().is_empty());
editor.handle(Input::Char('u'));
assert_eq!(editor.lines(), ["e\u{301}界"]);
assert!(editor.semantic_tokens().is_empty());
assert!(
!editor
.replace_semantic_tokens(1, 0, Some(tokens.clone()))
.unwrap()
);
editor.execute("buffer 2");
assert_eq!(editor.semantic_tokens(), tokens.as_ref());
editor.clear_semantic_tokens();
assert!(editor.semantic_tokens().is_empty());
editor.execute("buffer 1");
assert!(editor.semantic_tokens().is_empty());
}
#[test]
fn semantic_validation_rejects_malformed_live_tokens_but_ignores_stale_and_unloaded() {
use crate::plugin::SemanticTokenKind;
let mut editor = editor();
editor
.apply(vec![Effect::Replace("e\u{301}界\nx".into())])
.unwrap();
let revision = editor.text_revision();
let token = SemanticToken {
start: TextPosition { row: 0, column: 0 },
end: TextPosition { row: 0, column: 2 },
kind: SemanticTokenKind::Variable,
modifiers: 0,
};
let invalid = [
SemanticToken {
end: TextPosition { row: 0, column: 3 },
..token
},
SemanticToken {
end: token.start,
..token
},
SemanticToken {
end: TextPosition { row: 1, column: 1 },
..token
},
SemanticToken {
start: TextPosition { row: 9, column: 0 },
end: TextPosition { row: 9, column: 1 },
..token
},
SemanticToken {
modifiers: 1024,
..token
},
];
for token in invalid {
assert!(
editor
.validate_semantic_tokens(1, revision, Some(&[token]))
.is_err()
);
assert!(
!editor
.validate_semantic_tokens(1, revision - 1, Some(&[token]))
.unwrap()
);
assert!(
!editor
.validate_semantic_tokens(999, revision, Some(&[token]))
.unwrap()
);
}
let later = SemanticToken {
start: TextPosition { row: 0, column: 1 },
..token
};
assert!(
editor
.validate_semantic_tokens(1, revision, Some(&[later, token]))
.is_err()
);
// Raw adjacent LSP tokens may map outward to the same grapheme.
assert!(
editor
.validate_semantic_tokens(1, revision, Some(&[token, token, later]))
.unwrap()
);
let oversized = vec![token; MAX_EDIT_BYTES / std::mem::size_of::<SemanticToken>() + 1];
assert!(
editor
.validate_semantic_tokens(1, revision, Some(&oversized))
.is_err()
);
let tokens: Arc<[SemanticToken]> = Arc::from([token]);
editor
.replace_semantic_tokens(1, revision, Some(tokens))
.unwrap();
assert!(editor.replace_semantic_tokens(1, revision, None).unwrap());
assert!(editor.semantic_tokens().is_empty());
}
#[test]
fn hover_scroll_and_escape_preserve_carets_and_edits_dismiss_normally() {
let mut editor = editor();
src/lsp.rs
+522 -9
View File
@@ -19,7 +19,8 @@ use crate::{
editor::Context,
plugin::{
Diagnostic, DiagnosticSeverity, DocumentEvent, DocumentEventKind, DocumentSnapshot,
PluginUpdate, TextPosition, UpdateEffect,
PluginUpdate, SemanticToken, SemanticTokenKind, SemanticTokenModifier, TextPosition,
UpdateEffect,
},
};
@@ -49,6 +50,7 @@ struct CachedDocument {
snapshot: Arc<DocumentSnapshot>,
servers: Vec<ServerKey>,
diagnostics: HashMap<usize, Contribution>,
semantic_tokens: Option<Arc<[SemanticToken]>>,
}
struct Contribution {
@@ -62,6 +64,10 @@ enum Output {
snapshot: Arc<DocumentSnapshot>,
items: Vec<Diagnostic>,
},
SemanticTokens {
snapshot: Arc<DocumentSnapshot>,
tokens: Option<Arc<[SemanticToken]>>,
},
}
struct Worker {
@@ -78,6 +84,7 @@ enum Input {
snapshot: Arc<DocumentSnapshot>,
event: DocumentEventKind,
served: bool,
primary: bool,
},
Position(PositionKind, Source),
Diagnostics(usize),
@@ -115,6 +122,7 @@ pub struct Lsp {
servers: BTreeMap<ServerKey, Worker>,
updates: Vec<PluginUpdate>,
changed_diagnostics: HashSet<usize>,
changed_semantics: HashSet<usize>,
}
impl Lsp {
@@ -126,6 +134,7 @@ impl Lsp {
servers: BTreeMap::new(),
updates: Vec::new(),
changed_diagnostics: HashSet::new(),
changed_semantics: HashSet::new(),
}
}
@@ -152,6 +161,7 @@ impl Lsp {
if matches!(event.event, DocumentEventKind::Close) {
self.documents.remove(&snapshot.id);
self.changed_diagnostics.remove(&snapshot.id);
self.changed_semantics.remove(&snapshot.id);
} else if self.documents.get(&snapshot.id).is_none_or(|cached| {
cached.snapshot.path != snapshot.path || cached.snapshot.root != snapshot.root
}) {
@@ -193,9 +203,11 @@ impl Lsp {
snapshot: Arc::clone(&snapshot),
servers: keys,
diagnostics: HashMap::new(),
semantic_tokens: None,
},
);
self.changed_diagnostics.insert(snapshot.id);
self.changed_semantics.insert(snapshot.id);
for index in 0..self.documents[&snapshot.id].servers.len() {
let key = self.documents[&snapshot.id].servers[index].clone();
if let Err(error) = self.start(&key) {
@@ -207,6 +219,8 @@ impl Lsp {
if !Arc::ptr_eq(&cached.snapshot, &snapshot) {
cached.snapshot = Arc::clone(&snapshot);
cached.diagnostics.clear();
cached.semantic_tokens = None;
self.changed_semantics.insert(snapshot.id);
self.changed_diagnostics.insert(snapshot.id);
}
}
@@ -224,6 +238,10 @@ impl Lsp {
.documents
.get(&snapshot.id)
.is_some_and(|doc| doc.servers.contains(server_key)),
primary: self
.documents
.get(&snapshot.id)
.is_some_and(|doc| doc.servers.first() == Some(server_key)),
})
.is_err()
{
@@ -330,6 +348,7 @@ impl Lsp {
worker,
&mut self.documents,
&mut self.changed_diagnostics,
&mut self.changed_semantics,
&mut self.updates,
"LSP server stopped: combined diagnostic messages exceed 4 MiB"
.into(),
@@ -344,10 +363,20 @@ impl Lsp {
}
self.changed_diagnostics.insert(snapshot.id);
}
Output::SemanticTokens { snapshot, tokens } => {
let Some(doc) = self.documents.get_mut(&snapshot.id).filter(|doc| {
doc.servers.first() == Some(key)
&& Arc::ptr_eq(&doc.snapshot, &snapshot)
}) else {
continue;
};
doc.semantic_tokens = tokens;
self.changed_semantics.insert(snapshot.id);
}
}
}
}
let effects: Vec<_> = self
let mut effects: Vec<_> = self
.changed_diagnostics
.drain()
.filter_map(|id| {
@@ -370,6 +399,14 @@ impl Lsp {
})
})
.collect();
effects.extend(self.changed_semantics.drain().filter_map(|id| {
let doc = self.documents.get(&id)?;
Some(UpdateEffect::SemanticTokens {
buffer_id: id,
revision: doc.snapshot.revision,
tokens: doc.semantic_tokens.clone(),
})
}));
if !effects.is_empty() {
self.updates.push(PluginUpdate {
request_id: None,
@@ -396,7 +433,13 @@ impl Lsp {
let initial: Vec<_> = self
.documents
.values()
.map(|doc| (Arc::clone(&doc.snapshot), doc.servers.contains(key)))
.map(|doc| {
(
Arc::clone(&doc.snapshot),
doc.servers.contains(key),
doc.servers.first() == Some(key),
)
})
.collect();
let started = thread::Builder::new()
.name(format!("lsp-{}", config.language_id))
@@ -415,8 +458,8 @@ impl Lsp {
};
let result = initial
.into_iter()
.try_for_each(|(snapshot, served)| {
runtime.document(snapshot, DocumentEventKind::Open, served)
.try_for_each(|(snapshot, served, primary)| {
runtime.document(snapshot, DocumentEventKind::Open, served, primary)
})
.and_then(|()| runtime.run());
if let Err(reason) = result {
@@ -483,6 +526,7 @@ impl Lsp {
worker,
&mut self.documents,
&mut self.changed_diagnostics,
&mut self.changed_semantics,
&mut self.updates,
reason,
);
@@ -493,6 +537,7 @@ impl Lsp {
worker: &mut Worker,
documents: &mut HashMap<usize, CachedDocument>,
changed: &mut HashSet<usize>,
changed_semantics: &mut HashSet<usize>,
updates: &mut Vec<PluginUpdate>,
reason: String,
) {
@@ -502,6 +547,10 @@ impl Lsp {
worker.retired = true;
worker.cancelled.store(true, Ordering::Release);
for (id, doc) in documents {
if doc.servers.first() == Some(key) {
doc.semantic_tokens = None;
changed_semantics.insert(*id);
}
if doc.servers.contains(key) {
doc.diagnostics.remove(&key.config);
changed.insert(*id);
@@ -535,6 +584,8 @@ struct Document {
version: u64,
opened: bool,
saved: bool,
primary: bool,
semantic_dirty: bool,
}
#[derive(Default)]
@@ -549,6 +600,7 @@ enum Request {
Initialize,
Definition(Source),
Hover(Source),
SemanticTokens(Arc<DocumentSnapshot>),
Shutdown,
}
@@ -563,10 +615,12 @@ struct Runtime {
documents: HashMap<usize, Document>,
requests: HashMap<u64, Request>,
pending: VecDeque<(PositionKind, Source)>,
semantic_inflight: HashSet<usize>,
next_id: u64,
ready: bool,
definition: bool,
hover: bool,
semantic: Option<SemanticOptions>,
sync: SyncOptions,
}
@@ -590,10 +644,12 @@ impl Runtime {
documents: HashMap::new(),
requests: HashMap::new(),
pending: VecDeque::new(),
semantic_inflight: HashSet::new(),
next_id: 1,
ready: false,
definition: false,
hover: false,
semantic: None,
sync: SyncOptions::default(),
};
runtime.request(
@@ -606,14 +662,26 @@ impl Runtime {
"initializationOptions": runtime.config.initialization_options,
"capabilities": {
"general": {"positionEncodings": ["utf-16"]},
"workspace": {"configuration": true, "workspaceFolders": true},
"workspace": {"configuration": true, "workspaceFolders": true,
"semanticTokens": {"refreshSupport": true}},
"textDocument": {
"synchronization": {"dynamicRegistration": false, "willSave": false,
"willSaveWaitUntil": false, "didSave": true},
"definition": {"dynamicRegistration": false, "linkSupport": true},
"hover": {"dynamicRegistration": false,
"contentFormat": ["plaintext", "markdown"]},
"publishDiagnostics": {"relatedInformation": false, "versionSupport": true}
"publishDiagnostics": {"relatedInformation": false, "versionSupport": true},
"semanticTokens": {
"dynamicRegistration": false,
"requests": {"range": true, "full": true},
"tokenTypes": SEMANTIC_TYPES.iter().map(|(name, _)| *name).collect::<Vec<_>>(),
"tokenModifiers": SEMANTIC_MODIFIERS.iter().map(|(name, _)| *name).collect::<Vec<_>>(),
"formats": ["relative"],
"overlappingTokenSupport": false,
"multilineTokenSupport": false,
"serverCancelSupport": false,
"augmentsSyntaxTokens": true
}
}
}
})),
@@ -630,7 +698,8 @@ impl Runtime {
snapshot,
event,
served,
}) => self.document(snapshot, event, served)?,
primary,
}) => self.document(snapshot, event, served, primary)?,
Ok(Input::Position(kind, source)) => self.position(kind, source)?,
Ok(Input::Diagnostics(id)) => {
let text = if !self.documents.contains_key(&id) {
@@ -661,6 +730,19 @@ impl Runtime {
};
self.incoming(frame)?;
}
if self.ready && self.semantic.is_some() && self.requests.len() < QUEUE_SIZE {
let pending: Vec<_> = self
.documents
.iter()
.filter(|(id, doc)| {
doc.primary && doc.semantic_dirty && !self.semantic_inflight.contains(*id)
})
.map(|(id, _)| *id)
.collect();
for id in pending {
self.semantic_request(id)?;
}
}
self.transport.check_alive()?;
thread::sleep(TICK);
}
@@ -683,6 +765,17 @@ impl Runtime {
})
}
fn emit_semantics(
&self,
snapshot: &Arc<DocumentSnapshot>,
tokens: Option<Arc<[SemanticToken]>>,
) -> Result<(), String> {
self.send(Output::SemanticTokens {
snapshot: Arc::clone(snapshot),
tokens,
})
}
fn send(&self, output: Output) -> Result<(), String> {
self.output.try_send(output).map_err(|error| match error {
TrySendError::Full(_) => "LSP incoming update queue is full".into(),
@@ -695,6 +788,7 @@ impl Runtime {
snapshot: Arc<DocumentSnapshot>,
event: DocumentEventKind,
served: bool,
primary: bool,
) -> Result<(), String> {
let id = snapshot.id;
let old = self.documents.remove(&id);
@@ -733,11 +827,16 @@ impl Runtime {
return Err("LSP document version exhausted".into());
}
let mut doc = Document {
snapshot,
snapshot: Arc::clone(&snapshot),
uri,
version,
opened: same_uri && old.as_ref().is_some_and(|old| old.opened),
saved: matches!(event, DocumentEventKind::Save),
primary,
semantic_dirty: primary
&& old.as_ref().is_none_or(|old| {
old.semantic_dirty || !old.primary || !Arc::ptr_eq(&old.snapshot, &snapshot)
}),
};
if changed {
self.emit_diagnostics(&doc.snapshot, Vec::new())?;
@@ -753,6 +852,7 @@ impl Runtime {
}
}
self.documents.insert(id, doc);
self.semantic_request(id)?;
Ok(())
}
@@ -857,6 +957,60 @@ impl Runtime {
self.request(method, Some(params), request)
}
fn semantic_request(&mut self, id: usize) -> Result<(), String> {
let Some(options) = &self.semantic else {
return Ok(());
};
let Some(doc) = self.documents.get(&id).filter(|doc| {
self.ready && doc.primary && doc.semantic_dirty && !self.semantic_inflight.contains(&id)
}) else {
return Ok(());
};
if self.requests.len() == QUEUE_SIZE {
return Ok(());
}
let snapshot = Arc::clone(&doc.snapshot);
let mut params = json!({"textDocument": {"uri": doc.uri}});
let method = if options.full {
"textDocument/semanticTokens/full"
} else {
let text = normalized(&snapshot.text);
let (row, line) = text.split('\n').enumerate().last().unwrap();
params["range"] = json!({"start": {"line": 0, "character": 0},
"end": {"line": row, "character": line.encode_utf16().count()}});
"textDocument/semanticTokens/range"
};
self.request(method, Some(params), Request::SemanticTokens(snapshot))?;
self.semantic_inflight.insert(id);
self.documents.get_mut(&id).unwrap().semantic_dirty = false;
Ok(())
}
fn semantic_result(
&mut self,
snapshot: Arc<DocumentSnapshot>,
result: Result<&Value, String>,
) -> Result<(), String> {
self.semantic_inflight.remove(&snapshot.id);
let fresh = self.documents.get(&snapshot.id).is_some_and(|doc| {
doc.primary && !doc.semantic_dirty && Arc::ptr_eq(&doc.snapshot, &snapshot)
});
// Stale responses are discarded before inspecting their token payload.
if fresh {
let decoded = result.and_then(|result| {
decode_semantic_tokens(&snapshot.text, result, self.semantic.as_ref().unwrap())
});
match decoded {
Ok(tokens) => self.emit_semantics(&snapshot, tokens)?,
Err(reason) => {
self.emit_semantics(&snapshot, None)?;
self.emit(message(&format!("semantic tokens failed: {reason}"), None))?;
}
}
}
self.semantic_request(snapshot.id)
}
fn request(
&mut self,
method: &str,
@@ -920,6 +1074,9 @@ impl Runtime {
Request::Hover(source) => {
self.hover_result(source, Err("unexpected LSP response".into()))
}
Request::SemanticTokens(snapshot) => {
self.semantic_result(snapshot, Err("unexpected LSP response".into()))
}
_ => Err("unexpected LSP response".into()),
};
}
@@ -932,6 +1089,9 @@ impl Runtime {
Request::Hover(source) => {
self.hover_result(source, Err("invalid LSP error response".into()))
}
Request::SemanticTokens(snapshot) => {
self.semantic_result(snapshot, Err("invalid LSP error response".into()))
}
_ => Err("invalid LSP error response".into()),
};
};
@@ -942,6 +1102,9 @@ impl Runtime {
Some(source.request_id),
)),
Request::Hover(source) => self.hover_result(source, Err(text.to_owned())),
Request::SemanticTokens(snapshot) => {
self.semantic_result(snapshot, Err(text.to_owned()))
}
Request::Shutdown => Ok(()),
}
} else {
@@ -949,6 +1112,9 @@ impl Runtime {
Request::Initialize => self.initialized(&frame["result"]),
Request::Definition(source) => self.definition_result(source, &frame["result"]),
Request::Hover(source) => self.hover_result(source, Ok(&frame["result"])),
Request::SemanticTokens(snapshot) => {
self.semantic_result(snapshot, Ok(&frame["result"]))
}
Request::Shutdown => self.notify("exit", None),
}
}
@@ -972,6 +1138,12 @@ impl Runtime {
self.hover = capabilities
.get("hoverProvider")
.is_some_and(|value| value == true || value.is_object());
match semantic_options(capabilities.get("semanticTokensProvider")) {
Ok(options) => self.semantic = options,
Err(reason) => {
self.emit(message(&format!("semantic tokens failed: {reason}"), None))?
}
}
self.notify("initialized", Some(json!({})))?;
self.notify(
"workspace/didChangeConfiguration",
@@ -985,6 +1157,7 @@ impl Runtime {
self.save(&mut doc)?;
}
self.documents.insert(id, doc);
self.semantic_request(id)?;
}
while let Some((kind, source)) = self.pending.pop_front() {
self.position(kind, source)?;
@@ -1032,6 +1205,24 @@ impl Runtime {
}
"workspace/workspaceFolders" => self.workspace_folders()?,
"window/showMessageRequest" => Value::Null,
"workspace/semanticTokens/refresh" => {
self.transport
.send(json!({"jsonrpc": "2.0", "id": id, "result": null}))?;
if self.semantic.is_some() {
let ids: Vec<_> = self
.documents
.iter()
.filter(|(_, doc)| doc.primary)
.map(|(id, _)| *id)
.collect();
for id in ids {
let doc = self.documents.get_mut(&id).unwrap();
doc.semantic_dirty = true;
self.semantic_request(id)?;
}
}
return Ok(());
}
_ => {
response.as_object_mut().unwrap().remove("result");
response["error"] =
@@ -1252,6 +1443,267 @@ impl Runtime {
}
}
const SEMANTIC_TYPES: &[(&str, SemanticTokenKind)] = &[
("namespace", SemanticTokenKind::Namespace),
("type", SemanticTokenKind::Type),
("class", SemanticTokenKind::Class),
("enum", SemanticTokenKind::Enum),
("interface", SemanticTokenKind::Interface),
("struct", SemanticTokenKind::Struct),
("typeParameter", SemanticTokenKind::TypeParameter),
("parameter", SemanticTokenKind::Parameter),
("variable", SemanticTokenKind::Variable),
("property", SemanticTokenKind::Property),
("enumMember", SemanticTokenKind::EnumMember),
("event", SemanticTokenKind::Event),
("function", SemanticTokenKind::Function),
("method", SemanticTokenKind::Method),
("macro", SemanticTokenKind::Macro),
("keyword", SemanticTokenKind::Keyword),
("modifier", SemanticTokenKind::Modifier),
("comment", SemanticTokenKind::Comment),
("string", SemanticTokenKind::String),
("number", SemanticTokenKind::Number),
("regexp", SemanticTokenKind::Regexp),
("operator", SemanticTokenKind::Operator),
("decorator", SemanticTokenKind::Decorator),
];
const SEMANTIC_MODIFIERS: &[(&str, SemanticTokenModifier)] = &[
("declaration", SemanticTokenModifier::Declaration),
("definition", SemanticTokenModifier::Definition),
("readonly", SemanticTokenModifier::Readonly),
("static", SemanticTokenModifier::Static),
("deprecated", SemanticTokenModifier::Deprecated),
("abstract", SemanticTokenModifier::Abstract),
("async", SemanticTokenModifier::Async),
("modification", SemanticTokenModifier::Modification),
("documentation", SemanticTokenModifier::Documentation),
("defaultLibrary", SemanticTokenModifier::DefaultLibrary),
];
struct SemanticOptions {
full: bool,
types: Vec<Option<SemanticTokenKind>>,
modifiers: [u16; 31],
modifier_mask: u32,
}
fn semantic_options(value: Option<&Value>) -> Result<Option<SemanticOptions>, String> {
let Some(value) = value.filter(|value| !value.is_null() && *value != false) else {
return Ok(None);
};
let object = value
.as_object()
.ok_or("invalid semantic tokens capability")?;
let supported = |name| -> Result<bool, String> {
match object.get(name) {
None | Some(Value::Bool(false)) => Ok(false),
Some(Value::Bool(true) | Value::Object(_)) => Ok(true),
_ => Err("invalid semantic tokens request capability".into()),
}
};
let full = supported("full")?;
if !full && !supported("range")? {
return Ok(None);
}
let types = value["legend"]["tokenTypes"]
.as_array()
.filter(|items| items.len() <= 65536)
.ok_or("invalid semantic token type legend")?;
let names = value["legend"]["tokenModifiers"]
.as_array()
.ok_or("invalid semantic token modifier legend")?;
let types = types
.iter()
.map(|value| {
let name = value.as_str().ok_or("invalid semantic token type legend")?;
Ok(SEMANTIC_TYPES
.iter()
.find(|(candidate, _)| *candidate == name)
.map(|(_, kind)| *kind))
})
.collect::<Result<Vec<_>, String>>()?;
let mut modifiers = [0; 31];
for (index, value) in names.iter().enumerate() {
let name = value
.as_str()
.ok_or("invalid semantic token modifier legend")?;
if let Some(slot) = modifiers.get_mut(index) {
*slot = SEMANTIC_MODIFIERS
.iter()
.find(|(candidate, _)| *candidate == name)
.map_or(0, |(_, modifier)| *modifier as u16);
}
}
Ok(Some(SemanticOptions {
full,
types,
modifiers,
modifier_mask: (1u32 << names.len().min(31)) - 1,
}))
}
// Both cursors only advance: strict codepoint boundaries and outward grapheme
// rounding cost O(document + tokens), including many tokens in one grapheme.
struct SemanticLine<'a> {
chars: std::str::Chars<'a>,
graphemes: unicode_segmentation::Graphemes<'a>,
units: usize,
bytes: usize,
column: usize,
grapheme_start: usize,
grapheme_end: usize,
}
impl<'a> SemanticLine<'a> {
fn new(line: &'a str) -> Self {
let mut graphemes = line.graphemes(true);
let grapheme_end = graphemes.next().map_or(0, str::len);
Self {
chars: line.chars(),
graphemes,
units: 0,
bytes: 0,
column: 0,
grapheme_start: 0,
grapheme_end,
}
}
fn column(&mut self, target: usize, end: bool) -> Result<usize, String> {
while self.units < target {
let Some(character) = self.chars.next() else {
if end {
break; // LSP: an overlong end truncates at EOL, never wraps.
}
return Err("semantic token start is beyond its line".into());
};
let next = self.units + character.len_utf16();
if next > target {
return Err("semantic token splits a UTF-16 surrogate pair".into());
}
self.units = next;
self.bytes += character.len_utf8();
}
while self.grapheme_end > self.grapheme_start && self.bytes >= self.grapheme_end {
self.grapheme_start = self.grapheme_end;
self.column += 1;
self.grapheme_end += self.graphemes.next().map_or(0, str::len);
}
Ok(self.column + usize::from(end && self.bytes > self.grapheme_start))
}
}
fn decode_semantic_tokens(
text: &str,
result: &Value,
options: &SemanticOptions,
) -> Result<Option<Arc<[SemanticToken]>>, String> {
if result.is_null() {
return Ok(None);
}
if result
.get("resultId")
.is_some_and(|value| !value.is_string())
{
return Err("invalid semantic token result ID".into());
}
let data = result["data"]
.as_array()
.ok_or("invalid semantic token data")?;
if !data.len().is_multiple_of(5) {
return Err("incomplete semantic token data".into());
}
let count = data.len() / 5;
if count > MAX_BODY / std::mem::size_of::<SemanticToken>() {
return Err("decoded semantic tokens exceed 4 MiB".into());
}
if count == 0 {
return Ok(None);
}
let text = normalized(text);
let mut lines = text.split('\n');
let mut line = SemanticLine::new(lines.next().unwrap());
let mut row = 0usize;
let mut start = 0usize;
let mut previous_end = 0usize;
let mut tokens = Vec::with_capacity(count);
for chunk in data.chunks_exact(5) {
let integer = |index: usize| -> Result<usize, String> {
chunk[index]
.as_u64()
.filter(|value| *value <= MAX_VERSION)
.map(|value| value as usize)
.ok_or_else(|| "invalid semantic token integer".into())
};
let delta_row = integer(0)?;
let delta_start = integer(1)?;
let length = integer(2)?;
let kind = integer(3)?;
let mask = integer(4)? as u32;
let next_row = row
.checked_add(delta_row)
.filter(|value| *value <= MAX_VERSION as usize)
.ok_or("semantic token row overflow")?;
if delta_row != 0 {
for _ in row..next_row {
line =
SemanticLine::new(lines.next().ok_or("semantic token row is out of bounds")?);
}
row = next_row;
start = delta_start;
previous_end = 0;
} else {
start = start
.checked_add(delta_start)
.filter(|value| *value <= MAX_VERSION as usize)
.ok_or("semantic token column overflow")?;
}
let end = start
.checked_add(length)
.ok_or("semantic token length overflow")?;
let token_kind = options
.types
.get(kind)
.filter(|_| kind < 65536)
.ok_or("semantic token type is outside its legend")?;
if mask & !options.modifier_mask != 0 {
return Err("semantic token modifier is outside its legend".into());
}
if start < previous_end {
return Err("overlapping semantic tokens are unsupported".into());
}
let start_column = line.column(start, false)?;
let end_column = line.column(end, true)?;
previous_end = end.min(line.units);
if length == 0 || start_column >= end_column {
continue;
}
if let Some(kind) = token_kind {
let mut modifiers = 0;
let mut remaining = mask;
while remaining != 0 {
modifiers |= options.modifiers[remaining.trailing_zeros() as usize];
remaining &= remaining - 1;
}
tokens.push(SemanticToken {
start: TextPosition {
row,
column: start_column,
},
end: TextPosition {
row,
column: end_column,
},
kind: *kind,
modifiers,
});
}
}
Ok((!tokens.is_empty()).then(|| tokens.into()))
}
fn sync_options(value: &Value) -> Result<SyncOptions, String> {
if let Some(kind) = value.as_u64().filter(|kind| *kind <= 2) {
return Ok(SyncOptions {
@@ -2064,6 +2516,67 @@ fn uri_path(uri: &str) -> Result<PathBuf, String> {
mod tests {
use super::*;
#[test]
fn semantic_decoder_validates_raw_coordinates_before_grapheme_rounding() {
let options = semantic_options(Some(&json!({
"full": true,
"legend": {"tokenTypes": ["custom", "function"],
"tokenModifiers": ["custom", "readonly", "declaration", "async"]}
})))
.unwrap()
.unwrap();
let tokens = decode_semantic_tokens(
"A😀e\u{301}Z\r\n\r\nx",
&json!({"data": [
0, 1, 2, 1, 2,
0, 2, 1, 1, 12,
0, 1, 1, 1, 0,
0, 1, 999, 0, 0,
1, 0, 1, 1, 0,
1, 0, 0, 1, 0,
0, 0, 100, 1, 0
]}),
&options,
)
.unwrap()
.unwrap();
assert_eq!(tokens.len(), 4);
assert_eq!(tokens[0].start, TextPosition { row: 0, column: 1 });
assert_eq!(tokens[0].end, TextPosition { row: 0, column: 2 });
assert_eq!(tokens[0].modifiers, SemanticTokenModifier::Readonly as u16);
assert_eq!(tokens[1].start, TextPosition { row: 0, column: 2 });
assert_eq!(tokens[1].end, TextPosition { row: 0, column: 3 });
assert_eq!(
tokens[1].modifiers,
SemanticTokenModifier::Declaration as u16 | SemanticTokenModifier::Async as u16
);
assert_eq!(tokens[2].start, tokens[1].start);
assert_eq!(tokens[3].end, TextPosition { row: 2, column: 1 });
for data in [
json!([0, 0, 1, 1]),
json!([-1, 0, 1, 1, 0]),
json!([0, 0.5, 1, 1, 0]),
json!([0, 0, 1, 1, 2147483648u64]),
json!([0, 0, 1, 65536, 0]),
json!([0, 0, 1, 2, 0]),
json!([0, 0, 1, 1, 16]),
json!([0, 2, 1, 0, 0]),
json!([0, 1, 1, 1, 0]),
json!([0, 7, 1, 1, 0]),
json!([2, 0, 1, 1, 0]),
json!([0, 1, 2, 1, 0, 0, 1, 1, 1, 0]),
] {
assert!(
decode_semantic_tokens("A😀e\u{301}Z\n", &json!({"data": data}), &options).is_err(),
"{data}"
);
}
let count = MAX_BODY / std::mem::size_of::<SemanticToken>() + 1;
assert!(
decode_semantic_tokens("x", &json!({"data": vec![0; count * 5]}), &options).is_err()
);
}
#[test]
fn unicode_positions_and_local_uri_boundaries() {
let text = "A😀e\u{301}Z\n";
src/main.rs
+1
View File
@@ -41,6 +41,7 @@ Presets: rust -> rust-analyzer, python -> ty server + ruff server, ruby -> solar
Custom languages: config.lsp[\"my-language\"] = { command: [...], extensions: [...] }
Server stacks: config.lsp[\"python\"] = { servers: [{ command: [...] }, { command: [...] }] }
LSP commands: :lsp.definition, :lsp.diagnostics, :lsp.hover
Semantic highlighting: automatic when the first enabled server supports semantic tokens
Hover binding: config.bind \"K\", config.core.lsp.hover (no default key)";
struct Options {
src/plugin.rs
+61 -1
View File
@@ -1,4 +1,4 @@
use std::path::PathBuf;
use std::{path::PathBuf, sync::Arc};
use serde::{Deserialize, Serialize};
@@ -52,6 +52,59 @@ pub struct Diagnostic {
pub message: String,
}
/// Standard LSP semantic token categories, independent of frontend colors.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum SemanticTokenKind {
Namespace,
Type,
Class,
Enum,
Interface,
Struct,
TypeParameter,
Parameter,
Variable,
Property,
EnumMember,
Event,
Function,
Method,
Macro,
Keyword,
Modifier,
Comment,
String,
Number,
Regexp,
Operator,
Decorator,
}
/// Canonical modifier bits; server legend bit positions are translated by LSP.
#[repr(u16)]
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum SemanticTokenModifier {
Declaration = 1,
Definition = 1 << 1,
Readonly = 1 << 2,
Static = 1 << 3,
Deprecated = 1 << 4,
Abstract = 1 << 5,
Async = 1 << 6,
Modification = 1 << 7,
Documentation = 1 << 8,
DefaultLibrary = 1 << 9,
}
/// A single-line foreground annotation using whole extended-grapheme positions.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct SemanticToken {
pub start: TextPosition,
pub end: TextPosition,
pub kind: SemanticTokenKind,
pub modifiers: u16,
}
#[derive(Debug, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)]
pub enum UpdateEffect {
@@ -60,6 +113,13 @@ pub enum UpdateEffect {
revision: u64,
items: Vec<Diagnostic>,
},
#[serde(skip_deserializing)]
SemanticTokens {
buffer_id: usize,
revision: u64,
/// None clears the overlay. Immutable slices are shared without copying.
tokens: Option<Arc<[SemanticToken]>>,
},
Navigate {
buffer_id: usize,
revision: u64,
src/session.rs
+166 -40
View File
@@ -390,50 +390,73 @@ impl EditorSession {
return Ok(false);
}
// Validate the whole applicable annotation batch before changing any buffer.
for effect in &update.effects {
match effect {
UpdateEffect::Diagnostics {
buffer_id,
revision,
items,
} => {
self.editor
.validate_diagnostics(*buffer_id, *revision, items)?;
}
UpdateEffect::Navigate {
buffer_id,
revision,
path,
..
} => {
let source = InvocationSource {
buffer_id: *buffer_id,
revision: *revision,
};
if self.request_is_current(owner, update.request_id, source)
&& (path.as_os_str().is_empty()
|| path.to_string_lossy().chars().any(char::is_control))
{
return Err(
"Navigation path must be nonempty and contain no controls".into()
);
let validation = (|| -> Result<(), String> {
for effect in &update.effects {
match effect {
UpdateEffect::Diagnostics {
buffer_id,
revision,
items,
} => {
self.editor
.validate_diagnostics(*buffer_id, *revision, items)?;
}
}
UpdateEffect::Hover {
buffer_id,
revision,
text,
} => {
let source = InvocationSource {
buffer_id: *buffer_id,
revision: *revision,
};
if self.hover_is_current(owner, update.request_id, source) {
crate::editor::validate_hover_text(text)?;
UpdateEffect::SemanticTokens {
buffer_id,
revision,
tokens,
} => {
if owner != "core" {
return Err("Semantic tokens are native LSP updates only".into());
}
self.editor.validate_semantic_tokens(
*buffer_id,
*revision,
tokens.as_deref(),
)?;
}
UpdateEffect::Navigate {
buffer_id,
revision,
path,
..
} => {
let source = InvocationSource {
buffer_id: *buffer_id,
revision: *revision,
};
if self.request_is_current(owner, update.request_id, source)
&& (path.as_os_str().is_empty()
|| path.to_string_lossy().chars().any(char::is_control))
{
return Err(
"Navigation path must be nonempty and contain no controls".into()
);
}
}
UpdateEffect::Hover {
buffer_id,
revision,
text,
} => {
let source = InvocationSource {
buffer_id: *buffer_id,
revision: *revision,
};
if self.hover_is_current(owner, update.request_id, source) {
crate::editor::validate_hover_text(text)?;
}
}
UpdateEffect::Message { .. } => {}
}
UpdateEffect::Message { .. } => {}
}
Ok(())
})();
if let Err(error) = validation {
if owner == "core" {
self.editor.clear_semantic_tokens();
}
return Err(error);
}
let mut changed = false;
for effect in update.effects {
@@ -447,6 +470,15 @@ impl EditorSession {
.editor
.replace_validated_diagnostics(owner, buffer_id, revision, items);
}
UpdateEffect::SemanticTokens {
buffer_id,
revision,
tokens,
} => {
changed |= self
.editor
.replace_validated_semantic_tokens(buffer_id, revision, tokens);
}
UpdateEffect::Navigate {
buffer_id,
revision,
@@ -595,6 +627,100 @@ mod tests {
static NEXT_FIXTURE: AtomicU64 = AtomicU64::new(0);
#[test]
fn native_semantic_batches_are_atomic_and_malformed_updates_clear_hidden_overlays() {
use crate::plugin::{SemanticToken, SemanticTokenKind};
use std::sync::Arc;
let fixture = Fixture::new();
fixture.file("buffer.txt", "source");
let target = fixture.file("target.txt", "target");
let mut session = fixture.open("");
let token = SemanticToken {
start: TextPosition { row: 0, column: 0 },
end: TextPosition { row: 0, column: 1 },
kind: SemanticTokenKind::Variable,
modifiers: 0,
};
let tokens: Arc<[SemanticToken]> = Arc::from([token]);
let semantic = |buffer_id, revision, tokens| UpdateEffect::SemanticTokens {
buffer_id,
revision,
tokens,
};
let update = |effects| PluginUpdate {
request_id: None,
effects,
};
session
.apply_update("core", update(vec![semantic(1, 0, Some(tokens.clone()))]))
.unwrap();
session.execute(&format!("edit {}", target.display()));
session
.apply_update("core", update(vec![semantic(2, 0, Some(tokens.clone()))]))
.unwrap();
let diagnostic = crate::plugin::Diagnostic {
start: token.start,
end: token.end,
severity: crate::plugin::DiagnosticSeverity::Warning,
message: "unchanged".into(),
};
session
.editor
.replace_diagnostics("plugin", 2, 0, vec![diagnostic.clone()])
.unwrap();
let malformed: Arc<[SemanticToken]> = Arc::from([SemanticToken {
end: TextPosition { row: 99, column: 0 },
..token
}]);
let before_message = session.editor.message().to_owned();
let result = session.apply_update(
"core",
update(vec![
UpdateEffect::Message {
text: "must not apply".into(),
},
UpdateEffect::Diagnostics {
buffer_id: 2,
revision: 0,
items: vec![diagnostic.clone()],
},
semantic(1, 0, Some(tokens.clone())),
semantic(2, 0, Some(malformed.clone())),
]),
);
assert!(result.is_err());
assert_eq!(session.editor.message(), before_message);
assert!(session.editor.semantic_tokens().is_empty());
assert_eq!(session.editor.diagnostics().count(), 1);
session.execute("buffer 1");
assert!(session.editor.semantic_tokens().is_empty());
assert!(
session
.apply_update(
"core",
update(vec![
semantic(999, 0, Some(malformed.clone())),
semantic(1, 99, Some(malformed)),
semantic(1, 0, Some(tokens.clone())),
])
)
.unwrap()
);
assert!(std::ptr::eq(
session.editor.semantic_tokens(),
tokens.as_ref()
));
assert_eq!(session.editor.lines(), ["source"]);
assert_eq!(session.editor.text_revision(), 0);
assert!(!session.editor.dirty());
assert!(
session
.apply_update("core", update(vec![semantic(1, 0, None)]))
.unwrap()
);
assert!(session.editor.semantic_tokens().is_empty());
}
struct Fixture(PathBuf);
impl Fixture {
src/syntax.rs
+1 -1
View File
@@ -20,7 +20,7 @@ pub enum SyntaxKind {
Markup,
}
/// A semantic foreground range in the original line's bytes, excluding its newline.
/// A syntax foreground range in the original line's bytes, excluding its newline.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub struct SyntaxSpan {
pub start_byte: usize,
src/tui.rs
+187 -22
View File
@@ -31,7 +31,9 @@ use unicode_width::UnicodeWidthStr;
use crate::{
editor::{Editor, Hover, Input, Mode, Palette, StartupDiagnostic},
files::{DockSide, FilePicker, FilePresentation, FileView},
plugin::{Diagnostic, DiagnosticSeverity},
plugin::{
Diagnostic, DiagnosticSeverity, SemanticToken, SemanticTokenKind, SemanticTokenModifier,
},
session::EditorSession,
syntax::{SyntaxCache, SyntaxKind},
};
@@ -345,15 +347,25 @@ fn draw(
.collect();
frame.render_widget(Paragraph::new(lines), viewport);
let buffer = frame.buffer_mut();
let semantic = editor.semantic_tokens();
let mut semantic_index = semantic.partition_point(|token| token.start.row < *top);
for (row, line) in editor.lines().iter().enumerate().skip(*top).take(height) {
let spans = syntax.line(row);
if spans.is_empty() {
while semantic_index < semantic.len() && semantic[semantic_index].start.row < row {
semantic_index += 1;
}
if spans.is_empty()
&& semantic
.get(semantic_index)
.is_none_or(|token| token.start.row != row)
{
continue;
}
let mut span_index = 0;
let mut token_style = None;
let mut column = 0;
let right = left.saturating_add(width);
for (byte, grapheme) in line.grapheme_indices(true) {
for (index, (byte, grapheme)) in line.grapheme_indices(true).enumerate() {
if column >= right {
break;
}
@@ -366,22 +378,39 @@ fn draw(
while span_index < spans.len() && spans[span_index].end_byte <= byte {
span_index += 1;
}
if let Some(span) = spans.get(span_index)
&& span.start_byte <= byte
while semantic_index < semantic.len()
&& semantic[semantic_index].start.row == row
&& semantic[semantic_index].end.column <= index
{
semantic_index += 1;
token_style = None;
}
let style = semantic
.get(semantic_index)
.filter(|token| token.start.row == row && token.start.column <= index)
.map(|token| *token_style.get_or_insert_with(|| semantic_style(token)))
.or_else(|| {
spans
.get(span_index)
.filter(|span| span.start_byte <= byte)
.map(|span| {
Style::default().fg(match span.kind {
SyntaxKind::Comment => Color::DarkGray,
SyntaxKind::String => Color::Green,
SyntaxKind::Keyword => Color::Magenta,
SyntaxKind::Number => Color::Yellow,
SyntaxKind::Type | SyntaxKind::Markup => Color::Cyan,
SyntaxKind::Function => Color::LightBlue,
})
})
});
if let Some(style) = style
&& end > *left
&& (grapheme == "\t" || (column >= *left && end <= right))
{
let color = match span.kind {
SyntaxKind::Comment => Color::DarkGray,
SyntaxKind::String => Color::Green,
SyntaxKind::Keyword => Color::Magenta,
SyntaxKind::Number => Color::Yellow,
SyntaxKind::Type | SyntaxKind::Markup => Color::Cyan,
SyntaxKind::Function => Color::LightBlue,
};
let y = viewport.y + (row - *top) as u16;
for cell in column.max(*left)..end.min(right) {
buffer[(viewport.x + (cell - *left) as u16, y)].set_fg(color);
buffer[(viewport.x + (cell - *left) as u16, y)].set_style(style);
}
}
column = end;
@@ -1044,6 +1073,36 @@ fn draw_palette(frame: &mut Frame, palette: &Palette, mode: Mode, navigation_hin
);
}
fn semantic_style(token: &SemanticToken) -> Style {
use SemanticTokenKind::*;
let readonly = token.modifiers & SemanticTokenModifier::Readonly as u16 != 0;
let color = match token.kind {
Namespace | Type | Class | Enum | Interface | Struct | TypeParameter => Color::Cyan,
Function | Method | Event | Macro => Color::LightBlue,
Parameter | Variable | Property if readonly => Color::Yellow,
Parameter | Variable => Color::LightCyan,
Property => Color::Blue,
EnumMember | Number => Color::Yellow,
Keyword | Modifier | Operator | Decorator => Color::Magenta,
String | Regexp => Color::Green,
Comment => Color::DarkGray,
};
let mut style = Style::default().fg(color);
if token.modifiers
& (SemanticTokenModifier::Declaration as u16 | SemanticTokenModifier::Definition as u16)
!= 0
{
style = style.add_modifier(ratatui::style::Modifier::BOLD);
}
if token.kind == Comment && token.modifiers & SemanticTokenModifier::Documentation as u16 != 0 {
style = style.add_modifier(ratatui::style::Modifier::ITALIC);
}
if token.modifiers & SemanticTokenModifier::Deprecated as u16 != 0 {
style = style.add_modifier(ratatui::style::Modifier::CROSSED_OUT);
}
style
}
#[cfg(test)]
mod tests {
use super::*;
@@ -1150,6 +1209,21 @@ mod tests {
}
}
fn semantic_token(
row: usize,
start: usize,
end: usize,
kind: SemanticTokenKind,
modifiers: u16,
) -> SemanticToken {
SemanticToken {
start: crate::plugin::TextPosition { row, column: start },
end: crate::plugin::TextPosition { row, column: end },
kind,
modifiers,
}
}
#[test]
fn diagnostic_underlines_preserve_syntax_and_whole_graphemes_at_viewport_edges() {
let directory = Directory::new();
@@ -1579,15 +1653,54 @@ end
}
#[test]
fn syntax_colors_whole_unicode_graphemes_and_clipped_tabs_without_painting_wide_blanks() {
fn semantic_colors_modifiers_and_syntax_fallback_cover_whole_graphemes_and_clipped_tabs() {
let directory = Directory::new();
let source = "#a\t界e\u{301}👩‍🚀z";
let mut editor = Editor::open(Some(directory.file("unicode.py", source)), vec![]).unwrap();
editor.set_clipboard_mode("separate").unwrap();
editor
.replace_semantic_tokens(
editor.buffer_id(),
editor.text_revision(),
Some(Arc::from([
semantic_token(
0,
0,
1,
SemanticTokenKind::Comment,
SemanticTokenModifier::Documentation as u16,
),
semantic_token(
0,
2,
6,
SemanticTokenKind::Property,
SemanticTokenModifier::Readonly as u16
| SemanticTokenModifier::Definition as u16
| SemanticTokenModifier::Deprecated as u16,
),
semantic_token(0, 2, 3, SemanticTokenKind::Variable, 0),
])),
)
.unwrap();
editor
.replace_diagnostics(
"fixture",
editor.buffer_id(),
editor.text_revision(),
vec![annotation(
(0, 2),
(0, 6),
DiagnosticSeverity::Warning,
"semantic",
)],
)
.unwrap();
let mut syntax = SyntaxCache::new();
for (offset, width, caret, colors) in [
(2, 5, 4, vec![Color::DarkGray; 5]),
(5, 2, 4, vec![Color::Reset, Color::DarkGray]),
(7, 2, 5, vec![Color::DarkGray; 2]),
(2, 5, 4, vec![Color::Yellow; 5]),
(5, 2, 4, vec![Color::Reset, Color::Yellow]),
(7, 2, 5, vec![Color::Yellow; 2]),
(8, 2, 6, vec![Color::Reset, Color::DarkGray]),
(
0,
@@ -1596,8 +1709,8 @@ end
vec![
Color::DarkGray,
Color::DarkGray,
Color::DarkGray,
Color::DarkGray,
Color::Yellow,
Color::Yellow,
Color::Reset,
],
),
@@ -1632,21 +1745,73 @@ end
assert_eq!(buffer[position].symbol(), expected[position].symbol());
assert_eq!(rendered[position].fg, color, "left={offset}, cell={cell}");
assert_eq!(rendered[position].bg, Color::Reset);
let semantic_cell = color == Color::Yellow;
assert_eq!(
rendered[position].modifier.contains(Modifier::BOLD),
semantic_cell
);
assert_eq!(
rendered[position].modifier.contains(Modifier::CROSSED_OUT),
semantic_cell
);
assert_eq!(
rendered[position].modifier.contains(Modifier::UNDERLINED),
semantic_cell
);
if semantic_cell {
assert_eq!(rendered[position].underline_color, Color::Yellow);
}
assert_eq!(
rendered[position].modifier.contains(Modifier::ITALIC),
offset == 0 && cell == 0
);
}
assert_eq!(buffer[(area.x, area.y)].fg, Color::Cyan);
assert_eq!(buffer[(area.x, area.y)].fg, Color::Yellow);
assert_eq!(buffer[(0, 0)].fg, Color::Reset);
assert_eq!(editor.lines(), &[source]);
}
editor.handle(Input::Char('x'));
assert!(editor.semantic_tokens().is_empty());
let mut terminal = Terminal::new(ratatui::backend::TestBackend::new(20, 4)).unwrap();
let (mut top, mut left) = (0, 0);
terminal
.draw(|frame| draw(frame, &editor, &mut top, &mut left, &mut syntax))
.unwrap();
assert_eq!(terminal.backend().buffer()[(2, 0)].fg, Color::DarkGray);
assert!(
!terminal.backend().buffer()[(2, 0)]
.modifier
.contains(Modifier::ITALIC)
);
}
#[test]
fn search_visual_selection_and_secondary_carets_override_syntax_foreground() {
fn search_visual_selection_and_secondary_carets_override_semantic_foreground() {
let directory = Directory::new();
let mut editor = Editor::open(
Some(directory.file("overlays.py", "# word\n# word\n# word")),
vec![],
)
.unwrap();
editor
.replace_semantic_tokens(
editor.buffer_id(),
editor.text_revision(),
Some(Arc::from(
(0..3)
.map(|row| {
semantic_token(
row,
2,
6,
SemanticTokenKind::Function,
SemanticTokenModifier::Declaration as u16,
)
})
.collect::<Vec<_>>(),
)),
)
.unwrap();
editor
.replace_diagnostics(
"fixture",