hl-valdemar/vamp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

feat: enable LSP semantic token highlighting by default

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-10-09 22:17:30 +02:00
parent 548711980c
commit d418d670ee
13 changed files with 1714 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lsp.rs
+522 -9
View File
@@ -19,7 +19,8 @@ use crate::{
editor::Context,
plugin::{
Diagnostic, DiagnosticSeverity, DocumentEvent, DocumentEventKind, DocumentSnapshot,
PluginUpdate, TextPosition, UpdateEffect,
PluginUpdate, SemanticToken, SemanticTokenKind, SemanticTokenModifier, TextPosition,
UpdateEffect,
},
};
@@ -49,6 +50,7 @@ struct CachedDocument {
snapshot: Arc<DocumentSnapshot>,
servers: Vec<ServerKey>,
diagnostics: HashMap<usize, Contribution>,
semantic_tokens: Option<Arc<[SemanticToken]>>,
}
struct Contribution {
@@ -62,6 +64,10 @@ enum Output {
snapshot: Arc<DocumentSnapshot>,
items: Vec<Diagnostic>,
},
SemanticTokens {
snapshot: Arc<DocumentSnapshot>,
tokens: Option<Arc<[SemanticToken]>>,
},
}
struct Worker {
@@ -78,6 +84,7 @@ enum Input {
snapshot: Arc<DocumentSnapshot>,
event: DocumentEventKind,
served: bool,
primary: bool,
},
Position(PositionKind, Source),
Diagnostics(usize),
@@ -115,6 +122,7 @@ pub struct Lsp {
servers: BTreeMap<ServerKey, Worker>,
updates: Vec<PluginUpdate>,
changed_diagnostics: HashSet<usize>,
changed_semantics: HashSet<usize>,
}
impl Lsp {
@@ -126,6 +134,7 @@ impl Lsp {
servers: BTreeMap::new(),
updates: Vec::new(),
changed_diagnostics: HashSet::new(),
changed_semantics: HashSet::new(),
}
}
@@ -152,6 +161,7 @@ impl Lsp {
if matches!(event.event, DocumentEventKind::Close) {
self.documents.remove(&snapshot.id);
self.changed_diagnostics.remove(&snapshot.id);
self.changed_semantics.remove(&snapshot.id);
} else if self.documents.get(&snapshot.id).is_none_or(|cached| {
cached.snapshot.path != snapshot.path || cached.snapshot.root != snapshot.root
}) {
@@ -193,9 +203,11 @@ impl Lsp {
snapshot: Arc::clone(&snapshot),
servers: keys,
diagnostics: HashMap::new(),
semantic_tokens: None,
},
);
self.changed_diagnostics.insert(snapshot.id);
self.changed_semantics.insert(snapshot.id);
for index in 0..self.documents[&snapshot.id].servers.len() {
let key = self.documents[&snapshot.id].servers[index].clone();
if let Err(error) = self.start(&key) {
@@ -207,6 +219,8 @@ impl Lsp {
if !Arc::ptr_eq(&cached.snapshot, &snapshot) {
cached.snapshot = Arc::clone(&snapshot);
cached.diagnostics.clear();
cached.semantic_tokens = None;
self.changed_semantics.insert(snapshot.id);
self.changed_diagnostics.insert(snapshot.id);
}
}
@@ -224,6 +238,10 @@ impl Lsp {
.documents
.get(&snapshot.id)
.is_some_and(|doc| doc.servers.contains(server_key)),
primary: self
.documents
.get(&snapshot.id)
.is_some_and(|doc| doc.servers.first() == Some(server_key)),
})
.is_err()
{
@@ -330,6 +348,7 @@ impl Lsp {
worker,
&mut self.documents,
&mut self.changed_diagnostics,
&mut self.changed_semantics,
&mut self.updates,
"LSP server stopped: combined diagnostic messages exceed 4 MiB"
.into(),
@@ -344,10 +363,20 @@ impl Lsp {
}
self.changed_diagnostics.insert(snapshot.id);
}
Output::SemanticTokens { snapshot, tokens } => {
let Some(doc) = self.documents.get_mut(&snapshot.id).filter(|doc| {
doc.servers.first() == Some(key)
&& Arc::ptr_eq(&doc.snapshot, &snapshot)
}) else {
continue;
};
doc.semantic_tokens = tokens;
self.changed_semantics.insert(snapshot.id);
}
}
}
}
let effects: Vec<_> = self
let mut effects: Vec<_> = self
.changed_diagnostics
.drain()
.filter_map(|id| {
@@ -370,6 +399,14 @@ impl Lsp {
})
})
.collect();
effects.extend(self.changed_semantics.drain().filter_map(|id| {
let doc = self.documents.get(&id)?;
Some(UpdateEffect::SemanticTokens {
buffer_id: id,
revision: doc.snapshot.revision,
tokens: doc.semantic_tokens.clone(),
})
}));
if !effects.is_empty() {
self.updates.push(PluginUpdate {
request_id: None,
@@ -396,7 +433,13 @@ impl Lsp {
let initial: Vec<_> = self
.documents
.values()
.map(|doc| (Arc::clone(&doc.snapshot), doc.servers.contains(key)))
.map(|doc| {
(
Arc::clone(&doc.snapshot),
doc.servers.contains(key),
doc.servers.first() == Some(key),
)
})
.collect();
let started = thread::Builder::new()
.name(format!("lsp-{}", config.language_id))
@@ -415,8 +458,8 @@ impl Lsp {
};
let result = initial
.into_iter()
.try_for_each(|(snapshot, served)| {
runtime.document(snapshot, DocumentEventKind::Open, served)
.try_for_each(|(snapshot, served, primary)| {
runtime.document(snapshot, DocumentEventKind::Open, served, primary)
})
.and_then(|()| runtime.run());
if let Err(reason) = result {
@@ -483,6 +526,7 @@ impl Lsp {
worker,
&mut self.documents,
&mut self.changed_diagnostics,
&mut self.changed_semantics,
&mut self.updates,
reason,
);
@@ -493,6 +537,7 @@ impl Lsp {
worker: &mut Worker,
documents: &mut HashMap<usize, CachedDocument>,
changed: &mut HashSet<usize>,
changed_semantics: &mut HashSet<usize>,
updates: &mut Vec<PluginUpdate>,
reason: String,
) {
@@ -502,6 +547,10 @@ impl Lsp {
worker.retired = true;
worker.cancelled.store(true, Ordering::Release);
for (id, doc) in documents {
if doc.servers.first() == Some(key) {
doc.semantic_tokens = None;
changed_semantics.insert(*id);
}
if doc.servers.contains(key) {
doc.diagnostics.remove(&key.config);
changed.insert(*id);
@@ -535,6 +584,8 @@ struct Document {
version: u64,
opened: bool,
saved: bool,
primary: bool,
semantic_dirty: bool,
}
#[derive(Default)]
@@ -549,6 +600,7 @@ enum Request {
Initialize,
Definition(Source),
Hover(Source),
SemanticTokens(Arc<DocumentSnapshot>),
Shutdown,
}
@@ -563,10 +615,12 @@ struct Runtime {
documents: HashMap<usize, Document>,
requests: HashMap<u64, Request>,
pending: VecDeque<(PositionKind, Source)>,
semantic_inflight: HashSet<usize>,
next_id: u64,
ready: bool,
definition: bool,
hover: bool,
semantic: Option<SemanticOptions>,
sync: SyncOptions,
}
@@ -590,10 +644,12 @@ impl Runtime {
documents: HashMap::new(),
requests: HashMap::new(),
pending: VecDeque::new(),
semantic_inflight: HashSet::new(),
next_id: 1,
ready: false,
definition: false,
hover: false,
semantic: None,
sync: SyncOptions::default(),
};
runtime.request(
@@ -606,14 +662,26 @@ impl Runtime {
"initializationOptions": runtime.config.initialization_options,
"capabilities": {
"general": {"positionEncodings": ["utf-16"]},
"workspace": {"configuration": true, "workspaceFolders": true},
"workspace": {"configuration": true, "workspaceFolders": true,
"semanticTokens": {"refreshSupport": true}},
"textDocument": {
"synchronization": {"dynamicRegistration": false, "willSave": false,
"willSaveWaitUntil": false, "didSave": true},
"definition": {"dynamicRegistration": false, "linkSupport": true},
"hover": {"dynamicRegistration": false,
"contentFormat": ["plaintext", "markdown"]},
"publishDiagnostics": {"relatedInformation": false, "versionSupport": true}
"publishDiagnostics": {"relatedInformation": false, "versionSupport": true},
"semanticTokens": {
"dynamicRegistration": false,
"requests": {"range": true, "full": true},
"tokenTypes": SEMANTIC_TYPES.iter().map(|(name, _)| *name).collect::<Vec<_>>(),
"tokenModifiers": SEMANTIC_MODIFIERS.iter().map(|(name, _)| *name).collect::<Vec<_>>(),
"formats": ["relative"],
"overlappingTokenSupport": false,
"multilineTokenSupport": false,
"serverCancelSupport": false,
"augmentsSyntaxTokens": true
}
}
}
})),
@@ -630,7 +698,8 @@ impl Runtime {
snapshot,
event,
served,
}) => self.document(snapshot, event, served)?,
primary,
}) => self.document(snapshot, event, served, primary)?,
Ok(Input::Position(kind, source)) => self.position(kind, source)?,
Ok(Input::Diagnostics(id)) => {
let text = if !self.documents.contains_key(&id) {
@@ -661,6 +730,19 @@ impl Runtime {
};
self.incoming(frame)?;
}
if self.ready && self.semantic.is_some() && self.requests.len() < QUEUE_SIZE {
let pending: Vec<_> = self
.documents
.iter()
.filter(|(id, doc)| {
doc.primary && doc.semantic_dirty && !self.semantic_inflight.contains(*id)
})
.map(|(id, _)| *id)
.collect();
for id in pending {
self.semantic_request(id)?;
}
}
self.transport.check_alive()?;
thread::sleep(TICK);
}
@@ -683,6 +765,17 @@ impl Runtime {
})
}
fn emit_semantics(
&self,
snapshot: &Arc<DocumentSnapshot>,
tokens: Option<Arc<[SemanticToken]>>,
) -> Result<(), String> {
self.send(Output::SemanticTokens {
snapshot: Arc::clone(snapshot),
tokens,
})
}
fn send(&self, output: Output) -> Result<(), String> {
self.output.try_send(output).map_err(|error| match error {
TrySendError::Full(_) => "LSP incoming update queue is full".into(),
@@ -695,6 +788,7 @@ impl Runtime {
snapshot: Arc<DocumentSnapshot>,
event: DocumentEventKind,
served: bool,
primary: bool,
) -> Result<(), String> {
let id = snapshot.id;
let old = self.documents.remove(&id);
@@ -733,11 +827,16 @@ impl Runtime {
return Err("LSP document version exhausted".into());
}
let mut doc = Document {
snapshot,
snapshot: Arc::clone(&snapshot),
uri,
version,
opened: same_uri && old.as_ref().is_some_and(|old| old.opened),
saved: matches!(event, DocumentEventKind::Save),
primary,
semantic_dirty: primary
&& old.as_ref().is_none_or(|old| {
old.semantic_dirty || !old.primary || !Arc::ptr_eq(&old.snapshot, &snapshot)
}),
};
if changed {
self.emit_diagnostics(&doc.snapshot, Vec::new())?;
@@ -753,6 +852,7 @@ impl Runtime {
}
}
self.documents.insert(id, doc);
self.semantic_request(id)?;
Ok(())
}
@@ -857,6 +957,60 @@ impl Runtime {
self.request(method, Some(params), request)
}
fn semantic_request(&mut self, id: usize) -> Result<(), String> {
let Some(options) = &self.semantic else {
return Ok(());
};
let Some(doc) = self.documents.get(&id).filter(|doc| {
self.ready && doc.primary && doc.semantic_dirty && !self.semantic_inflight.contains(&id)
}) else {
return Ok(());
};
if self.requests.len() == QUEUE_SIZE {
return Ok(());
}
let snapshot = Arc::clone(&doc.snapshot);
let mut params = json!({"textDocument": {"uri": doc.uri}});
let method = if options.full {
"textDocument/semanticTokens/full"
} else {
let text = normalized(&snapshot.text);
let (row, line) = text.split('\n').enumerate().last().unwrap();
params["range"] = json!({"start": {"line": 0, "character": 0},
"end": {"line": row, "character": line.encode_utf16().count()}});
"textDocument/semanticTokens/range"
};
self.request(method, Some(params), Request::SemanticTokens(snapshot))?;
self.semantic_inflight.insert(id);
self.documents.get_mut(&id).unwrap().semantic_dirty = false;
Ok(())
}
fn semantic_result(
&mut self,
snapshot: Arc<DocumentSnapshot>,
result: Result<&Value, String>,
) -> Result<(), String> {
self.semantic_inflight.remove(&snapshot.id);
let fresh = self.documents.get(&snapshot.id).is_some_and(|doc| {
doc.primary && !doc.semantic_dirty && Arc::ptr_eq(&doc.snapshot, &snapshot)
});
// Stale responses are discarded before inspecting their token payload.
if fresh {
let decoded = result.and_then(|result| {
decode_semantic_tokens(&snapshot.text, result, self.semantic.as_ref().unwrap())
});
match decoded {
Ok(tokens) => self.emit_semantics(&snapshot, tokens)?,
Err(reason) => {
self.emit_semantics(&snapshot, None)?;
self.emit(message(&format!("semantic tokens failed: {reason}"), None))?;
}
}
}
self.semantic_request(snapshot.id)
}
fn request(
&mut self,
method: &str,
@@ -920,6 +1074,9 @@ impl Runtime {
Request::Hover(source) => {
self.hover_result(source, Err("unexpected LSP response".into()))
}
Request::SemanticTokens(snapshot) => {
self.semantic_result(snapshot, Err("unexpected LSP response".into()))
}
_ => Err("unexpected LSP response".into()),
};
}
@@ -932,6 +1089,9 @@ impl Runtime {
Request::Hover(source) => {
self.hover_result(source, Err("invalid LSP error response".into()))
}
Request::SemanticTokens(snapshot) => {
self.semantic_result(snapshot, Err("invalid LSP error response".into()))
}
_ => Err("invalid LSP error response".into()),
};
};
@@ -942,6 +1102,9 @@ impl Runtime {
Some(source.request_id),
)),
Request::Hover(source) => self.hover_result(source, Err(text.to_owned())),
Request::SemanticTokens(snapshot) => {
self.semantic_result(snapshot, Err(text.to_owned()))
}
Request::Shutdown => Ok(()),
}
} else {
@@ -949,6 +1112,9 @@ impl Runtime {
Request::Initialize => self.initialized(&frame["result"]),
Request::Definition(source) => self.definition_result(source, &frame["result"]),
Request::Hover(source) => self.hover_result(source, Ok(&frame["result"])),
Request::SemanticTokens(snapshot) => {
self.semantic_result(snapshot, Ok(&frame["result"]))
}
Request::Shutdown => self.notify("exit", None),
}
}
@@ -972,6 +1138,12 @@ impl Runtime {
self.hover = capabilities
.get("hoverProvider")
.is_some_and(|value| value == true || value.is_object());
match semantic_options(capabilities.get("semanticTokensProvider")) {
Ok(options) => self.semantic = options,
Err(reason) => {
self.emit(message(&format!("semantic tokens failed: {reason}"), None))?
}
}
self.notify("initialized", Some(json!({})))?;
self.notify(
"workspace/didChangeConfiguration",
@@ -985,6 +1157,7 @@ impl Runtime {
self.save(&mut doc)?;
}
self.documents.insert(id, doc);
self.semantic_request(id)?;
}
while let Some((kind, source)) = self.pending.pop_front() {
self.position(kind, source)?;
@@ -1032,6 +1205,24 @@ impl Runtime {
}
"workspace/workspaceFolders" => self.workspace_folders()?,
"window/showMessageRequest" => Value::Null,
"workspace/semanticTokens/refresh" => {
self.transport
.send(json!({"jsonrpc": "2.0", "id": id, "result": null}))?;
if self.semantic.is_some() {
let ids: Vec<_> = self
.documents
.iter()
.filter(|(_, doc)| doc.primary)
.map(|(id, _)| *id)
.collect();
for id in ids {
let doc = self.documents.get_mut(&id).unwrap();
doc.semantic_dirty = true;
self.semantic_request(id)?;
}
}
return Ok(());
}
_ => {
response.as_object_mut().unwrap().remove("result");
response["error"] =
@@ -1252,6 +1443,267 @@ impl Runtime {
}
}
const SEMANTIC_TYPES: &[(&str, SemanticTokenKind)] = &[
("namespace", SemanticTokenKind::Namespace),
("type", SemanticTokenKind::Type),
("class", SemanticTokenKind::Class),
("enum", SemanticTokenKind::Enum),
("interface", SemanticTokenKind::Interface),
("struct", SemanticTokenKind::Struct),
("typeParameter", SemanticTokenKind::TypeParameter),
("parameter", SemanticTokenKind::Parameter),
("variable", SemanticTokenKind::Variable),
("property", SemanticTokenKind::Property),
("enumMember", SemanticTokenKind::EnumMember),
("event", SemanticTokenKind::Event),
("function", SemanticTokenKind::Function),
("method", SemanticTokenKind::Method),
("macro", SemanticTokenKind::Macro),
("keyword", SemanticTokenKind::Keyword),
("modifier", SemanticTokenKind::Modifier),
("comment", SemanticTokenKind::Comment),
("string", SemanticTokenKind::String),
("number", SemanticTokenKind::Number),
("regexp", SemanticTokenKind::Regexp),
("operator", SemanticTokenKind::Operator),
("decorator", SemanticTokenKind::Decorator),
];
const SEMANTIC_MODIFIERS: &[(&str, SemanticTokenModifier)] = &[
("declaration", SemanticTokenModifier::Declaration),
("definition", SemanticTokenModifier::Definition),
("readonly", SemanticTokenModifier::Readonly),
("static", SemanticTokenModifier::Static),
("deprecated", SemanticTokenModifier::Deprecated),
("abstract", SemanticTokenModifier::Abstract),
("async", SemanticTokenModifier::Async),
("modification", SemanticTokenModifier::Modification),
("documentation", SemanticTokenModifier::Documentation),
("defaultLibrary", SemanticTokenModifier::DefaultLibrary),
];
struct SemanticOptions {
full: bool,
types: Vec<Option<SemanticTokenKind>>,
modifiers: [u16; 31],
modifier_mask: u32,
}
fn semantic_options(value: Option<&Value>) -> Result<Option<SemanticOptions>, String> {
let Some(value) = value.filter(|value| !value.is_null() && *value != false) else {
return Ok(None);
};
let object = value
.as_object()
.ok_or("invalid semantic tokens capability")?;
let supported = |name| -> Result<bool, String> {
match object.get(name) {
None | Some(Value::Bool(false)) => Ok(false),
Some(Value::Bool(true) | Value::Object(_)) => Ok(true),
_ => Err("invalid semantic tokens request capability".into()),
}
};
let full = supported("full")?;
if !full && !supported("range")? {
return Ok(None);
}
let types = value["legend"]["tokenTypes"]
.as_array()
.filter(|items| items.len() <= 65536)
.ok_or("invalid semantic token type legend")?;
let names = value["legend"]["tokenModifiers"]
.as_array()
.ok_or("invalid semantic token modifier legend")?;
let types = types
.iter()
.map(|value| {
let name = value.as_str().ok_or("invalid semantic token type legend")?;
Ok(SEMANTIC_TYPES
.iter()
.find(|(candidate, _)| *candidate == name)
.map(|(_, kind)| *kind))
})
.collect::<Result<Vec<_>, String>>()?;
let mut modifiers = [0; 31];
for (index, value) in names.iter().enumerate() {
let name = value
.as_str()
.ok_or("invalid semantic token modifier legend")?;
if let Some(slot) = modifiers.get_mut(index) {
*slot = SEMANTIC_MODIFIERS
.iter()
.find(|(candidate, _)| *candidate == name)
.map_or(0, |(_, modifier)| *modifier as u16);
}
}
Ok(Some(SemanticOptions {
full,
types,
modifiers,
modifier_mask: (1u32 << names.len().min(31)) - 1,
}))
}
// Both cursors only advance: strict codepoint boundaries and outward grapheme
// rounding cost O(document + tokens), including many tokens in one grapheme.
struct SemanticLine<'a> {
chars: std::str::Chars<'a>,
graphemes: unicode_segmentation::Graphemes<'a>,
units: usize,
bytes: usize,
column: usize,
grapheme_start: usize,
grapheme_end: usize,
}
impl<'a> SemanticLine<'a> {
fn new(line: &'a str) -> Self {
let mut graphemes = line.graphemes(true);
let grapheme_end = graphemes.next().map_or(0, str::len);
Self {
chars: line.chars(),
graphemes,
units: 0,
bytes: 0,
column: 0,
grapheme_start: 0,
grapheme_end,
}
}
fn column(&mut self, target: usize, end: bool) -> Result<usize, String> {
while self.units < target {
let Some(character) = self.chars.next() else {
if end {
break; // LSP: an overlong end truncates at EOL, never wraps.
}
return Err("semantic token start is beyond its line".into());
};
let next = self.units + character.len_utf16();
if next > target {
return Err("semantic token splits a UTF-16 surrogate pair".into());
}
self.units = next;
self.bytes += character.len_utf8();
}
while self.grapheme_end > self.grapheme_start && self.bytes >= self.grapheme_end {
self.grapheme_start = self.grapheme_end;
self.column += 1;
self.grapheme_end += self.graphemes.next().map_or(0, str::len);
}
Ok(self.column + usize::from(end && self.bytes > self.grapheme_start))
}
}
fn decode_semantic_tokens(
text: &str,
result: &Value,
options: &SemanticOptions,
) -> Result<Option<Arc<[SemanticToken]>>, String> {
if result.is_null() {
return Ok(None);
}
if result
.get("resultId")
.is_some_and(|value| !value.is_string())
{
return Err("invalid semantic token result ID".into());
}
let data = result["data"]
.as_array()
.ok_or("invalid semantic token data")?;
if !data.len().is_multiple_of(5) {
return Err("incomplete semantic token data".into());
}
let count = data.len() / 5;
if count > MAX_BODY / std::mem::size_of::<SemanticToken>() {
return Err("decoded semantic tokens exceed 4 MiB".into());
}
if count == 0 {
return Ok(None);
}
let text = normalized(text);
let mut lines = text.split('\n');
let mut line = SemanticLine::new(lines.next().unwrap());
let mut row = 0usize;
let mut start = 0usize;
let mut previous_end = 0usize;
let mut tokens = Vec::with_capacity(count);
for chunk in data.chunks_exact(5) {
let integer = |index: usize| -> Result<usize, String> {
chunk[index]
.as_u64()
.filter(|value| *value <= MAX_VERSION)
.map(|value| value as usize)
.ok_or_else(|| "invalid semantic token integer".into())
};
let delta_row = integer(0)?;
let delta_start = integer(1)?;
let length = integer(2)?;
let kind = integer(3)?;
let mask = integer(4)? as u32;
let next_row = row
.checked_add(delta_row)
.filter(|value| *value <= MAX_VERSION as usize)
.ok_or("semantic token row overflow")?;
if delta_row != 0 {
for _ in row..next_row {
line =
SemanticLine::new(lines.next().ok_or("semantic token row is out of bounds")?);
}
row = next_row;
start = delta_start;
previous_end = 0;
} else {
start = start
.checked_add(delta_start)
.filter(|value| *value <= MAX_VERSION as usize)
.ok_or("semantic token column overflow")?;
}
let end = start
.checked_add(length)
.ok_or("semantic token length overflow")?;
let token_kind = options
.types
.get(kind)
.filter(|_| kind < 65536)
.ok_or("semantic token type is outside its legend")?;
if mask & !options.modifier_mask != 0 {
return Err("semantic token modifier is outside its legend".into());
}
if start < previous_end {
return Err("overlapping semantic tokens are unsupported".into());
}
let start_column = line.column(start, false)?;
let end_column = line.column(end, true)?;
previous_end = end.min(line.units);
if length == 0 || start_column >= end_column {
continue;
}
if let Some(kind) = token_kind {
let mut modifiers = 0;
let mut remaining = mask;
while remaining != 0 {
modifiers |= options.modifiers[remaining.trailing_zeros() as usize];
remaining &= remaining - 1;
}
tokens.push(SemanticToken {
start: TextPosition {
row,
column: start_column,
},
end: TextPosition {
row,
column: end_column,
},
kind: *kind,
modifiers,
});
}
}
Ok((!tokens.is_empty()).then(|| tokens.into()))
}
fn sync_options(value: &Value) -> Result<SyncOptions, String> {
if let Some(kind) = value.as_u64().filter(|kind| *kind <= 2) {
return Ok(SyncOptions {
@@ -2064,6 +2516,67 @@ fn uri_path(uri: &str) -> Result<PathBuf, String> {
mod tests {
use super::*;
#[test]
fn semantic_decoder_validates_raw_coordinates_before_grapheme_rounding() {
let options = semantic_options(Some(&json!({
"full": true,
"legend": {"tokenTypes": ["custom", "function"],
"tokenModifiers": ["custom", "readonly", "declaration", "async"]}
})))
.unwrap()
.unwrap();
let tokens = decode_semantic_tokens(
"A😀e\u{301}Z\r\n\r\nx",
&json!({"data": [
0, 1, 2, 1, 2,
0, 2, 1, 1, 12,
0, 1, 1, 1, 0,
0, 1, 999, 0, 0,
1, 0, 1, 1, 0,
1, 0, 0, 1, 0,
0, 0, 100, 1, 0
]}),
&options,
)
.unwrap()
.unwrap();
assert_eq!(tokens.len(), 4);
assert_eq!(tokens[0].start, TextPosition { row: 0, column: 1 });
assert_eq!(tokens[0].end, TextPosition { row: 0, column: 2 });
assert_eq!(tokens[0].modifiers, SemanticTokenModifier::Readonly as u16);
assert_eq!(tokens[1].start, TextPosition { row: 0, column: 2 });
assert_eq!(tokens[1].end, TextPosition { row: 0, column: 3 });
assert_eq!(
tokens[1].modifiers,
SemanticTokenModifier::Declaration as u16 | SemanticTokenModifier::Async as u16
);
assert_eq!(tokens[2].start, tokens[1].start);
assert_eq!(tokens[3].end, TextPosition { row: 2, column: 1 });
for data in [
json!([0, 0, 1, 1]),
json!([-1, 0, 1, 1, 0]),
json!([0, 0.5, 1, 1, 0]),
json!([0, 0, 1, 1, 2147483648u64]),
json!([0, 0, 1, 65536, 0]),
json!([0, 0, 1, 2, 0]),
json!([0, 0, 1, 1, 16]),
json!([0, 2, 1, 0, 0]),
json!([0, 1, 1, 1, 0]),
json!([0, 7, 1, 1, 0]),
json!([2, 0, 1, 1, 0]),
json!([0, 1, 2, 1, 0, 0, 1, 1, 1, 0]),
] {
assert!(
decode_semantic_tokens("A😀e\u{301}Z\n", &json!({"data": data}), &options).is_err(),
"{data}"
);
}
let count = MAX_BODY / std::mem::size_of::<SemanticToken>() + 1;
assert!(
decode_semantic_tokens("x", &json!({"data": vec![0; count * 5]}), &options).is_err()
);
}
#[test]
fn unicode_positions_and_local_uri_boundaries() {
let text = "A😀e\u{301}Z\n";