feat: enable LSP semantic token highlighting by default

src/editor.rs

@@ -1,6 +1,6 @@
 use crate::clipboard::{ClipboardMode, SystemClipboard};
 use crate::files::{DockSide, FilePicker, FilePresentation, FileView};
-use crate::plugin::{Diagnostic, TextPosition};
+use crate::plugin::{Diagnostic, SemanticToken, TextPosition};
 use serde::Serialize;
 use std::borrow::Cow;
 use std::collections::{BTreeMap, HashMap, HashSet};
@@ -10,6 +10,7 @@ use std::io::{self, Write};
 #[cfg(unix)]
 use std::os::unix::fs::OpenOptionsExt;
 use std::path::{Component, Path, PathBuf};
+use std::sync::Arc;
 use std::sync::atomic::{AtomicU64, Ordering};
 use unicode_segmentation::{GraphemeCursor, UnicodeSegmentation};
 use unicode_width::UnicodeWidthStr;
@@ -773,6 +774,7 @@ struct Buffer {
     text_revision: u64,
     save_revision: u64,
     diagnostics: BTreeMap<String, AnnotationSet>,
+    semantic_tokens: Option<Arc<[SemanticToken]>>,
     targets: Vec<Target>,
     primary: usize,
     mode: Mode,
@@ -814,6 +816,7 @@ impl Buffer {
             text_revision: 0,
             save_revision: 0,
             diagnostics: BTreeMap::new(),
+            semantic_tokens: None,
             targets: vec![Target::caret(Cursor::default())],
             primary: 0,
             mode: Mode::Normal,
@@ -855,6 +858,7 @@ impl Buffer {
     fn text_changed(&mut self) {
         self.text_revision = self.text_revision.wrapping_add(1);
         self.diagnostics.clear();
+        self.semantic_tokens = None;
     }
 }
 
@@ -1074,6 +1078,101 @@ impl Editor {
         }
     }
 
+    pub fn semantic_tokens(&self) -> &[SemanticToken] {
+        self.active.semantic_tokens.as_deref().unwrap_or_default()
+    }
+
+    pub(crate) fn validate_semantic_tokens(
+        &self,
+        buffer_id: usize,
+        revision: u64,
+        tokens: Option<&[SemanticToken]>,
+    ) -> Result<bool, String> {
+        let Some(buffer) = self.buffer(buffer_id) else {
+            return Ok(false);
+        };
+        if buffer.text_revision != revision {
+            return Ok(false);
+        }
+        let Some(tokens) = tokens else {
+            return Ok(true);
+        };
+        if tokens.len() > MAX_EDIT_BYTES / std::mem::size_of::<SemanticToken>() {
+            return Err("Semantic tokens exceed 4 MiB".into());
+        }
+        let mut previous = None;
+        let mut row_length = None;
+        for token in tokens {
+            if previous.is_some_and(|start| token.start < start) {
+                return Err("Semantic token starts must be sorted".into());
+            }
+            if token.start.row != token.end.row || token.start >= token.end {
+                return Err("Semantic tokens must have nonempty single-line ranges".into());
+            }
+            if token.modifiers & !1023 != 0 {
+                return Err("Semantic token modifiers are not canonical".into());
+            }
+            if row_length.is_none_or(|(row, _)| row != token.start.row) {
+                let line = buffer
+                    .lines
+                    .get(token.start.row)
+                    .ok_or("Semantic token position is outside the document")?;
+                row_length = Some((token.start.row, grapheme_count(line)));
+            }
+            if token.end.column > row_length.unwrap().1 {
+                return Err("Semantic token position is outside the document".into());
+            }
+            previous = Some(token.start);
+        }
+        Ok(true)
+    }
+
+    pub fn replace_semantic_tokens(
+        &mut self,
+        buffer_id: usize,
+        revision: u64,
+        tokens: Option<Arc<[SemanticToken]>>,
+    ) -> Result<bool, String> {
+        if !self.validate_semantic_tokens(buffer_id, revision, tokens.as_deref())? {
+            return Ok(false);
+        }
+        Ok(self.replace_validated_semantic_tokens(buffer_id, revision, tokens))
+    }
+
+    /// Only session's fully prevalidated batch may use this storage path.
+    pub(crate) fn replace_validated_semantic_tokens(
+        &mut self,
+        buffer_id: usize,
+        revision: u64,
+        tokens: Option<Arc<[SemanticToken]>>,
+    ) -> bool {
+        let buffer = if buffer_id == self.active.id {
+            &mut self.active
+        } else if let Some(buffer) = self.parked.get_mut(&buffer_id) {
+            buffer
+        } else {
+            return false;
+        };
+        if buffer.text_revision != revision {
+            return false;
+        }
+        let tokens = tokens.filter(|tokens| !tokens.is_empty());
+        match (&buffer.semantic_tokens, &tokens) {
+            (None, None) => return false,
+            (Some(current), Some(next)) if Arc::ptr_eq(current, next) => return false,
+            _ => {}
+        }
+        buffer.semantic_tokens = tokens;
+        true
+    }
+
+    pub fn clear_semantic_tokens(&mut self) {
+        self.active.semantic_tokens = None;
+        for buffer in self.parked.values_mut() {
+            buffer.semantic_tokens = None;
+        }
+    }
+
     pub fn diagnostics(&self) -> impl Iterator<Item = &Diagnostic> {
         self.active
             .diagnostics
@@ -2754,6 +2853,9 @@ impl Editor {
             Ok(()) => {
                 self.active.saved_lines.clone_from(&self.active.lines);
                 self.active.save_revision = self.active.save_revision.wrapping_add(1);
+                if self.active.path.as_ref() != Some(&path) {
+                    self.active.semantic_tokens = None;
+                }
                 self.active.path = Some(path);
                 self.active.identity = Some(identity);
                 self.set_message("Written");
@@ -4316,6 +4418,155 @@ mod tests {
         editor
     }
 
+    #[test]
+    fn semantic_snapshots_are_borrowed_buffer_local_and_invalidated_by_text_and_path() {
+        use crate::plugin::SemanticTokenKind;
+        let directory = Directory::new();
+        let a = directory.path("a.py");
+        let b = directory.path("b.py");
+        fs::write(&a, "e\u{301}界").unwrap();
+        fs::write(&b, "other").unwrap();
+        let mut editor = Editor::open(Some(a.clone()), vec![]).unwrap();
+        editor.set_clipboard_mode("separate").unwrap();
+        let tokens: Arc<[SemanticToken]> = Arc::from([SemanticToken {
+            start: TextPosition { row: 0, column: 0 },
+            end: TextPosition { row: 0, column: 2 },
+            kind: SemanticTokenKind::Variable,
+            modifiers: 1023,
+        }]);
+        assert!(
+            editor
+                .replace_semantic_tokens(1, 0, Some(tokens.clone()))
+                .unwrap()
+        );
+        assert!(std::ptr::eq(editor.semantic_tokens(), tokens.as_ref()));
+        assert!(!editor.dirty());
+        assert!(editor.active.undo.is_empty());
+        editor.execute(&format!("edit {}", b.display()));
+        assert!(editor.semantic_tokens().is_empty());
+        assert!(
+            editor
+                .replace_semantic_tokens(2, 0, Some(tokens.clone()))
+                .unwrap()
+        );
+        editor.execute("buffer 1");
+        assert!(std::ptr::eq(editor.semantic_tokens(), tokens.as_ref()));
+        assert!(
+            !editor
+                .replace_semantic_tokens(1, 0, Some(tokens.clone()))
+                .unwrap()
+        );
+        editor.execute("w");
+        assert_eq!(editor.semantic_tokens(), tokens.as_ref());
+        editor.execute(&format!("w {}", directory.path("renamed.py").display()));
+        assert_eq!(editor.text_revision(), 0);
+        assert!(editor.semantic_tokens().is_empty());
+        assert!(
+            editor
+                .replace_semantic_tokens(1, 0, Some(tokens.clone()))
+                .unwrap()
+        );
+        editor.handle(Input::Char('x'));
+        assert!(editor.semantic_tokens().is_empty());
+        editor.handle(Input::Char('u'));
+        assert_eq!(editor.lines(), ["e\u{301}界"]);
+        assert!(editor.semantic_tokens().is_empty());
+        assert!(
+            !editor
+                .replace_semantic_tokens(1, 0, Some(tokens.clone()))
+                .unwrap()
+        );
+        editor.execute("buffer 2");
+        assert_eq!(editor.semantic_tokens(), tokens.as_ref());
+        editor.clear_semantic_tokens();
+        assert!(editor.semantic_tokens().is_empty());
+        editor.execute("buffer 1");
+        assert!(editor.semantic_tokens().is_empty());
+    }
+
+    #[test]
+    fn semantic_validation_rejects_malformed_live_tokens_but_ignores_stale_and_unloaded() {
+        use crate::plugin::SemanticTokenKind;
+        let mut editor = editor();
+        editor
+            .apply(vec![Effect::Replace("e\u{301}界\nx".into())])
+            .unwrap();
+        let revision = editor.text_revision();
+        let token = SemanticToken {
+            start: TextPosition { row: 0, column: 0 },
+            end: TextPosition { row: 0, column: 2 },
+            kind: SemanticTokenKind::Variable,
+            modifiers: 0,
+        };
+        let invalid = [
+            SemanticToken {
+                end: TextPosition { row: 0, column: 3 },
+                ..token
+            },
+            SemanticToken {
+                end: token.start,
+                ..token
+            },
+            SemanticToken {
+                end: TextPosition { row: 1, column: 1 },
+                ..token
+            },
+            SemanticToken {
+                start: TextPosition { row: 9, column: 0 },
+                end: TextPosition { row: 9, column: 1 },
+                ..token
+            },
+            SemanticToken {
+                modifiers: 1024,
+                ..token
+            },
+        ];
+        for token in invalid {
+            assert!(
+                editor
+                    .validate_semantic_tokens(1, revision, Some(&[token]))
+                    .is_err()
+            );
+            assert!(
+                !editor
+                    .validate_semantic_tokens(1, revision - 1, Some(&[token]))
+                    .unwrap()
+            );
+            assert!(
+                !editor
+                    .validate_semantic_tokens(999, revision, Some(&[token]))
+                    .unwrap()
+            );
+        }
+        let later = SemanticToken {
+            start: TextPosition { row: 0, column: 1 },
+            ..token
+        };
+        assert!(
+            editor
+                .validate_semantic_tokens(1, revision, Some(&[later, token]))
+                .is_err()
+        );
+        // Raw adjacent LSP tokens may map outward to the same grapheme.
+        assert!(
+            editor
+                .validate_semantic_tokens(1, revision, Some(&[token, token, later]))
+                .unwrap()
+        );
+        let oversized = vec![token; MAX_EDIT_BYTES / std::mem::size_of::<SemanticToken>() + 1];
+        assert!(
+            editor
+                .validate_semantic_tokens(1, revision, Some(&oversized))
+                .is_err()
+        );
+        let tokens: Arc<[SemanticToken]> = Arc::from([token]);
+        editor
+            .replace_semantic_tokens(1, revision, Some(tokens))
+            .unwrap();
+        assert!(editor.replace_semantic_tokens(1, revision, None).unwrap());
+        assert!(editor.semantic_tokens().is_empty());
+    }
+
     #[test]
     fn hover_scroll_and_escape_preserve_carets_and_edits_dismiss_normally() {
         let mut editor = editor();