fix: retain semantic colors across editable diff updates

src/editor.rs

@@ -1861,11 +1861,10 @@ impl Buffer {
         {
             return Ok(false);
         }
-        self.lines = split_lines(&text);
+        self.replace_lines(split_lines(&text));
         self.saved_lines.clone_from(&self.lines);
         self.undo.clear();
         self.redo.clear();
-        self.text_changed();
         self.view.reload(&self.lines);
         Ok(true)
     }
@@ -1893,10 +1892,17 @@ impl Buffer {
         }
     }
 
-    fn text_changed(&mut self) {
-        self.text_revision = self.text_revision.wrapping_add(1);
-        self.diagnostics.clear();
-        self.semantic_tokens = None;
+    /// Keep unaffected semantic colors until the server supplies this revision's tokens.
+    fn replace_lines(&mut self, lines: Vec<String>) -> Vec<String> {
+        if self.lines != lines {
+            if let Some(tokens) = &mut self.semantic_tokens {
+                retain_semantic_tokens(tokens, &self.lines, &lines);
+            }
+            self.text_revision = self.text_revision.wrapping_add(1);
+            self.diagnostics.clear();
+            self.view.search_match = None;
+        }
+        std::mem::replace(&mut self.lines, lines)
     }
 }
 
@@ -2455,8 +2461,9 @@ impl Editor {
                             };
                             let seed = split_lines(projection.seed());
                             if buffer.lines != seed {
+                                let previous = buffer.replace_lines(seed);
                                 buffer.undo.push(Snapshot {
-                                    lines: std::mem::replace(&mut buffer.lines, seed),
+                                    lines: previous,
                                     targets: if buffer.view.targets.is_empty() {
                                         vec![Target::caret(Cursor::default())]
                                     } else {
@@ -2465,7 +2472,6 @@ impl Editor {
                                     primary: buffer.view.primary,
                                 });
                                 buffer.redo.clear();
-                                buffer.text_changed();
                                 buffer.view.clamp(&buffer.lines);
                                 for pane in self.panes.values_mut() {
                                     if let Some(view) = pane.views.get_mut(&draft_buffer) {
@@ -2526,8 +2532,7 @@ impl Editor {
                         } {
                             if let Some(saved) = saved_text {
                                 if !was_text && !buffer.dirty() {
-                                    buffer.lines.clone_from(&saved);
-                                    buffer.text_changed();
+                                    buffer.replace_lines(saved.clone());
                                     buffer.view.clamp(&buffer.lines);
                                 }
                                 buffer.saved_lines = saved;
@@ -2535,12 +2540,11 @@ impl Editor {
                                 buffer.path = Some(target_path);
                                 buffer.resolver_readonly = false;
                             } else if !buffer.dirty() {
-                                buffer.lines = vec!["Non-text resolution saved; reopen the file to inspect its current type.".into()];
+                                buffer.replace_lines(vec!["Non-text resolution saved; reopen the file to inspect its current type.".into()]);
                                 buffer.saved_lines.clone_from(&buffer.lines);
                                 buffer.path = None;
                                 buffer.identity = None;
                                 buffer.resolver_readonly = true;
-                                buffer.text_changed();
                                 buffer.view.clamp(&buffer.lines);
                             }
                             buffer.disk_stamp = buffer
@@ -2832,9 +2836,8 @@ impl Editor {
                     self.parked.get_mut(&alternative_buffer).unwrap()
                 };
                 if buffer.lines != alternative {
-                    buffer.lines.clone_from(&alternative);
+                    buffer.replace_lines(alternative.clone());
                     buffer.saved_lines = alternative;
-                    buffer.text_changed();
                     buffer.view.clamp(&buffer.lines);
                     for pane in self.panes.values_mut() {
                         if let Some(view) = pane.views.get_mut(&alternative_buffer) {
@@ -5913,10 +5916,9 @@ impl Editor {
         if state.popup.selected == 0 {
             self.snippet = None;
             if self.active.lines != state.original.lines {
-                self.active.text_changed();
+                self.active.replace_lines(state.original.lines.clone());
                 self.active.view.search_match = None;
             }
-            self.active.lines.clone_from(&state.original.lines);
             self.clamp_other_views();
             self.active.view.targets.clone_from(&state.original.targets);
             self.active.view.primary = state.original.primary;
@@ -9515,11 +9517,7 @@ impl Editor {
             match effect {
                 Effect::Insert(text) => self.insert_text(&normalize_newlines(&text)),
                 Effect::Replace(text) => {
-                    let lines = split_lines(&text);
-                    if lines != self.active.lines {
-                        self.active.text_changed();
-                    }
-                    self.active.lines = lines;
+                    self.active.replace_lines(split_lines(&text));
                     self.clamp_other_views();
                     self.active.view.search_match = None;
                     self.normalize_targets();
@@ -10991,7 +10989,7 @@ impl Editor {
         let search_match = self.active.view.search_match;
         let revision = self.active.text_revision;
         let diagnostics = std::mem::take(&mut self.active.diagnostics);
-        let semantic_tokens = self.active.semantic_tokens.take();
+        let semantic_tokens = self.active.semantic_tokens.clone();
         let mut restored = false;
         let result = self
             .replay_edit(&recipe)
@@ -11288,10 +11286,7 @@ impl Editor {
         if self.mutation_blocked() {
             return;
         }
-        if self.active.lines != snapshot.lines {
-            self.active.text_changed();
-        }
-        self.active.lines = snapshot.lines;
+        self.active.replace_lines(snapshot.lines);
         self.clamp_other_views();
         self.active.view.targets = snapshot.targets;
         self.active.view.primary = snapshot.primary;
@@ -11666,18 +11661,11 @@ impl Editor {
                 }
             }
         }
-        if !text
-            .split('\n')
-            .eq(self.active.lines.iter().map(String::as_str))
-        {
-            self.active.text_changed();
-            self.active.view.search_match = None;
-        }
         if let Some(original) = original {
             self.active.view.targets.clone_from(&original.targets);
             self.active.view.primary = original.primary;
         }
-        self.active.lines = new_lines;
+        self.active.replace_lines(new_lines);
         let offsets = new_offsets;
         for (target, (byte, round_up)) in self.active.view.targets.iter_mut().zip(destinations) {
             target.active = offset_cursor(&self.active.lines, &offsets, byte, round_up);
@@ -12055,6 +12043,102 @@ fn grapheme_cursor(lines: &[String], mut index: usize) -> Option<Cursor> {
     None
 }
 
+fn document_graphemes(lines: &[String]) -> impl DoubleEndedIterator<Item = &str> {
+    lines.iter().enumerate().flat_map(move |(row, line)| {
+        line.graphemes(true)
+            .chain((row + 1 < lines.len()).then_some("\n"))
+    })
+}
+
+fn retain_semantic_tokens(tokens: &mut Arc<[SemanticToken]>, before: &[String], after: &[String]) {
+    // Skip whole unchanged lines before doing Unicode segmentation at the edit boundaries.
+    let first = before
+        .iter()
+        .zip(after)
+        .take_while(|(left, right)| left == right)
+        .count()
+        .min(before.len() - 1)
+        .min(after.len() - 1);
+    let suffix = before[first..]
+        .iter()
+        .rev()
+        .zip(after[first..].iter().rev())
+        .take_while(|(left, right)| left == right)
+        .count();
+    // Include one common suffix line so its preceding newline remains part of the comparison.
+    let before = &before[first..before.len() - suffix.saturating_sub(1)];
+    let after = &after[first..after.len() - suffix.saturating_sub(1)];
+    let mut old = document_graphemes(before).peekable();
+    let mut new = document_graphemes(after).peekable();
+    let mut start = TextPosition {
+        row: first,
+        column: 0,
+    };
+    while let (Some(left), Some(right)) = (old.peek(), new.peek()) {
+        if left != right {
+            break;
+        }
+        if *left == "\n" {
+            start.row += 1;
+            start.column = 0;
+        } else {
+            start.column += 1;
+        }
+        old.next();
+        new.next();
+    }
+    let end = |lines: &[String]| TextPosition {
+        row: first + lines.len() - 1,
+        column: grapheme_count(lines.last().unwrap()),
+    };
+    let mut old_end = end(before);
+    let mut new_end = end(after);
+    for (left, right) in old.rev().zip(new.rev()) {
+        if left != right {
+            break;
+        }
+        for (position, lines) in [(&mut old_end, before), (&mut new_end, after)] {
+            if left == "\n" {
+                position.row -= 1;
+                position.column = grapheme_count(&lines[position.row - first]);
+            } else {
+                position.column -= 1;
+            }
+        }
+    }
+    if tokens
+        .iter()
+        .all(|token| token.end <= start || (token.start >= old_end && old_end == new_end))
+    {
+        return;
+    }
+    let shift = |position: TextPosition| TextPosition {
+        row: new_end.row + (position.row - old_end.row),
+        column: if position.row == old_end.row {
+            new_end.column + (position.column - old_end.column)
+        } else {
+            position.column
+        },
+    };
+    // ponytail: one changed range; retain between disjoint edits with per-edit mapping if needed.
+    *tokens = tokens
+        .iter()
+        .filter_map(|token| {
+            if token.end <= start {
+                Some(*token)
+            } else if token.start >= old_end {
+                Some(SemanticToken {
+                    start: shift(token.start),
+                    end: shift(token.end),
+                    ..*token
+                })
+            } else {
+                None
+            }
+        })
+        .collect();
+}
+
 fn translate_edit_offset<'a, 'b>(
     position: usize,
     after: bool,
@@ -16171,8 +16255,9 @@ mod tests {
             false,
         )
         .unwrap();
-        live.lines[1] = "dirty after".into();
-        live.text_changed();
+        let mut lines = live.lines.clone();
+        lines[1] = "dirty after".into();
+        live.replace_lines(lines);
         let live_id = live.id;
         editor.next_buffer_id += 1;
         editor.parked.insert(live_id, live);
@@ -17709,8 +17794,11 @@ mod tests {
         editor
             .navigate_to(b, TextPosition { row: 0, column: 0 })
             .unwrap();
-        editor.parked.get_mut(&1).unwrap().lines = vec!["中👩‍💻".into()];
-        editor.parked.get_mut(&1).unwrap().text_changed();
+        editor
+            .parked
+            .get_mut(&1)
+            .unwrap()
+            .replace_lines(vec!["中👩‍💻".into()]);
         editor.handle(Input::Ctrl('o'));
         assert_eq!(editor.cursor(), Cursor { row: 0, column: 1 });
         assert!(editor.dirty());
@@ -18595,6 +18683,116 @@ mod tests {
         assert!(editor.semantic_tokens().is_empty());
     }
 
+    #[test]
+    fn semantic_colors_follow_untouched_text_and_drop_overlapping_edits() {
+        use crate::plugin::SemanticTokenKind;
+        let position = |(row, column)| TextPosition { row, column };
+        for (before, after, start, end, expected) in [
+            (
+                "a value z",
+                "aa value z",
+                (0, 2),
+                (0, 7),
+                Some(((0, 3), (0, 8))),
+            ),
+            (
+                "a value z",
+                "value z",
+                (0, 2),
+                (0, 7),
+                Some(((0, 0), (0, 5))),
+            ),
+            (
+                "a value z",
+                "a \nvalue z",
+                (0, 2),
+                (0, 7),
+                Some(((1, 0), (1, 5))),
+            ),
+            (
+                "a\nvalue z",
+                "avalue z",
+                (1, 0),
+                (1, 5),
+                Some(((0, 1), (0, 6))),
+            ),
+            (
+                "e\u{301}value",
+                "e\u{301}界value",
+                (0, 1),
+                (0, 6),
+                Some(((0, 2), (0, 7))),
+            ),
+            (
+                "x\nvalue",
+                "x\n👩‍🚀value",
+                (1, 0),
+                (1, 5),
+                Some(((1, 1), (1, 6))),
+            ),
+            (
+                "head\nvalue\ntail\nlast",
+                "head\n界\nvalue\ntail\nlast",
+                (1, 0),
+                (1, 5),
+                Some(((2, 0), (2, 5))),
+            ),
+            (
+                "head\n界\nvalue\ntail",
+                "head\nvalue\ntail",
+                (2, 0),
+                (2, 5),
+                Some(((1, 0), (1, 5))),
+            ),
+            (
+                "value z",
+                "value \nz",
+                (0, 0),
+                (0, 5),
+                Some(((0, 0), (0, 5))),
+            ),
+            ("value", "value!", (0, 0), (0, 5), Some(((0, 0), (0, 5)))),
+            ("value", "valXue", (0, 0), (0, 5), None),
+            ("value", "", (0, 0), (0, 5), None),
+            ("value", "VALUE", (0, 0), (0, 5), None),
+            ("evalue", "e\u{301}value", (0, 0), (0, 1), None),
+        ] {
+            let mut editor = editor();
+            editor.apply(vec![Effect::Replace(before.into())]).unwrap();
+            let revision = editor.text_revision();
+            let token = SemanticToken {
+                start: position(start),
+                end: position(end),
+                kind: SemanticTokenKind::Function,
+                modifiers: 5,
+            };
+            let original: Arc<[SemanticToken]> = Arc::from([token]);
+            editor
+                .replace_semantic_tokens(editor.buffer_id(), revision, Some(original.clone()))
+                .unwrap();
+            editor.apply(vec![Effect::Replace(after.into())]).unwrap();
+            let expected: Vec<_> = expected
+                .map(|(start, end)| SemanticToken {
+                    start: position(start),
+                    end: position(end),
+                    ..token
+                })
+                .into_iter()
+                .collect();
+            assert_eq!(
+                editor.semantic_tokens(),
+                expected,
+                "{before:?} -> {after:?}"
+            );
+            assert!(
+                !editor
+                    .replace_semantic_tokens(editor.buffer_id(), revision, Some(original))
+                    .unwrap()
+            );
+            assert_eq!(editor.semantic_tokens(), expected);
+        }
+    }
+
     #[test]
     fn semantic_validation_rejects_malformed_live_tokens_but_ignores_stale_and_unloaded() {
         use crate::plugin::SemanticTokenKind;