hl-valdemar/vamp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

perf: avoid eager diff loading and full-file typing work

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-10-11 15:06:05 +02:00
parent 7b4bc80538
commit 2e05725a38
11 changed files with 1881 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/repository.rs
+297 -122
View File
@@ -297,10 +297,18 @@ struct Remote {
struct FileData {
mode: String,
bytes: Vec<u8>,
object: Option<(PathBuf, String)>,
notice: Option<String>,
}
type Snapshot = BTreeMap<PathBuf, FileData>;
struct PreparedComparison {
id: u64,
left: Snapshot,
right: Snapshot,
files: Vec<(PathBuf, Option<PathBuf>, FileStatus)>,
}
pub struct Repository {
root: PathBuf,
kind: RepoKind,
@@ -311,6 +319,7 @@ pub struct Repository {
remote_refs: BTreeMap<(String, String), String>,
loaded: BTreeSet<String>,
registered: BTreeSet<String>,
comparison: Option<PreparedComparison>,
cancelled: Arc<AtomicBool>,
}
impl Repository {
@@ -348,6 +357,7 @@ impl Repository {
remote_refs: BTreeMap::new(),
loaded: BTreeSet::new(),
registered: BTreeSet::new(),
comparison: None,
cancelled,
})
}
@@ -562,42 +572,53 @@ impl Repository {
self.revisions(root, store, &mut entries, &mut notices)?;
}
for remote in &self.remotes {
entries.push(Revision {
target: RevisionTarget::FetchRemote(remote.name.clone()),
label: format!("Load history from {}", remote.name),
description: "Fetch reachable history into a private comparison repository".into(),
category,
});
match self.advertised(&remote.url) {
Ok(refs) => {
self.remote_refs.retain(|(name, _), _| name != &remote.name);
for (reference, id) in refs {
let tag = reference.starts_with("refs/tags/");
let name = reference
.strip_prefix(if tag { "refs/tags/" } else { "refs/heads/" })
.unwrap();
if !tag || self.kind == RepoKind::Git {
entries.push(Revision {
target: RevisionTarget::Remote {
remote: remote.name.clone(),
reference: reference.clone(),
},
label: format!("{name}@{}", remote.name),
description: id.clone(),
category: if tag {
Category::Tags
} else if self.kind == RepoKind::Jujutsu {
Category::Bookmarks
} else {
Category::Branches
},
});
}
self.remote_refs
.insert((remote.name.clone(), reference), id);
for &view in self.kind.categories() {
let history = view == category;
entries.push(Revision {
target: if history {
RevisionTarget::FetchRemote(remote.name.clone())
} else {
RevisionTarget::ListRemote(remote.name.clone())
},
label: format!(
"Load {} from {}",
if history { "history" } else { "refs" },
remote.name
),
description: if history {
"Fetch reachable history into a private comparison repository"
} else {
"List remote references without fetching history"
}
.into(),
category: view,
});
}
for ((name, reference), id) in &self.remote_refs {
if name != &remote.name {
continue;
}
let tag = reference.starts_with("refs/tags/");
let label = reference
.strip_prefix(if tag { "refs/tags/" } else { "refs/heads/" })
.unwrap();
if !tag || self.kind == RepoKind::Git {
entries.push(Revision {
target: RevisionTarget::Remote {
remote: name.clone(),
reference: reference.clone(),
},
label: format!("{label}@{name}"),
description: id.clone(),
category: if tag {
Category::Tags
} else if self.kind == RepoKind::Jujutsu {
Category::Bookmarks
} else {
Category::Branches
},
});
}
Err(e) => notices.push(format!("{}: {e}", remote.name)),
}
}
self.check_cancelled()?;
@@ -645,7 +666,7 @@ impl Repository {
}
Ok(())
}
fn load_remote(&mut self, name: &str) -> Result<(), String> {
fn load_remote(&mut self, name: &str, reference: Option<&str>) -> Result<(), String> {
if self.remotes.is_empty() {
self.discover_remotes()?;
}
@@ -674,7 +695,27 @@ impl Repository {
"--remote",
&format!("exact:{name}"),
"--branch",
"glob:*",
&reference.map_or_else(
|| Ok("glob:*".to_owned()),
|reference| {
reference
.strip_prefix("refs/heads/")
.map(|branch| format!("exact:{branch}"))
.ok_or("Jujutsu remote selection requires a bookmark")
},
)?,
],
)?;
} else if let Some(reference) = reference {
self.git(
store,
&[
"fetch",
"--no-tags",
"--no-write-fetch-head",
"--",
&url,
&format!("+{reference}:refs/vamp-selected/{name}/{reference}"),
],
)?;
} else {
@@ -690,16 +731,33 @@ impl Repository {
],
)?;
}
self.loaded.insert(name.into());
let refs = self.advertised(&url)?;
self.remote_refs.retain(|(remote, _), _| remote != name);
for (reference, id) in refs {
self.remote_refs.insert((name.into(), reference), id);
if reference.is_none() {
self.loaded.insert(name.into());
let refs = self.advertised(&url)?;
self.remote_refs.retain(|(remote, _), _| remote != name);
for (reference, id) in refs {
self.remote_refs.insert((name.into(), reference), id);
}
}
Ok(())
}
pub fn list_remote(&mut self, name: &str) -> Result<Catalog, String> {
self.discover_remotes()?;
let remote = self
.remotes
.iter()
.find(|remote| remote.name == name)
.ok_or_else(|| format!("Unknown configured remote: {name}"))?;
let refs = self.advertised(&remote.url)?;
self.remote_refs.retain(|(remote, _), _| remote != name);
self.remote_refs.extend(
refs.into_iter()
.map(|(reference, id)| ((name.into(), reference), id)),
);
self.catalog()
}
pub fn fetch_remote(&mut self, remote: &str) -> Result<Catalog, String> {
self.load_remote(remote)?;
self.load_remote(remote, None)?;
self.catalog()
}
fn resolve(&mut self, target: &RevisionTarget) -> Result<(PathBuf, String), String> {
@@ -719,22 +777,33 @@ impl Repository {
}
id.clone()
}
RevisionTarget::Remote { remote, reference } => {
if !self.loaded.contains(remote) {
self.load_remote(remote)?;
}
self.remote_refs
.get(&(remote.clone(), reference.clone()))
.ok_or_else(|| {
format!("Remote reference is no longer advertised: {reference}@{remote}")
})?
.clone()
}
RevisionTarget::Remote { remote, reference } => self
.remote_refs
.get(&(remote.clone(), reference.clone()))
.ok_or_else(|| {
format!("Remote reference is no longer advertised: {reference}@{remote}")
})?
.clone(),
RevisionTarget::Working => return Err("Working checkout is not a commit".into()),
RevisionTarget::FetchRemote(_) => {
RevisionTarget::FetchRemote(_) | RevisionTarget::ListRemote(_) => {
return Err("Select a revision, not a history loader".into());
}
};
if let Some(resolved) = self.resolve_stored(&id)? {
return Ok(resolved);
}
if let RevisionTarget::Remote { remote, reference } = target {
self.load_remote(remote, Some(reference))?;
if let Some(resolved) = self.resolve_stored(&id)? {
return Ok(resolved);
}
}
Err(format!(
"Commit {id} is unavailable locally or in loaded remote history"
))
}
fn resolve_stored(&self, id: &str) -> Result<Option<(PathBuf, String)>, String> {
for store in
std::iter::once(&self.store).chain(self.private.as_ref().map(|(_, store)| store))
{
@@ -743,15 +812,13 @@ impl Repository {
.current_dir(&self.root);
let output = run(cmd, vec![], &self.cancelled)?;
if output.code == Some(0) {
return Ok((store.clone(), text(output.stdout)?.trim().into()));
return Ok(Some((store.clone(), text(output.stdout)?.trim().into())));
}
if !matches!(output.code, Some(1 | 128)) {
checked(output)?;
}
}
Err(format!(
"Commit {id} is unavailable locally or in loaded remote history"
))
Ok(None)
}
fn conflicted_snapshot(&self, root: &Path, id: &str) -> Result<Snapshot, String> {
let revset = format!("commit_id({})", serde_json::to_string(id).unwrap());
@@ -820,6 +887,7 @@ impl Repository {
FileData {
mode: mode.into(),
bytes,
object: None,
notice,
},
);
@@ -853,8 +921,7 @@ impl Repository {
}
}
let output = self.git(store, &["ls-tree", "-r", "-z", "--full-tree", id])?;
let mut entries = vec![];
let mut request = Vec::new();
let mut snapshot = BTreeMap::new();
for record in output.split(|b| *b == 0).filter(|r| !r.is_empty()) {
let tab = record
.iter()
@@ -868,32 +935,60 @@ impl Repository {
}
let path = path_bytes(&record[tab + 1..])?;
safe_path(&path)?;
if fields[1] == "commit" {
entries.push((
path,
fields[0].to_owned(),
let (object, notice) = if fields[1] == "commit" {
(
None,
Some(format!(
"Submodule commit {} (contents are not traversed)",
fields[2]
)),
));
)
} else if fields[1] == "blob" {
request.extend_from_slice(fields[2].as_bytes());
request.push(b'\n');
entries.push((path, fields[0].to_owned(), Some(fields[2].to_owned()), None));
(Some((store.to_path_buf(), fields[2].to_owned())), None)
} else {
return Err(format!("Unsupported tree entry type: {}", fields[1]));
}
};
snapshot.insert(
path,
FileData {
mode: fields[0].into(),
bytes: vec![],
object,
notice,
},
);
}
let mut cmd = git_command(store);
cmd.arg("cat-file").arg("--batch");
let data = checked(run(cmd, request, &self.cancelled)?)?;
let mut offset = 0;
let mut snapshot = BTreeMap::new();
for (path, mode, object, notice) in entries {
self.check_cancelled()?;
let bytes = if object.is_some() {
Ok(snapshot)
}
fn materialize(&self, snapshot: &mut Snapshot) -> Result<(), String> {
let stores: BTreeSet<_> = snapshot
.values()
.filter_map(|file| file.object.as_ref().map(|(store, _)| store.clone()))
.collect();
for store in stores {
let mut request = Vec::new();
for file in snapshot.values() {
if let Some((source, id)) = &file.object
&& source == &store
{
request.extend_from_slice(id.as_bytes());
request.push(b'\n');
}
}
let mut cmd = git_command(&store);
cmd.args(["cat-file", "--batch"]);
let data = checked(run(cmd, request, &self.cancelled)?)?;
let mut offset = 0;
for (path, file) in snapshot.iter_mut() {
if file
.object
.as_ref()
.is_none_or(|(source, _)| source != &store)
{
continue;
}
self.check_cancelled()?;
let end = offset
+ data[offset..]
.iter()
@@ -917,22 +1012,12 @@ impl Repository {
if size >= data.len().saturating_sub(offset) {
return Err("Truncated Git object content".into());
}
let bytes = data[offset..offset + size].to_vec();
file.bytes = data[offset..offset + size].to_vec();
file.object = None;
offset += size + 1;
bytes
} else {
vec![]
};
snapshot.insert(
path,
FileData {
mode,
bytes,
notice,
},
);
}
}
Ok(snapshot)
Ok(())
}
fn working_snapshot(&self, buffers: &[(PathBuf, String)]) -> Result<Snapshot, String> {
let mut tracked = BTreeSet::new();
@@ -1100,6 +1185,7 @@ impl Repository {
FileData {
mode: "160000".into(),
bytes: vec![],
object: None,
notice: Some(notice),
},
);
@@ -1178,6 +1264,7 @@ impl Repository {
FileData {
mode,
bytes,
object: None,
notice,
},
);
@@ -1192,38 +1279,72 @@ impl Repository {
self.commit_snapshot(&store, &id)
}
}
/// Freeze comparison sources and return path/status metadata only.
/// A new comparison replaces sources retained for the previous request ID.
pub fn compare(
&mut self,
comparison_id: u64,
before: &Revision,
after: &Revision,
buffers: &[(PathBuf, String)],
) -> Result<Comparison, String> {
let left = self.snapshot(&before.target)?;
self.comparison = None;
let mut left = self.snapshot(&before.target)?;
let working_after = after.target == RevisionTarget::Working;
let right = if before.target == after.target && (!working_after || buffers.is_empty()) {
None
let mut right = if before.target == after.target && (!working_after || buffers.is_empty()) {
BTreeMap::new()
} else if working_after {
Some(self.working_snapshot(buffers)?)
self.working_snapshot(buffers)?
} else {
Some(self.snapshot(&after.target)?)
self.snapshot(&after.target)?
};
let right = right.as_ref().unwrap_or(&left);
let same = before.target == after.target && (!working_after || buffers.is_empty());
// Frozen checkout/conflict bytes have no object identity. Compare them in a
// single batch, while immutable tree/tree comparisons need only object IDs.
let frozen = |file: &FileData| {
file.object.is_none() && matches!(file.mode.as_str(), "100644" | "100755" | "120000")
};
if !same && (left.values().any(frozen) || right.values().any(frozen)) {
self.materialize(&mut left)?;
self.materialize(&mut right)?;
}
let right_view = if same { &left } else { &right };
let mut files = vec![];
let paths: BTreeSet<_> = left.keys().chain(right.keys()).collect();
let paths: BTreeSet<_> = left.keys().chain(right_view.keys()).collect();
for path in paths {
self.check_cancelled()?;
let a = left.get(path);
let b = right.get(path);
if a == b {
let b = right_view.get(path);
if a.map(file_identity) == b.map(file_identity) {
continue;
}
files.push(self.changed_file(path, a, b)?);
files.push(ChangedFile {
path: path.clone(),
old_path: None,
status: if a.is_none() {
FileStatus::Added
} else if b.is_none() {
FileStatus::Removed
} else {
FileStatus::Modified
},
loaded: false,
before: vec![],
after: vec![],
before_eof: false,
after_eof: false,
hunks: vec![],
notice: None,
});
}
// Match only identical data; modified-content moves remain add/remove pairs.
let mut candidates: BTreeMap<&FileData, Vec<usize>> = BTreeMap::new();
let mut candidates = BTreeMap::new();
for (i, file) in files.iter().enumerate() {
if file.status == FileStatus::Removed {
candidates.entry(&left[&file.path]).or_default().push(i);
candidates
.entry(file_identity(&left[&file.path]))
.or_insert_with(Vec::new)
.push(i);
}
}
let mut removed = BTreeSet::new();
@@ -1232,18 +1353,11 @@ impl Repository {
continue;
}
if let Some(j) = candidates
.get_mut(&right[&files[i].path])
.get_mut(&file_identity(&right_view[&files[i].path]))
.and_then(Vec::pop)
{
let old_path = files[j].path.clone();
let mut renamed = self.changed_file(
&files[i].path,
left.get(&old_path),
right.get(&files[i].path),
)?;
renamed.status = FileStatus::Renamed;
renamed.old_path = Some(old_path);
files[i] = renamed;
files[i].status = FileStatus::Renamed;
files[i].old_path = Some(files[j].path.clone());
removed.insert(j);
}
}
@@ -1256,21 +1370,24 @@ impl Repository {
files
.iter()
.filter(|file| {
right.get(&file.path).is_some_and(|data| {
matches!(data.mode.as_str(), "100644" | "100755")
&& !data.bytes.contains(&0)
&& std::str::from_utf8(&data.bytes).is_ok()
&& file
.notice
.as_deref()
.is_none_or(|notice| !notice.contains("Binary or non-UTF-8"))
})
right_view
.get(&file.path)
.is_some_and(|data| matches!(data.mode.as_str(), "100644" | "100755"))
})
.map(|file| file.path.clone())
.collect()
} else {
BTreeSet::new()
};
self.comparison = Some(PreparedComparison {
id: comparison_id,
left,
right,
files: files
.iter()
.map(|file| (file.path.clone(), file.old_path.clone(), file.status))
.collect(),
});
Ok(Comparison {
before: before.label.clone(),
after: after.label.clone(),
@@ -1279,6 +1396,45 @@ impl Repository {
editable_paths,
})
}
/// Prepare one catalog entry using its original comparison's frozen sources.
pub fn load_file(
&mut self,
comparison_id: u64,
file_index: usize,
) -> Result<ChangedFile, String> {
self.check_cancelled()?;
let prepared = self
.comparison
.as_ref()
.filter(|comparison| comparison.id == comparison_id)
.ok_or("Comparison is no longer available")?;
let (path, old_path, status) = prepared
.files
.get(file_index)
.ok_or("Changed file is no longer available")?;
let before_path = old_path.as_ref().unwrap_or(path);
let mut left = BTreeMap::new();
let mut right = BTreeMap::new();
let before = prepared.left.get(before_path);
let after = prepared.right.get(path);
if let Some(data) = before.filter(|data| data.object.is_some()) {
left.insert(before_path.clone(), clone_file_data(data));
}
if let Some(data) = after.filter(|data| data.object.is_some()) {
right.insert(path.clone(), clone_file_data(data));
}
self.materialize(&mut left)?;
self.materialize(&mut right)?;
let mut file = self.changed_file(
path,
left.get(before_path).or(before),
right.get(path).or(after),
)?;
file.status = *status;
file.old_path = old_path.clone();
Ok(file)
}
fn changed_file(
&self,
path: &Path,
@@ -1337,6 +1493,7 @@ impl Repository {
path: path.to_path_buf(),
old_path: None,
status,
loaded: true,
before: before_lines,
after: after_lines,
before_eof: a.ends_with(b"\n"),
@@ -1350,6 +1507,24 @@ impl Repository {
})
}
}
fn file_identity(file: &FileData) -> (&str, Option<&str>, &[u8], Option<&str>) {
(
&file.mode,
file.object.as_ref().map(|(_, id)| id.as_str()),
&file.bytes,
file.notice.as_deref(),
)
}
fn clone_file_data(file: &FileData) -> FileData {
FileData {
mode: file.mode.clone(),
bytes: file.bytes.clone(),
object: file.object.clone(),
notice: file.notice.clone(),
}
}
fn lines(bytes: &[u8]) -> Vec<String> {
if bytes.is_empty() {
return vec![];