hl-valdemar/vamp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

fix: support large files in comparisons

Browse Source 
Replace the 16 MiB comparison rejection with independent bounded per-side loading and frozen Working blobs. Bound diff computation, defer large inline highlighting, index source-row navigation, and expose actual load failures. Verified the real 52.3 MB parser through repository/session APIs and terminal rendering with EOF/start navigation.
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-10-17 00:03:43 +02:00
parent adbd2c3f81
commit b9f29d258c
11 changed files with 986 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/repository.rs
+116 -113
View File
@@ -21,6 +21,10 @@ use std::{
const OUTPUT_LIMIT: usize = 64 * 1024 * 1024;
const FILE_LIMIT: usize = 16 * 1024 * 1024;
// Comparison payloads are independent of picker/gutter annotation budgets.
const COMPARISON_FILE_LIMIT: usize = 64 * 1024 * 1024;
const COMPARISON_LINE_LIMIT: usize = 4_000_000;
const WORKING_SNAPSHOT_LIMIT: usize = 512 * 1024 * 1024;
const TIMEOUT: Duration = Duration::from_secs(60);
static NEXT_TEMP: AtomicU64 = AtomicU64::new(0);
@@ -1948,94 +1952,54 @@ impl Repository {
}
fn materialize(&self, snapshot: &mut Snapshot) -> Result<(), String> {
let stores: BTreeSet<_> = snapshot
.values()
.filter(|file| file.projection.is_none())
.filter_map(|file| file.object.as_ref().map(|(store, _)| store.clone()))
.collect();
for store in stores {
let mut request = Vec::new();
for file in snapshot.values() {
if let Some((source, id)) = &file.object
&& source == &store
&& file.projection.is_none()
{
request.extend_from_slice(id.as_bytes());
request.push(b'\n');
}
}
let mut cmd = git_command(&store);
cmd.args(["cat-file", "--batch"]);
let data = checked(run(cmd, request, &self.cancelled)?)?;
let mut offset = 0;
for (path, file) in snapshot.iter_mut() {
if file.projection.is_some()
|| file
.object
.as_ref()
.is_none_or(|(source, _)| source != &store)
{
continue;
}
self.check_cancelled()?;
let end = offset
+ data[offset..]
.iter()
.position(|b| *b == b'\n')
.ok_or("Truncated Git object header")?;
let header = std::str::from_utf8(&data[offset..end])
.map_err(|_| "Invalid Git object header")?;
let size: usize = header
.rsplit(' ')
.next()
.ok_or("Missing Git object size")?
.parse()
for (path, file) in snapshot.iter_mut() {
self.check_cancelled()?;
if let Some((root, id)) = &file.projection {
let revset = format!("commit_id({})", serde_json::to_string(id).unwrap());
let pattern =
jj_file_pattern(path.to_str().ok_or("Non-UTF-8 Jujutsu conflict path")?);
file.bytes = self
.jj(
root,
&["file", "show", "-r", &revset, "-T", "", "--", &pattern],
)
.map_err(|error| format!("{}: {error}", path.display()))?;
} else if let Some((store, id)) = &file.object {
let size = text(self.git(store, &["cat-file", "-s", id])?)?
.trim()
.parse::<usize>()
.map_err(|_| "Invalid Git object size")?;
if size > FILE_LIMIT {
if size > COMPARISON_FILE_LIMIT {
return Err(format!(
"{} exceeds the 16 MiB comparison file limit",
"{} exceeds the 64 MiB per-side comparison file limit",
path.display()
));
}
offset = end + 1;
if size >= data.len().saturating_sub(offset) {
return Err("Truncated Git object content".into());
}
file.bytes = data[offset..offset + size].to_vec();
file.object = None;
offset += size + 1;
}
}
let mut total = 0;
for (path, file) in snapshot.iter_mut() {
let Some((root, id)) = &file.projection else {
// One blob per subprocess output: never concatenate both sides or
// copy a cat-file batch into a second allocation.
file.bytes = self.git(store, &["cat-file", "blob", id])?;
} else {
continue;
};
let revset = format!("commit_id({})", serde_json::to_string(id).unwrap());
let pattern = jj_file_pattern(path.to_str().ok_or("Non-UTF-8 Jujutsu conflict path")?);
let bytes = self.jj(
root,
&["file", "show", "-r", &revset, "-T", "", "--", &pattern],
)?;
if bytes.len() > FILE_LIMIT {
}
if file.bytes.len() > COMPARISON_FILE_LIMIT {
return Err(format!(
"{} exceeds the 16 MiB comparison file limit",
"{} exceeds the 64 MiB per-side comparison file limit",
path.display()
));
}
total += bytes.len();
if total > OUTPUT_LIMIT {
return Err(
"Jujutsu conflict projection exceeds the 64 MiB comparison limit".into(),
);
}
file.bytes = bytes;
file.object = None;
file.projection = None;
}
Ok(())
}
fn working_snapshot(&self, buffers: &[(PathBuf, String)]) -> Result<Snapshot, String> {
fn working_snapshot(&mut self, buffers: &[(PathBuf, String)]) -> Result<Snapshot, String> {
self.ensure_private()?;
let store = self.private.as_ref().unwrap().1.clone();
// Freeze each file once, then let Git ingest them in one batch. Keeping
// object identities makes catalog comparison lazy even against Working.
let frozen = Temporary::new()?;
let mut frozen_paths = Vec::new();
let mut hash_input = Vec::new();
let mut tracked = BTreeSet::new();
let mut submodules = BTreeSet::new();
if self.kind == RepoKind::Git {
@@ -2220,39 +2184,46 @@ impl Repository {
let bytes = target.to_string_lossy().as_bytes().to_vec();
("120000".into(), bytes, None)
} else if metadata.is_file() {
if metadata.len() > FILE_LIMIT as u64 {
if metadata.len() > COMPARISON_FILE_LIMIT as u64 {
return Err(format!(
"{} exceeds the 16 MiB comparison file limit",
"{} exceeds the 64 MiB per-side comparison file limit",
path.display()
));
}
let bytes = fs::File::open(&full)
.and_then(|f| {
let mut bytes = Vec::new();
f.take(FILE_LIMIT as u64 + 1).read_to_end(&mut bytes)?;
Ok(bytes)
})
.map_err(|e| format!("{}: {e}", path.display()))?;
if bytes.len() > FILE_LIMIT {
return Err(format!(
"{} grew beyond the 16 MiB comparison file limit",
path.display()
));
}
let bytes = if !bytes.contains(&0)
&& std::str::from_utf8(&bytes).is_ok()
&& let Some(text) = overlays.get(&path)
{
if text.len() > FILE_LIMIT {
let mut source = fs::File::open(&full)
.map_err(|e| format!("{}: {e}", path.display()))?
.take(COMPARISON_FILE_LIMIT as u64 + 1);
let mut bytes = Vec::with_capacity(metadata.len() as usize);
let mut chunk = [0; 64 * 1024];
loop {
self.check_cancelled()?;
let count = source
.read(&mut chunk)
.map_err(|e| format!("{}: {e}", path.display()))?;
if count == 0 {
break;
}
if bytes.len() + count > COMPARISON_FILE_LIMIT {
return Err(format!(
"{} exceeds the 16 MiB comparison file limit",
"{} grew beyond the 64 MiB per-side comparison file limit",
path.display()
));
}
text.as_bytes().to_vec()
} else {
bytes
};
bytes.extend_from_slice(&chunk[..count]);
}
if !bytes.contains(&0)
&& std::str::from_utf8(&bytes).is_ok()
&& let Some(text) = overlays.get(&path)
{
if text.len() > COMPARISON_FILE_LIMIT {
return Err(format!(
"{} exceeds the 64 MiB per-side comparison file limit",
path.display()
));
}
bytes.clear();
bytes.extend_from_slice(text.as_bytes());
}
#[cfg(unix)]
let executable = {
use std::os::unix::fs::PermissionsExt;
@@ -2273,20 +2244,45 @@ impl Repository {
)
};
total += bytes.len();
if total > OUTPUT_LIMIT {
return Err("Working snapshot exceeds the 64 MiB comparison limit".into());
if total > WORKING_SNAPSHOT_LIMIT {
return Err("Working snapshot exceeds the 512 MiB comparison limit".into());
}
let object = if notice.is_none() {
let name = frozen_paths.len().to_string();
fs::write(frozen.0.join(&name), &bytes)
.map_err(|e| format!("{}: {e}", path.display()))?;
writeln!(&mut hash_input, "{name}").unwrap();
frozen_paths.push(path.clone());
Some((store.clone(), String::new()))
} else {
None
};
snapshot.insert(
path,
FileData {
mode,
bytes,
object: None,
bytes: vec![],
object,
projection: None,
notice,
},
);
}
if !frozen_paths.is_empty() {
let mut hash = git_command(&store);
hash.args(["hash-object", "--no-filters", "-w", "--stdin-paths"])
.current_dir(&frozen.0);
let hashes = text(checked(run(hash, hash_input, &self.cancelled)?)?)?;
let mut hashes = hashes.lines();
for path in frozen_paths {
self.check_cancelled()?;
let id = hashes.next().ok_or("Truncated Working snapshot hashes")?;
snapshot.get_mut(&path).unwrap().object = Some((store.clone(), id.into()));
}
if hashes.next().is_some() {
return Err("Unexpected Working snapshot hashes".into());
}
}
Ok(snapshot)
}
fn snapshot_revision(
@@ -2299,13 +2295,16 @@ impl Repository {
}
self.ensure_private()?;
let store = &self.private.as_ref().unwrap().1;
let mut input = b"vamp-working-snapshot-v1\0".to_vec();
let mut input = b"vamp-working-snapshot-v2\0".to_vec();
for (path, file) in snapshot {
self.check_cancelled()?;
for field in [
path.as_os_str().as_encoded_bytes(),
file.mode.as_bytes(),
file.bytes.as_slice(),
file.object
.as_ref()
.map_or("", |(_, id)| id.as_str())
.as_bytes(),
file.notice.as_deref().unwrap_or("").as_bytes(),
] {
input.extend_from_slice(&(field.len() as u64).to_be_bytes());
@@ -2760,15 +2759,6 @@ impl Repository {
) -> Result<Comparison, String> {
let left = &mut prepared.left;
let right = &mut prepared.right;
// Frozen checkout bytes have no object identity. Compare them in a single
// batch; immutable blobs and deferred conflict projections retain IDs.
let frozen = |file: &FileData| {
file.object.is_none() && matches!(file.mode.as_str(), "100644" | "100755" | "120000")
};
if !same && (left.values().any(frozen) || right.values().any(frozen)) {
self.materialize(left)?;
self.materialize(right)?;
}
let right_view = if same { &*left } else { &*right };
let mut files = vec![];
let paths: BTreeSet<_> = left.keys().chain(right_view.keys()).collect();
@@ -2950,6 +2940,19 @@ impl Repository {
notices.push("Binary or non-UTF-8 content; text diff is unavailable".into());
(vec![], vec![], vec![])
} else {
// Bound line-header allocations as well as payload bytes. The real
// generated Honey parser has more than 1.5 million lines.
for bytes in [a, b] {
let count = bytes.iter().filter(|&&byte| byte == b'\n').count()
+ usize::from(!bytes.is_empty() && !bytes.ends_with(b"\n"));
if count > COMPARISON_LINE_LIMIT {
return Err(format!(
"{} exceeds the 4000000-line per-side comparison limit",
path.display()
));
}
self.check_cancelled()?;
}
if (!a.is_empty() && !a.ends_with(b"\n")) || (!b.is_empty() && !b.ends_with(b"\n")) {
notices.push(format!(
"EOF newline: Before {}, After {}",