hl-valdemar/vamp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

fix: support large files in comparisons

Browse Source 
Replace the 16 MiB comparison rejection with independent bounded per-side loading and frozen Working blobs. Bound diff computation, defer large inline highlighting, index source-row navigation, and expose actual load failures. Verified the real 52.3 MB parser through repository/session APIs and terminal rendering with EOF/start navigation.
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-10-17 00:03:43 +02:00
parent adbd2c3f81
commit b9f29d258c
11 changed files with 986 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/repository.rs
+267 -18
View File
@@ -1728,18 +1728,30 @@ fn conflicted_catalog_preserves_symlinks_and_defers_unselected_contents() {
let fixture = Fixture::new();
let root = fixture.dir("lazy-conflict");
init_jj(&root, false);
fs::write(root.join("conflict"), "base\n").unwrap();
jj(
&root,
&[
"config",
"set",
"--repo",
"snapshot.max-new-file-size",
"100MiB",
],
);
let tail_line = "x".repeat(1023);
let tail = format!("{tail_line}\n").repeat(17 * 1024 + 1);
fs::write(root.join("conflict"), format!("base\n{tail}")).unwrap();
fs::write(root.join("unselected"), "old\n").unwrap();
symlink("unselected", root.join("unchanged-link")).unwrap();
jj(&root, &["describe", "-m", "base"]);
let base = jj(&root, &["log", "--no-graph", "-r", "@", "-T", "commit_id"]);
jj(&root, &["new", &base]);
fs::write(root.join("conflict"), "left\n").unwrap();
fs::write(root.join("conflict"), format!("left\n{tail}")).unwrap();
fs::write(root.join("unselected"), vec![b'x'; 17 * 1024 * 1024]).unwrap();
jj(&root, &["describe", "-m", "left"]);
let left = jj(&root, &["log", "--no-graph", "-r", "@", "-T", "commit_id"]);
jj(&root, &["new", &base]);
fs::write(root.join("conflict"), "right\n").unwrap();
fs::write(root.join("conflict"), format!("right\n{tail}")).unwrap();
jj(&root, &["describe", "-m", "right"]);
let right = jj(&root, &["log", "--no-graph", "-r", "@", "-T", "commit_id"]);
jj(&root, &["new", &left, &right, "-m", "conflicted merge"]);
@@ -1764,7 +1776,9 @@ fn conflicted_catalog_preserves_symlinks_and_defers_unselected_contents() {
[Path::new("conflict"), Path::new("unselected")]
);
let conflict = repository.load_file(1, 0).unwrap();
assert_eq!(conflict.before, ["base"]);
assert_eq!(conflict.before[0], "base");
assert_eq!(conflict.before.len(), 17 * 1024 + 2);
assert!(conflict.before[1..].iter().all(|line| line == &tail_line));
let projection = jj(
&root,
&[
@@ -1779,9 +1793,13 @@ fn conflicted_catalog_preserves_symlinks_and_defers_unselected_contents() {
"conflict",
],
);
assert!(projection.len() > 16 * 1024 * 1024);
assert_eq!(conflict.after, projection.lines().collect::<Vec<_>>());
assert!(!conflict.hunks.is_empty());
assert!(repository.load_file(1, 1).is_err());
let large = repository.load_file(1, 1).unwrap();
assert_eq!(large.after.len(), 1);
assert_eq!(large.after[0].len(), 17 * 1024 * 1024);
assert!(large.after[0].bytes().all(|byte| byte == b'x'));
assert_eq!(disk_state(&root), state);
jj(&root, &["new", &merged]);
@@ -1945,6 +1963,220 @@ fn immutable_catalog_skips_unchanged_blobs_and_prepares_only_requested_details()
assert!(repository.load_file(100, index).is_err());
}
#[test]
fn comparison_size_and_line_ceilings_are_lazy_and_explicit() {
let fixture = Fixture::new();
let root = fixture.dir("ceilings");
init_git(&root);
git(&root, &["commit", "--allow-empty", "-m", "before"]);
let before = revision(&git(&root, &["rev-parse", "HEAD"]), RepoKind::Git);
fs::File::create(root.join("too-many-bytes"))
.unwrap()
.set_len(64 * 1024 * 1024 + 1)
.unwrap();
fs::write(root.join("too-many-lines"), vec![b'\n'; 4_000_001]).unwrap();
fs::write(root.join("selected"), "still available\n").unwrap();
git(&root, &["add", "."]);
git(&root, &["commit", "-m", "after"]);
let after = revision(&git(&root, &["rev-parse", "HEAD"]), RepoKind::Git);
let mut repository = open(&root);
let catalog = repository
.compare(1, &before, &after, ComparisonMode::Snapshots, &[])
.unwrap();
assert_eq!(catalog.files.len(), 3);
assert!(catalog.files.iter().all(|file| !file.loaded));
for (index, entry) in catalog.files.iter().enumerate() {
match entry.path.to_str().unwrap() {
"selected" => assert_eq!(
repository.load_file(1, index).unwrap().after,
["still available"]
),
"too-many-bytes" => assert!(
repository
.load_file(1, index)
.unwrap_err()
.contains("64 MiB per-side comparison file limit")
),
"too-many-lines" => assert!(
repository
.load_file(1, index)
.unwrap_err()
.contains("4000000-line per-side comparison limit")
),
path => panic!("Unexpected file {path}"),
}
}
}
#[test]
fn large_git_and_jj_comparisons_load_exact_snapshots_and_frozen_working_overlays() {
let fixture = Fixture::new();
// Both modified sides exceed 52.3 MB; the Working pair exceeds the old
// aggregate 64 MiB budget as well as its old 16 MiB per-file limit.
let body_line = "x".repeat(1023);
let mut contents = format!(
"before\n{}",
format!("{body_line}\n").repeat(52_300_000 / 1024 + 1)
);
let line_count = contents.lines().count();
for kind in [RepoKind::Git, RepoKind::Jujutsu] {
let root = fixture
.dir(if kind == RepoKind::Git { "git" } else { "jj" })
.canonicalize()
.unwrap();
match kind {
RepoKind::Git => init_git(&root),
RepoKind::Jujutsu => {
init_jj(&root, false);
jj(
&root,
&[
"config",
"set",
"--repo",
"snapshot.max-new-file-size",
"100MiB",
],
);
}
}
let set_header = |contents: &mut String, header: &str| {
let end = contents.find('\n').unwrap();
contents.replace_range(..end, header);
};
set_header(&mut contents, "before");
fs::write(root.join("modified"), &contents).unwrap();
set_header(&mut contents, "removed");
fs::write(root.join("removed"), &contents).unwrap();
let before = match kind {
RepoKind::Git => {
git(&root, &["add", "."]);
git(&root, &["commit", "-m", "large before"]);
git(&root, &["rev-parse", "HEAD"])
}
RepoKind::Jujutsu => {
jj(&root, &["describe", "-m", "large before"]);
let id = jj(&root, &["log", "--no-graph", "-r", "@", "-T", "commit_id"]);
jj(&root, &["new", &id]);
id
}
};
set_header(&mut contents, "after");
fs::write(root.join("modified"), &contents).unwrap();
fs::remove_file(root.join("removed")).unwrap();
set_header(&mut contents, "added");
fs::write(root.join("added"), &contents).unwrap();
let after = match kind {
RepoKind::Git => {
git(&root, &["add", "."]);
git(&root, &["commit", "-m", "large after"]);
git(&root, &["rev-parse", "HEAD"])
}
RepoKind::Jujutsu => {
jj(&root, &["describe", "-m", "large after"]);
jj(&root, &["log", "--no-graph", "-r", "@", "-T", "commit_id"])
}
};
let assert_contents = |lines: &[String], header: &str| {
assert_eq!(lines.len(), line_count);
assert_eq!(lines[0], header);
assert!(lines[1..].iter().all(|line| line == &body_line));
};
let cancelled = Arc::new(AtomicBool::new(false));
let mut repository = Repository::open(&root, cancelled.clone()).unwrap();
let catalog = repository
.compare(
1,
&revision(&before, kind),
&revision(&after, kind),
ComparisonMode::Snapshots,
&[],
)
.unwrap();
assert_eq!(catalog.files.len(), 3);
assert!(catalog.files.iter().all(|file| !file.loaded
&& file.before.is_empty()
&& file.after.is_empty()
&& file.hunks.is_empty()));
for (index, entry) in catalog.files.iter().enumerate() {
let file = repository.load_file(1, index).unwrap();
assert!(file.loaded);
assert_eq!(file.hunks.len(), 1);
let hunk = &file.hunks[0];
let expected = match entry.path.to_str().unwrap() {
"added" => {
assert_eq!(file.status, FileStatus::Added);
assert!(file.before.is_empty());
assert_contents(&file.after, "added");
(0, 0, 0, line_count)
}
"removed" => {
assert_eq!(file.status, FileStatus::Removed);
assert_contents(&file.before, "removed");
assert!(file.after.is_empty());
(0, line_count, 0, 0)
}
"modified" => {
assert_eq!(file.status, FileStatus::Modified);
assert_contents(&file.before, "before");
assert_contents(&file.after, "after");
(0, 1, 0, 1)
}
path => panic!("Unexpected file {path}"),
};
assert_eq!(
(
hunk.before_start,
hunk.before_len,
hunk.after_start,
hunk.after_len
),
expected
);
assert!(file.before.is_empty() || file.before_eof);
assert!(file.after.is_empty() || file.after_eof);
}
set_header(&mut contents, "overlay");
let buffers = [(root.join("modified"), contents)];
let working = repository
.compare(
2,
&revision(&after, kind),
&working(kind),
ComparisonMode::Snapshots,
&buffers,
)
.unwrap();
assert_eq!(working.files.len(), 1);
assert!(!working.files[0].loaded);
assert!(working.editable_paths.contains(Path::new("modified")));
fs::write(root.join("modified"), "later disk contents\n").unwrap();
cancelled.store(true, Ordering::Release);
assert!(
repository
.load_file(2, 0)
.unwrap_err()
.contains("cancelled")
);
cancelled.store(false, Ordering::Release);
let file = repository.load_file(2, 0).unwrap();
assert_contents(&file.before, "after");
assert_contents(&file.after, "overlay");
assert_eq!(file.hunks.len(), 1);
let hunk = &file.hunks[0];
assert_eq!(
(
hunk.before_start,
hunk.before_len,
hunk.after_start,
hunk.after_len
),
(0, 1, 0, 1)
);
contents = buffers.into_iter().next().unwrap().1;
}
}
#[test]
fn deferred_file_details_keep_frozen_checkout_and_unsaved_after() {
let fixture = Fixture::new();
@@ -2112,8 +2344,16 @@ fn working_after_refresh_preserves_frozen_working_before_and_sources_on_failure(
assert!(repository.refresh_checkout(10, 11, true, &[]).is_err());
cancelled.store(false, Ordering::Release);
assert_eq!(repository.load_file(10, 0).unwrap().after, ["frozen after"]);
fs::write(root.join("oversized"), vec![b'x'; 17 * 1024 * 1024]).unwrap();
assert!(repository.refresh_checkout(10, 11, true, &[]).is_err());
fs::File::create(root.join("oversized"))
.unwrap()
.set_len(64 * 1024 * 1024 + 1)
.unwrap();
assert!(
repository
.refresh_checkout(10, 11, true, &[])
.unwrap_err()
.contains("64 MiB per-side comparison file limit")
);
let original = repository.load_file(10, 0).unwrap();
assert_eq!(original.before, ["frozen before"]);
assert_eq!(original.after, ["frozen after"]);
@@ -2194,7 +2434,7 @@ fn working_after_refresh_preserves_frozen_working_before_and_sources_on_failure(
}
#[test]
fn refresh_materialization_failure_restores_prepared_file_sources() {
fn refresh_keeps_historical_blob_sources_lazy_even_with_working_files() {
let fixture = Fixture::new();
let root = fixture.dir("failed-materialization");
init_git(&root);
@@ -2206,7 +2446,7 @@ fn refresh_materialization_failure_restores_prepared_file_sources() {
fs::remove_file(root.join("a-file")).unwrap();
fs::remove_file(root.join("z-large")).unwrap();
let mut repository = open(&root);
let original = repository
repository
.compare(
1,
&base,
@@ -2215,20 +2455,29 @@ fn refresh_materialization_failure_restores_prepared_file_sources() {
&[],
)
.unwrap();
let index = original
assert_eq!(
repository.load_file(1, 0).unwrap().before,
["frozen source"]
);
fs::write(root.join("a-file"), "new checkout\n").unwrap();
git(&root, &["add", "."]);
git(&root, &["commit", "-m", "descendant"]);
let refreshed = repository
.refresh_checkout(1, 2, true, &[])
.unwrap()
.map(checkout_comparison)
.unwrap();
assert!(refreshed.files.iter().all(|file| !file.loaded));
let index = refreshed
.files
.iter()
.position(|file| file.path == Path::new("a-file"))
.unwrap();
fs::write(root.join("a-file"), "new checkout\n").unwrap();
git(&root, &["add", "."]);
git(&root, &["commit", "-m", "descendant"]);
assert!(repository.refresh_checkout(1, 2, true, &[]).is_err());
let file = repository.load_file(1, index).unwrap();
let file = repository.load_file(2, index).unwrap();
assert_eq!(file.before, ["frozen source"]);
assert!(file.after.is_empty());
assert_eq!(file.status, FileStatus::Removed);
assert!(repository.load_file(2, index).is_err());
assert_eq!(file.after, ["new checkout"]);
assert_eq!(file.status, FileStatus::Modified);
assert!(repository.load_file(1, index).is_err());
}
#[test]