hl-valdemar/tubular
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

perf: batch channel fetching and prefetch earlier

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-10-12 01:04:09 +02:00
parent 0d69d050ee
commit 45a32de4c4
3 changed files with 407 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main.rs
+46 -19
View File
@@ -79,10 +79,11 @@
//! r reloads config and refreshes the current view.
//! Feed includes saved `[channels]` entries, even untagged, plus direct tag entries, without duplicates.
//! Untagging is not unsubscribing: remove mappings and tag references from the file to unsubscribe.
//! Search pages contain 20 results; Feed/tag pages request 20 uploads per channel's /videos tab.
//! Search pages contain 60 results; Feed/tag pages request 60 uploads per channel's /videos tab.
//! Search uses one extractor; feeds batch disjoint channels across up to four concurrent extractors.
//! Invalid individual video entries are discarded; raw entry counts still determine pagination.
//! Invalid JSON, playlist-level metadata, and extractor failures remain page errors.
//! In normal mode, approaching the loaded end fetches the next page without replacing rows/selection.
//! In unobstructed normal mode, 20 rows before the loaded end fetches the next page without replacing rows/selection.
//! Loading result/preview panes animate a spinner after their title, including additional pages.
//! Info spinners cover image download/decode/resize and description fetching; errors stop them.
//! Feed rows use approximate upload dates; older rows wait until newer channel pages catch up.
@@ -397,7 +398,7 @@ impl View {
|| self
.selection
.selected()
.is_some_and(|index| index.saturating_add(5) >= self.videos.len()))
.is_some_and(|index| index.saturating_add(PAGE_SIZE / 3) >= self.videos.len()))
}
}
@@ -7447,13 +7448,15 @@ tags.cancel = "z"
);
assert_eq!(view.videos[1].title, "Second");
assert_eq!(view.selection.selected(), Some(1));
assert_eq!(view.next_start, 41);
assert_eq!(view.next_start, PAGE_SIZE * 2 + 1);
assert!(view.has_more);
}
#[test]
fn feed_pages_hold_short_channel_tails_until_newer_long_histories_finish() {
let mut view = View::new("Feed".into(), ViewSource::Feed);
let second_base = 101 + PAGE_SIZE as i64;
let first_base = second_base + PAGE_SIZE as i64;
let long_page = |base: i64| {
serde_json::json!({
"entries": (0..PAGE_SIZE).map(|index| serde_json::json!({
@@ -7465,22 +7468,25 @@ tags.cancel = "z"
.to_string()
};
let short = r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Short old tail","timestamp":100},{"id":"rQ_J9WH6CGk","title":"Unknown date"}]}"#;
let first = youtube::feed_results(&format!("{}\n{short}", long_page(200))).unwrap();
let first = youtube::feed_results(&format!("{}\n{short}", long_page(first_base))).unwrap();
view.accept_page(first, true).unwrap();
assert_eq!(view.videos.len(), PAGE_SIZE);
assert_eq!(view.videos.last().unwrap().timestamp, Some(200));
assert_eq!(view.videos.last().unwrap().timestamp, Some(first_base));
assert_eq!(view.pending.len(), 2);
view.selection.select(Some(PAGE_SIZE - 1));
let second = youtube::feed_results(&format!("{}\n{short}", long_page(180))).unwrap();
let second =
youtube::feed_results(&format!("{}\n{short}", long_page(second_base))).unwrap();
view.accept_page(second, true).unwrap();
assert_eq!(view.videos.len(), PAGE_SIZE * 2);
assert_eq!(view.videos.last().unwrap().timestamp, Some(180));
assert_eq!(view.videos.last().unwrap().timestamp, Some(second_base));
assert_eq!(view.pending.len(), 2);
assert_eq!(view.selection.selected(), Some(PAGE_SIZE - 1));
view.accept_page(
youtube::feed_results(
r#"{"entries":[{"id":"l-GaaivVJCE","title":"Long last row","timestamp":179}]} {"entries":[]}"#,
).unwrap(),
youtube::feed_results(&format!(
r#"{{"entries":[{{"id":"l-GaaivVJCE","title":"Long last row","timestamp":{}}}]}} {{"entries":[]}}"#,
second_base - 1,
))
.unwrap(),
true,
).unwrap();
assert_eq!(
@@ -7517,11 +7523,17 @@ tags.cancel = "z"
assert!(view.needs_page());
let page = youtube::feed_results(&format!("{{\"entries\":[]}}\n{}", dated(200))).unwrap();
view.accept_page(page, true).unwrap();
assert_eq!(view.videos.first().unwrap().timestamp, Some(219));
assert_eq!(
view.videos.first().unwrap().timestamp,
Some(200 + PAGE_SIZE as i64 - 1)
);
assert_eq!(view.videos.last().unwrap().timestamp, Some(200));
view.accept_page(youtube::feed_results(r#"{"entries":[]}"#).unwrap(), true)
.unwrap();
assert_eq!(view.videos[PAGE_SIZE].timestamp, Some(119));
assert_eq!(
view.videos[PAGE_SIZE].timestamp,
Some(100 + PAGE_SIZE as i64 - 1)
);
assert_eq!(view.videos.last().unwrap().timestamp, Some(100));
assert!(!view.has_more);
}
@@ -7546,7 +7558,7 @@ tags.cancel = "z"
app.finish_page(Ok(serde_json::json!({"entries": entries}).to_string()));
assert!(app.views[1].videos.is_empty());
assert_eq!(app.views[1].pending.len(), PAGE_SIZE);
assert_eq!(app.views[1].next_start, 21);
assert_eq!(app.views[1].next_start, PAGE_SIZE + 1);
assert!(app.views[1].loaded);
assert!(app.can_prefetch());
app.key(KeyEvent::new(KeyCode::Char('j'), KeyModifiers::NONE));
@@ -7554,7 +7566,7 @@ tags.cancel = "z"
assert!(app.job.is_none());
app.finish_page(Err("Continuation failed".into()));
assert_eq!(app.views[1].pending.len(), PAGE_SIZE);
assert_eq!(app.views[1].next_start, 21);
assert_eq!(app.views[1].next_start, PAGE_SIZE + 1);
assert!(!app.can_prefetch());
app.finish_page(Ok(r#"{"entries":[]}"#.into()));
assert_eq!(app.views[1].videos.len(), PAGE_SIZE);
@@ -7594,12 +7606,12 @@ tags.cancel = "z"
})
.collect();
app.finish_page(Ok(serde_json::json!({"entries": entries}).to_string()));
assert_eq!(app.views[0].next_start, 21);
assert_eq!(app.views[0].next_start, PAGE_SIZE + 1);
app.views[0].selection.select(Some(17));
app.remember_description(app.views[0].videos[17].id.clone(), "Cached.".into());
app.switch_view(1);
app.switch_view(0);
assert_eq!(app.views[0].next_start, 21);
assert_eq!(app.views[0].next_start, PAGE_SIZE + 1);
assert_eq!(app.views[0].selection.selected(), Some(17));
app.finish_page(Ok("not JSON".into()));
app.update();
@@ -7609,7 +7621,7 @@ tags.cancel = "z"
cached_description(&app.descriptions, &app.views[0].videos[17].id),
Some("Cached.")
);
assert_eq!(app.views[0].next_start, 21);
assert_eq!(app.views[0].next_start, PAGE_SIZE + 1);
assert!(!app.views[0].needs_page());
let view = &mut app.views[0];
view.pending =
@@ -7664,6 +7676,7 @@ tags.cancel = "z"
})
})
.collect();
app.views[0].reset();
app.views[0]
.accept_page(
youtube::search_results(&serde_json::json!({"entries": entries}).to_string())
@@ -7673,11 +7686,25 @@ tags.cancel = "z"
.unwrap();
assert_eq!(app.views[0].selection.selected(), Some(0));
assert!(!app.can_prefetch());
app.views[0].selection.select(Some(15));
let threshold = app.views[0].videos.len() - PAGE_SIZE / 3;
app.views[0].selection.select(Some(threshold - 1));
assert!(!app.can_prefetch());
app.key(KeyEvent::new(KeyCode::Char('j'), KeyModifiers::NONE));
assert!(app.can_prefetch());
assert!(app.job.is_none());
assert_eq!(app.views[0].selection.selected(), Some(threshold));
app.views[0].loaded = false;
assert!(!app.can_prefetch());
app.views[0].loaded = true;
app.views[0].has_more = false;
assert!(!app.can_prefetch());
app.views[0].has_more = true;
app.help_popup = Some(HelpPopup::default());
assert!(!app.can_prefetch());
app.help_popup = None;
app.views[0].selection.select(None);
assert!(!app.can_prefetch());
app.views[0].selection.select(Some(threshold));
app.views[0].page_error = Some("Stop retrying".into());
assert!(!app.can_prefetch());
app.views[0].page_error = None;