From 45a32de4c4ac0febc5f6aa289d298b8295ddce47 Mon Sep 17 00:00:00 2001 From: hl-valdemar Date: Mon, 12 Oct 2026 01:04:09 +0200 Subject: [PATCH] perf: batch channel fetching and prefetch earlier --- README.md | 4 +- src/main.rs | 65 ++++++--- src/youtube.rs | 377 ++++++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 407 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 6bb3b06..e07294f 100644 --- a/README.md +++ b/README.md @@ -286,7 +286,9 @@ Using the same binding for overview and description preserves the default cycle. ## Loading and previews -Search loads 20 results per page; Feed and category views request 20 uploads per channel per page. Individual channel views request only that channel's uploads, using the same pagination. Moving near the loaded end in normal mode fetches more asynchronously, preserving selection and existing rows. Insert mode and open popups pause automatic prefetch. Deep channel archives can get slower because yt-dlp offsets traverse earlier pages again. +Search loads 60 results per page; Feed and category views request 60 uploads per channel per page. Feed requests divide channels into disjoint batches across at most four concurrent yt-dlp processes. Individual channel views use one batch. Search uses one extractor to preserve relevance without repeating the same continuation traversal in parallel. + +In normal mode, the next page starts fetching when 20 loaded rows remain, preserving selection and existing rows. Only one page request runs at a time; insert mode and open popups pause automatic prefetch. A feed page is accepted only after every batch succeeds. Switching views cancels all pending batches; returning resumes the same unfinished range. Deep channel archives can get slower because yt-dlp offsets traverse earlier pages again. Malformed individual entries—including invalid video IDs, missing required fields, and wrong field types—are discarded without losing valid videos from the same page. Skipped entries still count toward pagination, so a full page of bad or unavailable entries does not hide later results. Invalid JSON, malformed playlist-level metadata, and extractor failures still fail the page. diff --git a/src/main.rs b/src/main.rs index b68523b..e121203 100644 --- a/src/main.rs +++ b/src/main.rs @@ -79,10 +79,11 @@ //! r reloads config and refreshes the current view. //! Feed includes saved `[channels]` entries, even untagged, plus direct tag entries, without duplicates. //! Untagging is not unsubscribing: remove mappings and tag references from the file to unsubscribe. -//! Search pages contain 20 results; Feed/tag pages request 20 uploads per channel's /videos tab. +//! Search pages contain 60 results; Feed/tag pages request 60 uploads per channel's /videos tab. +//! Search uses one extractor; feeds batch disjoint channels across up to four concurrent extractors. //! Invalid individual video entries are discarded; raw entry counts still determine pagination. //! Invalid JSON, playlist-level metadata, and extractor failures remain page errors. -//! In normal mode, approaching the loaded end fetches the next page without replacing rows/selection. +//! In unobstructed normal mode, 20 rows before the loaded end fetches the next page without replacing rows/selection. //! Loading result/preview panes animate a spinner after their title, including additional pages. //! Info spinners cover image download/decode/resize and description fetching; errors stop them. //! Feed rows use approximate upload dates; older rows wait until newer channel pages catch up. @@ -397,7 +398,7 @@ impl View { || self .selection .selected() - .is_some_and(|index| index.saturating_add(5) >= self.videos.len())) + .is_some_and(|index| index.saturating_add(PAGE_SIZE / 3) >= self.videos.len())) } } @@ -7447,13 +7448,15 @@ tags.cancel = "z" ); assert_eq!(view.videos[1].title, "Second"); assert_eq!(view.selection.selected(), Some(1)); - assert_eq!(view.next_start, 41); + assert_eq!(view.next_start, PAGE_SIZE * 2 + 1); assert!(view.has_more); } #[test] fn feed_pages_hold_short_channel_tails_until_newer_long_histories_finish() { let mut view = View::new("Feed".into(), ViewSource::Feed); + let second_base = 101 + PAGE_SIZE as i64; + let first_base = second_base + PAGE_SIZE as i64; let long_page = |base: i64| { serde_json::json!({ "entries": (0..PAGE_SIZE).map(|index| serde_json::json!({ @@ -7465,22 +7468,25 @@ tags.cancel = "z" .to_string() }; let short = r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Short old tail","timestamp":100},{"id":"rQ_J9WH6CGk","title":"Unknown date"}]}"#; - let first = youtube::feed_results(&format!("{}\n{short}", long_page(200))).unwrap(); + let first = youtube::feed_results(&format!("{}\n{short}", long_page(first_base))).unwrap(); view.accept_page(first, true).unwrap(); assert_eq!(view.videos.len(), PAGE_SIZE); - assert_eq!(view.videos.last().unwrap().timestamp, Some(200)); + assert_eq!(view.videos.last().unwrap().timestamp, Some(first_base)); assert_eq!(view.pending.len(), 2); view.selection.select(Some(PAGE_SIZE - 1)); - let second = youtube::feed_results(&format!("{}\n{short}", long_page(180))).unwrap(); + let second = + youtube::feed_results(&format!("{}\n{short}", long_page(second_base))).unwrap(); view.accept_page(second, true).unwrap(); assert_eq!(view.videos.len(), PAGE_SIZE * 2); - assert_eq!(view.videos.last().unwrap().timestamp, Some(180)); + assert_eq!(view.videos.last().unwrap().timestamp, Some(second_base)); assert_eq!(view.pending.len(), 2); assert_eq!(view.selection.selected(), Some(PAGE_SIZE - 1)); view.accept_page( - youtube::feed_results( - r#"{"entries":[{"id":"l-GaaivVJCE","title":"Long last row","timestamp":179}]} {"entries":[]}"#, - ).unwrap(), + youtube::feed_results(&format!( + r#"{{"entries":[{{"id":"l-GaaivVJCE","title":"Long last row","timestamp":{}}}]}} {{"entries":[]}}"#, + second_base - 1, + )) + .unwrap(), true, ).unwrap(); assert_eq!( @@ -7517,11 +7523,17 @@ tags.cancel = "z" assert!(view.needs_page()); let page = youtube::feed_results(&format!("{{\"entries\":[]}}\n{}", dated(200))).unwrap(); view.accept_page(page, true).unwrap(); - assert_eq!(view.videos.first().unwrap().timestamp, Some(219)); + assert_eq!( + view.videos.first().unwrap().timestamp, + Some(200 + PAGE_SIZE as i64 - 1) + ); assert_eq!(view.videos.last().unwrap().timestamp, Some(200)); view.accept_page(youtube::feed_results(r#"{"entries":[]}"#).unwrap(), true) .unwrap(); - assert_eq!(view.videos[PAGE_SIZE].timestamp, Some(119)); + assert_eq!( + view.videos[PAGE_SIZE].timestamp, + Some(100 + PAGE_SIZE as i64 - 1) + ); assert_eq!(view.videos.last().unwrap().timestamp, Some(100)); assert!(!view.has_more); } @@ -7546,7 +7558,7 @@ tags.cancel = "z" app.finish_page(Ok(serde_json::json!({"entries": entries}).to_string())); assert!(app.views[1].videos.is_empty()); assert_eq!(app.views[1].pending.len(), PAGE_SIZE); - assert_eq!(app.views[1].next_start, 21); + assert_eq!(app.views[1].next_start, PAGE_SIZE + 1); assert!(app.views[1].loaded); assert!(app.can_prefetch()); app.key(KeyEvent::new(KeyCode::Char('j'), KeyModifiers::NONE)); @@ -7554,7 +7566,7 @@ tags.cancel = "z" assert!(app.job.is_none()); app.finish_page(Err("Continuation failed".into())); assert_eq!(app.views[1].pending.len(), PAGE_SIZE); - assert_eq!(app.views[1].next_start, 21); + assert_eq!(app.views[1].next_start, PAGE_SIZE + 1); assert!(!app.can_prefetch()); app.finish_page(Ok(r#"{"entries":[]}"#.into())); assert_eq!(app.views[1].videos.len(), PAGE_SIZE); @@ -7594,12 +7606,12 @@ tags.cancel = "z" }) .collect(); app.finish_page(Ok(serde_json::json!({"entries": entries}).to_string())); - assert_eq!(app.views[0].next_start, 21); + assert_eq!(app.views[0].next_start, PAGE_SIZE + 1); app.views[0].selection.select(Some(17)); app.remember_description(app.views[0].videos[17].id.clone(), "Cached.".into()); app.switch_view(1); app.switch_view(0); - assert_eq!(app.views[0].next_start, 21); + assert_eq!(app.views[0].next_start, PAGE_SIZE + 1); assert_eq!(app.views[0].selection.selected(), Some(17)); app.finish_page(Ok("not JSON".into())); app.update(); @@ -7609,7 +7621,7 @@ tags.cancel = "z" cached_description(&app.descriptions, &app.views[0].videos[17].id), Some("Cached.") ); - assert_eq!(app.views[0].next_start, 21); + assert_eq!(app.views[0].next_start, PAGE_SIZE + 1); assert!(!app.views[0].needs_page()); let view = &mut app.views[0]; view.pending = @@ -7664,6 +7676,7 @@ tags.cancel = "z" }) }) .collect(); + app.views[0].reset(); app.views[0] .accept_page( youtube::search_results(&serde_json::json!({"entries": entries}).to_string()) @@ -7673,11 +7686,25 @@ tags.cancel = "z" .unwrap(); assert_eq!(app.views[0].selection.selected(), Some(0)); assert!(!app.can_prefetch()); - app.views[0].selection.select(Some(15)); + let threshold = app.views[0].videos.len() - PAGE_SIZE / 3; + app.views[0].selection.select(Some(threshold - 1)); assert!(!app.can_prefetch()); app.key(KeyEvent::new(KeyCode::Char('j'), KeyModifiers::NONE)); assert!(app.can_prefetch()); assert!(app.job.is_none()); + assert_eq!(app.views[0].selection.selected(), Some(threshold)); + app.views[0].loaded = false; + assert!(!app.can_prefetch()); + app.views[0].loaded = true; + app.views[0].has_more = false; + assert!(!app.can_prefetch()); + app.views[0].has_more = true; + app.help_popup = Some(HelpPopup::default()); + assert!(!app.can_prefetch()); + app.help_popup = None; + app.views[0].selection.select(None); + assert!(!app.can_prefetch()); + app.views[0].selection.select(Some(threshold)); app.views[0].page_error = Some("Stop retrying".into()); assert!(!app.can_prefetch()); app.views[0].page_error = None; diff --git a/src/youtube.rs b/src/youtube.rs index de71604..4b9208f 100644 --- a/src/youtube.rs +++ b/src/youtube.rs @@ -9,7 +9,7 @@ use std::{ use serde::{Deserialize, Serialize}; -pub const PAGE_SIZE: usize = 20; +pub const PAGE_SIZE: usize = 60; pub struct Page { pub videos: Vec