hl-valdemar/tubular
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

fix: cap feed requests at 60 videos total

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-10-12 01:22:42 +02:00
parent 45a32de4c4
commit b15fa23b8a
3 changed files with 1255 additions and 386 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main.rs
+470 -232
View File
@@ -79,16 +79,16 @@
//! r reloads config and refreshes the current view.
//! Feed includes saved `[channels]` entries, even untagged, plus direct tag entries, without duplicates.
//! Untagging is not unsubscribing: remove mappings and tag references from the file to unsubscribe.
//! Search pages contain 60 results; Feed/tag pages request 60 uploads per channel's /videos tab.
//! Search uses one extractor; feeds batch disjoint channels across up to four concurrent extractors.
//! Search pages contain 60 results; Feed/tag pages share a total request budget of 60 uploads.
//! Search uses one extractor; feeds batch matching channel ranges across up to four concurrent extractors.
//! Invalid individual video entries are discarded; raw entry counts still determine pagination.
//! Invalid JSON, playlist-level metadata, and extractor failures remain page errors.
//! In unobstructed normal mode, 20 rows before the loaded end fetches the next page without replacing rows/selection.
//! Forward video navigation authorizes prefetch within 20 rows of the loaded end in unobstructed normal mode.
//! Each accepted page clears that intent, so short or invalid batches never auto-chain on startup.
//! Loading result/preview panes animate a spinner after their title, including additional pages.
//! Info spinners cover image download/decode/resize and description fetching; errors stop them.
//! Feed rows use approximate upload dates; older rows wait until newer channel pages catch up.
//! Undated feed rows appear at exhaustion. Cached views retain their page position across switches.
//! Switching views cancels in-flight extraction; returning resumes the same pending range.
//! Feed rows, including undated rows, are globally newest-first among loaded uploads; selection follows video ID.
//! Switching views cancels in-flight extraction; returning retains the unaccepted request and per-channel offsets.
//! Errors keep loaded rows and stop automatic retries; r reloads config and restarts the current view.
//! Deep pages may slow down because yt-dlp offsets re-traverse earlier continuation pages.
//! Video tables show Published as YYYY-MM-DD; ~ marks estimated UTC dates, Unknown missing metadata.
@@ -309,7 +309,9 @@ struct View {
loaded: bool,
next_start: usize,
has_more: bool,
pending: Vec<Video>,
feed_cursor: youtube::FeedCursor,
feed_requests: Vec<youtube::FeedRequest>,
prefetch_requested: bool,
page_error: Option<String>,
watch_filter: WatchFilter,
}
@@ -326,7 +328,9 @@ impl View {
loaded: false,
next_start: 1,
has_more: false,
pending: Vec::new(),
feed_cursor: youtube::FeedCursor::default(),
feed_requests: Vec::new(),
prefetch_requested: false,
page_error: None,
watch_filter: WatchFilter::All,
}
@@ -338,7 +342,9 @@ impl View {
self.loaded = false;
self.next_start = 1;
self.has_more = false;
self.pending.clear();
self.feed_cursor = youtube::FeedCursor::default();
self.feed_requests.clear();
self.prefetch_requested = false;
self.page_error = None;
}
@@ -347,45 +353,51 @@ impl View {
.next_start
.checked_add(PAGE_SIZE)
.ok_or("Page offset exceeds the supported range")?;
let mut videos = if feed {
self.pending.append(&mut page.videos);
std::mem::take(&mut self.pending)
} else {
page.videos
};
if feed {
videos.sort_by(|a, b| b.timestamp.cmp(&a.timestamp));
self.feed_cursor.accept(&page.feed_progress)?;
}
if !videos.is_empty() {
// Borrow IDs before moving rows, retaining the existing row data.
let keep: Vec<_> = {
let mut ids = HashSet::with_capacity(self.videos.len() + videos.len());
ids.extend(self.videos.iter().map(|video| video.id.as_str()));
videos
.iter()
.map(|video| ids.insert(video.id.as_str()))
.collect()
if !page.videos.is_empty() {
let selected = if feed {
self.selection.selected().and_then(|index| {
self.videos.get(index).map(|video| {
<[u8; 11]>::try_from(video.id.as_bytes()).expect("validated video ID")
})
})
} else {
None
};
let mut keep = keep.into_iter();
videos.retain(|_| keep.next().expect("one deduplication flag per video"));
}
let ready = if feed && page.has_more {
page.cutoff.map_or(0, |cutoff| {
videos.partition_point(|video| video.timestamp.is_some_and(|date| date >= cutoff))
})
} else {
videos.len()
};
self.videos.extend(videos.drain(..ready));
if feed {
self.pending = videos;
let mut ids = HashSet::with_capacity(self.videos.len() + page.videos.len());
ids.extend(self.videos.iter().map(|video| {
<[u8; 11]>::try_from(video.id.as_bytes()).expect("validated video ID")
}));
page.videos.retain(|video| {
ids.insert(<[u8; 11]>::try_from(video.id.as_bytes()).expect("validated video ID"))
});
self.videos.append(&mut page.videos);
if feed {
// ponytail: sort loaded metadata only; a globally newest archive merge needs
// per-channel frontier lookahead rather than fair quotas.
self.videos.sort_by(|a, b| b.timestamp.cmp(&a.timestamp));
if let Some(id) = selected {
self.selection.select(
self.videos
.iter()
.position(|video| video.id.as_bytes() == id.as_slice()),
);
}
}
}
if self.selection.selected().is_none() && !self.videos.is_empty() {
self.selection.select(Some(0));
}
self.loaded = true;
self.next_start = next_start;
self.has_more = page.has_more;
self.has_more = if feed {
self.feed_cursor.has_more()
} else {
page.has_more
};
self.prefetch_requested = false;
self.page_error = None;
Ok(())
}
@@ -393,6 +405,7 @@ impl View {
fn needs_page(&self) -> bool {
self.loaded
&& self.has_more
&& self.prefetch_requested
&& self.page_error.is_none()
&& (self.videos.is_empty()
|| self
@@ -890,7 +903,7 @@ struct App {
preview: Option<PreviewMode>,
help_popup: Option<HelpPopup>,
thumbnail: thumbnail::Thumbnail,
description_job: Option<(usize, Job)>,
description_job: Option<(String, Job)>,
description_error: Option<String>,
// ponytail: 64-entry linear lookup; use a keyed LRU if the buffer grows.
descriptions: std::collections::VecDeque<(String, String)>,
@@ -924,9 +937,13 @@ impl App {
fn with_state(
options: Options,
config_path: PathBuf,
config: config::Config,
mut config: config::Config,
video_state: video_state::State,
) -> Self {
// Cursor channel indices must remain stable when a config only reorders memberships.
for channels in config.categories.values_mut() {
channels.sort_unstable();
}
let query = options.query.clone().unwrap_or_default();
let feed_channels = config.subscriptions();
let key_hints = KeyHints::new(&config.keys);
@@ -1419,7 +1436,10 @@ impl App {
view.name,
view.videos.len()
);
Job::feed(channels, view.next_start)
view.feed_cursor.plan(channels.len()).and_then(|requests| {
view.feed_requests = requests;
Job::feed(channels, &view.feed_requests)
})
};
match result {
Ok(job) => self.job = Some(job),
@@ -1467,11 +1487,7 @@ impl App {
);
}
let continuation = if view.has_more {
if view.videos.is_empty() && !view.pending.is_empty() {
"newer pages pending"
} else {
"more available"
}
"more available"
} else {
"end"
};
@@ -1489,7 +1505,7 @@ impl App {
if self.views[self.view_index].source == ViewSource::Search {
youtube::search_results(&output)
} else {
youtube::feed_results(&output)
youtube::feed_results(&output, &self.views[self.view_index].feed_requests)
}
});
let feed = self.views[self.view_index].source != ViewSource::Search;
@@ -1716,7 +1732,10 @@ impl App {
self.close_preview();
}
fn apply_config(&mut self, config: config::Config) {
fn apply_config(&mut self, mut config: config::Config) {
for channels in config.categories.values_mut() {
channels.sort_unstable();
}
if config.keys != self.config.keys {
self.key_hints = KeyHints::new(&config.keys);
if let Some(help) = &mut self.help_popup {
@@ -1987,7 +2006,9 @@ impl App {
view.loaded = source.loaded;
view.next_start = source.next_start;
view.has_more = source.has_more;
view.pending = source.pending.clone();
view.feed_cursor = source.feed_cursor.clone();
view.feed_requests = source.feed_requests.clone();
view.prefetch_requested = source.prefetch_requested;
view.page_error = source.page_error.clone();
Some(view)
} else {
@@ -2385,7 +2406,7 @@ impl App {
return;
}
match Job::description(&video.id) {
Ok(job) => self.description_job = Some((index, job)),
Ok(job) => self.description_job = Some((video.id.clone(), job)),
Err(error) => self.description_error = Some(error.to_string()),
}
}
@@ -2434,6 +2455,14 @@ impl App {
}
}
fn finish_description(&mut self, id: String, result: Result<String, String>) {
match result.and_then(|output| youtube::full_description(&output, &id)) {
Ok(description) => self.remember_description(id, description),
Err(error) => self.description_error = Some(error),
}
self.description_viewport = None;
}
fn update(&mut self) {
if let Some(result) = self.job.as_mut().and_then(Job::poll) {
self.finish_page(result);
@@ -2461,17 +2490,11 @@ impl App {
.as_mut()
.and_then(|(_, job)| job.poll())
{
let (index, _) = self
let (id, _) = self
.description_job
.take()
.expect("completed description job exists");
if let Some(video) = self.views[self.view_index].videos.get(index) {
match result.and_then(|output| youtube::full_description(&output, &video.id)) {
Ok(description) => self.remember_description(video.id.clone(), description),
Err(error) => self.description_error = Some(error),
}
}
self.description_viewport = None;
self.finish_description(id, result);
}
if let Some(result) = self.player_launch.as_mut().and_then(PlayerLaunch::poll) {
self.player_launch = None;
@@ -2736,12 +2759,18 @@ impl App {
self.browse_category_collection(CollectionKind::Saved)
}
Some(Action::VideoPlay) => self.play(),
Some(Action::VideoNext) => self.views[self.view_index].selection.select_next(),
Some(Action::VideoNext) => {
self.views[self.view_index].prefetch_requested = true;
self.views[self.view_index].selection.select_next();
}
Some(Action::VideoPrevious) => {
self.views[self.view_index].selection.select_previous()
}
Some(Action::VideoFirst) => self.views[self.view_index].selection.select_first(),
Some(Action::VideoLast) => self.views[self.view_index].selection.select_last(),
Some(Action::VideoLast) => {
self.views[self.view_index].prefetch_requested = true;
self.views[self.view_index].selection.select_last();
}
_ => {}
}
}
@@ -3570,6 +3599,46 @@ mod tests {
)
}
// Unrelated UI fixtures still parse real feed envelopes with a shared-budget plan.
fn feed_fixture(output: &str) -> Result<Page, String> {
let channels = serde_json::Deserializer::from_str(output)
.into_iter::<serde_json::Value>()
.count();
let requests = youtube::FeedCursor::default().plan(channels).unwrap();
youtube::feed_results(output, &requests)
}
fn finish_feed_fixture(app: &mut App, output: &str, full: bool) {
let view = &mut app.views[app.view_index];
let channels = view
.source
.channels(&app.config, &app.feed_channels, &view.group_channels)
.len();
view.feed_requests = view.feed_cursor.plan(channels).unwrap();
let mut envelopes: Vec<_> = serde_json::Deserializer::from_str(output)
.into_iter::<serde_json::Value>()
.map(Result::unwrap)
.collect();
assert!(envelopes.len() <= view.feed_requests.len());
envelopes.resize_with(
view.feed_requests.len(),
|| serde_json::json!({"entries":[]}),
);
if full {
for (envelope, request) in envelopes.iter_mut().zip(&view.feed_requests) {
let entries = envelope["entries"].as_array_mut().unwrap();
assert!(entries.len() <= request.count);
entries.resize(request.count, serde_json::Value::Null);
}
}
let output = envelopes
.iter()
.map(ToString::to_string)
.collect::<Vec<_>>()
.join("\n");
app.finish_page(Ok(output));
}
fn choose_combined(app: &mut App, key: KeyEvent) {
app.key(key);
while app
@@ -3723,10 +3792,12 @@ mod tests {
"https://www.youtube.com/channel/UCaYhcUwRBNscFNUKTjgPFiA",
]
);
app.finish_page(Ok(
finish_feed_fixture(
&mut app,
r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Shared","timestamp":300},{"id":"l-GaaivVJCE","title":"Old","timestamp":100}]}
{"entries":[{"id":"jNQXAC9IVRw","title":"Shared again","timestamp":300},{"id":"rQ_J9WH6CGk","title":"Middle","timestamp":200}]}"#.into()
));
{"entries":[{"id":"jNQXAC9IVRw","title":"Shared again","timestamp":300},{"id":"rQ_J9WH6CGk","title":"Middle","timestamp":200}]}"#,
true,
);
assert_eq!(
app.views[app.view_index]
.videos
@@ -3736,7 +3807,11 @@ mod tests {
["jNQXAC9IVRw", "rQ_J9WH6CGk", "l-GaaivVJCE"]
);
rename_press(&mut app, KeyCode::Down);
app.finish_page(Ok(r#"{"entries":[{"id":"l-GaaivVJCE","title":"Duplicate","timestamp":100},{"id":"SLV9AJxnb1U","title":"Last","timestamp":50}]}"#.into()));
finish_feed_fixture(
&mut app,
r#"{"entries":[{"id":"l-GaaivVJCE","title":"Duplicate","timestamp":100},{"id":"SLV9AJxnb1U","title":"Last","timestamp":50}]}"#,
false,
);
let next_start = app.views[app.view_index].next_start;
let names = vec!["Alpha".into(), "Gamma".into()];
let mut replacement = View::new(group_name(&names), ViewSource::CategoryGroup(names));
@@ -3777,9 +3852,11 @@ mod tests {
#[test]
fn combined_cache_resets_only_for_changed_resolved_union_and_drops_deleted_members() {
let mut app = group_app("config", "");
app.finish_page(Ok(
r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Cached","timestamp":100}]}"#.into(),
));
finish_feed_fixture(
&mut app,
r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Cached","timestamp":100}]}"#,
false,
);
let group = app.views[app.view_index].source.clone();
let original = std::fs::read_to_string(&app.config_path).unwrap();
std::fs::write(
@@ -3825,9 +3902,11 @@ mod tests {
#[test]
fn checked_category_rename_preserves_canonical_group_pin_selection_and_history() {
let mut app = group_app("rename", "");
app.finish_page(Ok(
r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Cached","timestamp":100}]}"#.into(),
));
finish_feed_fixture(
&mut app,
r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Cached","timestamp":100}]}"#,
false,
);
let old = app.views[app.view_index].source.clone();
app.view_history.push(old.clone());
app.forward_history.push(old);
@@ -3869,16 +3948,20 @@ mod tests {
#[test]
fn saving_combined_view_copies_snapshot_and_preserves_cached_state_without_pinning() {
let mut app = group_app("save", "");
app.finish_page(Ok(r#"{"entries":[{"id":"jNQXAC9IVRw","title":"First","timestamp":200},{"id":"l-GaaivVJCE","title":"Second","timestamp":100}]}"#.into()));
finish_feed_fixture(
&mut app,
r#"{"entries":[{"id":"jNQXAC9IVRw","title":"First","timestamp":200},{"id":"l-GaaivVJCE","title":"Second","timestamp":100}]}"#,
true,
);
rename_press(&mut app, KeyCode::Down);
let source = app.views[app.view_index].source.clone();
let channels = app.views[app.view_index].group_channels.clone();
let pending = youtube::search_results(
r#"{"entries":[{"id":"rQ_J9WH6CGk","title":"Pending","timestamp":50}]}"#,
)
.unwrap()
.videos;
app.views[app.view_index].pending = pending;
let next_requests = app.views[app.view_index]
.feed_cursor
.plan(channels.len())
.unwrap();
app.views[app.view_index].feed_requests = next_requests.clone();
app.views[app.view_index].prefetch_requested = true;
app.views[app.view_index].has_more = true;
app.views[app.view_index].page_error = Some("keep failure".into());
let original = std::fs::read_to_string(&app.config_path).unwrap();
@@ -3921,7 +4004,12 @@ mod tests {
assert_eq!(view.selection.selected(), Some(1));
assert_eq!(view.videos.len(), 2);
assert_eq!(view.next_start, PAGE_SIZE + 1);
assert_eq!(view.pending[0].id, "rQ_J9WH6CGk");
assert_eq!(view.feed_requests, next_requests);
assert_eq!(
view.feed_cursor.clone().plan(channels.len()).unwrap(),
next_requests
);
assert!(view.prefetch_requested);
assert!(view.has_more);
assert_eq!(view.page_error.as_deref(), Some("keep failure"));
assert!(app.job.is_none());
@@ -4147,7 +4235,8 @@ mod tests {
app.view_index = 2;
let view = &mut app.views[2];
view.videos = videos.clone();
view.pending = vec![videos[0].clone()];
view.feed_requests = view.feed_cursor.plan(1).unwrap();
view.prefetch_requested = true;
view.selection.select(Some(1));
view.loaded = true;
view.next_start = 41;
@@ -4162,7 +4251,8 @@ mod tests {
assert_eq!(view.name, "Zeta");
assert_eq!(view.selection.selected(), Some(1));
assert_eq!(view.videos.len(), 2);
assert_eq!(view.pending.len(), 1);
assert_eq!(view.feed_requests.len(), 1);
assert!(view.prefetch_requested);
assert_eq!(view.next_start, 41);
assert!(view.loaded && view.has_more);
assert_eq!(view.page_error.as_deref(), Some("keep page failure"));
@@ -5851,7 +5941,7 @@ mod tests {
let videos = r#"{"entries":[{"id":"jNQXAC9IVRw","title":"First"},{"id":"rQ_J9WH6CGk","title":"Second"}]}"#;
for (label, identity) in app.config.saved_channels() {
let mut view = View::new(label, ViewSource::Channel(identity));
view.videos = youtube::feed_results(videos).unwrap().videos;
view.videos = feed_fixture(videos).unwrap().videos;
view.loaded = true;
view.next_start = 41;
view.selection.select(Some(1));
@@ -6131,6 +6221,7 @@ tags.cancel = "z"
let selected = app.views[app.view_index].selection.selected();
let press = |app: &mut App, code| app.key(KeyEvent::new(code, KeyModifiers::NONE));
app.views[app.view_index].has_more = true;
app.views[app.view_index].prefetch_requested = true;
assert!(app.can_prefetch());
press(&mut app, KeyCode::Char('?'));
assert!(!app.can_prefetch());
@@ -6682,10 +6773,9 @@ tags.cancel = "z"
loaded_channel_views(&mut app);
browse_channel(&mut app, "Alpha");
app.views[app.view_index].has_more = true;
app.views[app.view_index].pending =
youtube::feed_results(r#"{"entries":[{"id":"l-GaaivVJCE","title":"Pending"}]}"#)
.unwrap()
.videos;
app.views[app.view_index].feed_requests =
app.views[app.view_index].feed_cursor.plan(1).unwrap();
app.views[app.view_index].prefetch_requested = true;
app.views[app.view_index].page_error = Some("kept".into());
app.apply_config(config::Config {
keys: Default::default(),
@@ -6708,7 +6798,8 @@ tags.cancel = "z"
assert!(view.loaded && view.has_more);
assert_eq!(view.next_start, 41);
assert_eq!(view.selection.selected(), Some(1));
assert_eq!(view.pending.len(), 1);
assert_eq!(view.feed_requests.len(), 1);
assert!(view.prefetch_requested);
assert_eq!(view.page_error.as_deref(), Some("kept"));
let beta = app
.views
@@ -7453,136 +7544,283 @@ tags.cancel = "z"
}
#[test]
fn feed_pages_hold_short_channel_tails_until_newer_long_histories_finish() {
let mut view = View::new("Feed".into(), ViewSource::Feed);
let second_base = 101 + PAGE_SIZE as i64;
let first_base = second_base + PAGE_SIZE as i64;
let long_page = |base: i64| {
serde_json::json!({
"entries": (0..PAGE_SIZE).map(|index| serde_json::json!({
"id": format!("long{:07}", base + index as i64),
"title": "Long channel",
"timestamp": base + index as i64,
})).collect::<Vec<_>>()
})
.to_string()
};
let short = r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Short old tail","timestamp":100},{"id":"rQ_J9WH6CGk","title":"Unknown date"}]}"#;
let first = youtube::feed_results(&format!("{}\n{short}", long_page(first_base))).unwrap();
view.accept_page(first, true).unwrap();
assert_eq!(view.videos.len(), PAGE_SIZE);
assert_eq!(view.videos.last().unwrap().timestamp, Some(first_base));
assert_eq!(view.pending.len(), 2);
view.selection.select(Some(PAGE_SIZE - 1));
let second =
youtube::feed_results(&format!("{}\n{short}", long_page(second_base))).unwrap();
view.accept_page(second, true).unwrap();
assert_eq!(view.videos.len(), PAGE_SIZE * 2);
assert_eq!(view.videos.last().unwrap().timestamp, Some(second_base));
assert_eq!(view.pending.len(), 2);
assert_eq!(view.selection.selected(), Some(PAGE_SIZE - 1));
view.accept_page(
youtube::feed_results(&format!(
r#"{{"entries":[{{"id":"l-GaaivVJCE","title":"Long last row","timestamp":{}}}]}} {{"entries":[]}}"#,
second_base - 1,
))
.unwrap(),
true,
).unwrap();
fn many_channel_feed_consumes_only_sixty_requested_rows() {
let mut app = app(&["tubular"]);
app.view_index = 1;
app.feed_channels = (0..75)
.map(|index| format!("https://www.youtube.com/@creator{index}"))
.collect();
let view = &mut app.views[1];
view.feed_requests = view.feed_cursor.plan(app.feed_channels.len()).unwrap();
assert_eq!(
view.videos[PAGE_SIZE * 2..]
view.feed_requests
.iter()
.map(|request| request.count)
.sum::<usize>(),
PAGE_SIZE
);
assert_eq!(view.feed_requests.len(), PAGE_SIZE);
let output = view
.feed_requests
.iter()
.map(|request| {
serde_json::json!({"entries":[{
"id":format!("channel{:04}", request.channel),
"title":"Upload",
"timestamp":request.channel,
}]})
.to_string()
})
.collect::<Vec<_>>()
.join("\n");
app.finish_page(Ok(output));
assert_eq!(app.views[1].videos.len(), PAGE_SIZE);
assert!(app.views[1].has_more);
assert!(!app.can_prefetch());
let next = app.views[1]
.feed_cursor
.plan(app.feed_channels.len())
.unwrap();
assert!(
next.iter()
.any(|request| request.channel >= PAGE_SIZE && request.start == 1)
);
assert_eq!(
next.iter().map(|request| request.count).sum::<usize>(),
PAGE_SIZE
);
app.views[1].feed_requests = next.clone();
app.views[1].prefetch_requested = true;
app.finish_page(Ok(r#"{"entries":[]}"#.into()));
assert!(app.views[1].page_error.is_some());
assert_eq!(app.views[1].videos.len(), PAGE_SIZE);
assert_eq!(app.views[1].next_start, PAGE_SIZE + 1);
assert_eq!(
app.views[1]
.feed_cursor
.plan(app.feed_channels.len())
.unwrap(),
next
);
assert!(app.views[1].prefetch_requested);
}
#[test]
fn loaded_feed_rows_are_visible_sorted_and_keep_selection_and_description_identity() {
let mut app = app(&["tubular"]);
app.view_index = 1;
app.feed_channels = vec![
"https://www.youtube.com/@one".into(),
"https://www.youtube.com/@two".into(),
];
// Keep both channels active despite a short visible page.
finish_feed_fixture(
&mut app,
r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Old","timestamp":100},{"id":"rQ_J9WH6CGk","title":"Undated"}]}
{"entries":[{"id":"l-GaaivVJCE","title":"Middle","timestamp":200}]}"#,
true,
);
assert_eq!(
app.views[1]
.videos
.iter()
.map(|video| video.id.as_str())
.collect::<Vec<_>>(),
["l-GaaivVJCE", "jNQXAC9IVRw", "rQ_J9WH6CGk"]
);
assert!(view.pending.is_empty());
assert!(!view.has_more);
assert!(!view.needs_page());
}
#[test]
fn unavailable_full_channel_pages_do_not_release_other_channels_ahead_of_newer_uploads() {
let mut view = View::new("Feed".into(), ViewSource::Feed);
let dated = |base: i64| {
serde_json::json!({
"entries": (0..PAGE_SIZE).map(|index| serde_json::json!({
"id": format!("dated{:06}", base + index as i64),
"title": "Dated upload",
"timestamp": base + index as i64,
})).collect::<Vec<_>>()
})
};
let unavailable = serde_json::json!({
"entries": vec![serde_json::Value::Null; PAGE_SIZE]
});
let page = youtube::feed_results(&format!("{}\n{unavailable}", dated(100))).unwrap();
view.accept_page(page, true).unwrap();
assert!(view.videos.is_empty());
assert_eq!(view.selection.selected(), None);
assert!(view.needs_page());
let page = youtube::feed_results(&format!("{{\"entries\":[]}}\n{}", dated(200))).unwrap();
view.accept_page(page, true).unwrap();
assert_eq!(
view.videos.first().unwrap().timestamp,
Some(200 + PAGE_SIZE as i64 - 1)
app.views[1].selection.select(Some(1));
let description_id = app.views[1].videos[1].id.clone();
app.preview = Some(PreviewMode::Info);
app.remember_description("l-GaaivVJCE".into(), "Keep cached preview".into());
finish_feed_fixture(
&mut app,
r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Duplicate","timestamp":100},{"id":"SLV9AJxnb1U","title":"Newer","timestamp":300}]}
{"entries":[]}"#,
false,
);
assert_eq!(view.videos.last().unwrap().timestamp, Some(200));
view.accept_page(youtube::feed_results(r#"{"entries":[]}"#).unwrap(), true)
.unwrap();
assert_eq!(
view.videos[PAGE_SIZE].timestamp,
Some(100 + PAGE_SIZE as i64 - 1)
app.views[1]
.videos
.iter()
.map(|video| video.id.as_str())
.collect::<Vec<_>>(),
["SLV9AJxnb1U", "l-GaaivVJCE", "jNQXAC9IVRw", "rQ_J9WH6CGk"]
);
assert_eq!(view.videos.last().unwrap().timestamp, Some(100));
assert!(!view.has_more);
}
#[test]
fn undated_full_feed_pages_stay_pending_and_do_not_look_exhausted() {
let mut app = app(&["tubular", "rust"]);
app.view_index = 1;
app.feed_channels
.push("https://www.youtube.com/@jawed".into());
app.config
.channels
.insert("@jawed".into(), app.feed_channels[0].clone());
let entries: Vec<_> = (0..PAGE_SIZE)
.map(|index| {
serde_json::json!({
"id": format!("unknown{index:04}"),
"title": "Undated",
})
})
.collect();
app.finish_page(Ok(serde_json::json!({"entries": entries}).to_string()));
assert!(app.views[1].videos.is_empty());
assert_eq!(app.views[1].pending.len(), PAGE_SIZE);
assert_eq!(app.views[1].next_start, PAGE_SIZE + 1);
assert!(app.views[1].loaded);
assert!(app.can_prefetch());
app.key(KeyEvent::new(KeyCode::Char('j'), KeyModifiers::NONE));
assert_eq!(app.views[1].selection.selected(), None);
assert!(app.job.is_none());
app.finish_page(Err("Continuation failed".into()));
assert_eq!(app.views[1].pending.len(), PAGE_SIZE);
assert_eq!(app.views[1].next_start, PAGE_SIZE + 1);
assert!(!app.can_prefetch());
app.finish_page(Ok(r#"{"entries":[]}"#.into()));
assert_eq!(app.views[1].videos.len(), PAGE_SIZE);
assert!(app.views[1].pending.is_empty());
assert_eq!(app.views[1].selection.selected(), Some(0));
assert!(!app.can_prefetch());
app.views[1].next_start = 81;
app.views[1].has_more = true;
app.views[1].page_error = Some("Refresh resets this".into());
app.key(KeyEvent::new(KeyCode::Char('r'), KeyModifiers::NONE));
assert_eq!(app.views[1].next_start, 1);
assert!(app.views[1].videos.is_empty());
assert!(app.views[1].pending.is_empty());
assert_eq!(app.views[1].selection.selected(), Some(2));
assert_eq!(app.views[1].videos[2].title, "Old");
app.finish_description(
description_id,
Ok(r#"{"id":"jNQXAC9IVRw","description":"Selected upload description"}"#.into()),
);
assert_eq!(
cached_description(&app.descriptions, "jNQXAC9IVRw"),
Some("Selected upload description")
);
assert_eq!(
cached_description(&app.descriptions, "l-GaaivVJCE"),
Some("Keep cached preview")
);
assert_eq!(cached_description(&app.descriptions, "SLV9AJxnb1U"), None);
assert_eq!(app.preview, Some(PreviewMode::Info));
assert!(!app.views[1].has_more);
assert!(app.views[1].page_error.is_none());
assert!(app.job.is_none());
}
#[test]
fn short_undated_or_invalid_feed_pages_require_new_forward_navigation() {
for visible in [0, 1, 19] {
let mut app = app(&["tubular"]);
app.view_index = 1;
app.feed_channels
.push("https://www.youtube.com/@jawed".into());
let mut entries: Vec<_> = (0..visible)
.map(|index| {
serde_json::json!({
"id":format!("unknown{index:04}"), "title":"Undated",
})
})
.collect();
entries.resize(PAGE_SIZE - 1, serde_json::Value::Null);
entries.push(serde_json::json!({"id":"bad", "title":"Invalid ID"}));
finish_feed_fixture(
&mut app,
&serde_json::json!({"entries":entries}).to_string(),
false,
);
assert_eq!(app.views[1].videos.len(), visible);
assert!(app.views[1].has_more);
assert!(!app.can_prefetch());
app.update();
assert!(app.job.is_none());
app.key(KeyEvent::new(KeyCode::Char('j'), KeyModifiers::NONE));
assert!(app.can_prefetch());
assert_eq!(
app.views[1].selection.selected(),
if visible == 0 {
None
} else {
Some((visible - 1).min(1))
}
);
finish_feed_fixture(&mut app, r#"{"entries":[]}"#, true);
assert!(app.views[1].has_more);
assert!(!app.can_prefetch());
app.key(KeyEvent::new(KeyCode::Char('G'), KeyModifiers::SHIFT));
assert!(app.views[1].prefetch_requested);
let next = app.views[1].feed_cursor.plan(1).unwrap();
app.finish_page(Err("Continuation failed".into()));
assert_eq!(app.views[1].feed_cursor.plan(1).unwrap(), next);
assert!(!app.can_prefetch());
app.views[1].reset();
assert!(app.views[1].videos.is_empty());
assert!(app.views[1].feed_requests.is_empty());
assert!(!app.views[1].prefetch_requested);
assert_eq!(app.views[1].feed_cursor.plan(1).unwrap()[0].start, 1);
}
}
#[test]
fn feed_cursors_and_staged_requests_survive_cancellation_and_history_independently() {
let mut app = app(&["tubular"]);
app.feed_channels
.push("https://www.youtube.com/@one".into());
app.config
.categories
.insert("Two".into(), vec!["https://www.youtube.com/@two".into()]);
app.views
.push(View::new("Two".into(), ViewSource::Category("Two".into())));
app.views[0].loaded = true;
for index in [1, 2] {
app.view_index = index;
finish_feed_fixture(&mut app, r#"{"entries":[]}"#, true);
}
app.view_index = 1;
app.views[1].prefetch_requested = true;
app.views[1].feed_requests = app.views[1].feed_cursor.plan(1).unwrap();
let cancelled = app.views[1].feed_requests.clone();
app.switch_view(2);
finish_feed_fixture(&mut app, r#"{"entries":[]}"#, true);
assert_eq!(
app.views[2].feed_cursor.plan(1).unwrap()[0].start,
PAGE_SIZE * 2 + 1
);
app.back_view();
assert_eq!(app.view_index, 1);
assert_eq!(app.views[1].feed_requests, cancelled);
assert_eq!(app.views[1].feed_cursor.plan(1).unwrap(), cancelled);
assert!(app.views[1].prefetch_requested);
app.forward_view();
assert_eq!(app.view_index, 2);
assert!(!app.views[2].prefetch_requested);
assert_eq!(
app.views[2].feed_cursor.plan(1).unwrap()[0].start,
PAGE_SIZE * 2 + 1
);
}
#[test]
fn category_membership_reordering_keeps_cursor_request_intent_and_cache() {
let mut app = App::new(
Options::parse_from(["tubular"]),
PathBuf::new(),
config::Config {
categories: [(
"Creators".into(),
vec![
"https://www.youtube.com/@two".into(),
"https://www.youtube.com/@one".into(),
],
)]
.into(),
..Default::default()
},
);
app.view_index = 2;
finish_feed_fixture(
&mut app,
r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Cached","timestamp":100}]}"#,
true,
);
app.remember_description("jNQXAC9IVRw".into(), "Retained".into());
app.views[2].feed_requests = app.views[2].feed_cursor.plan(2).unwrap();
app.views[2].prefetch_requested = true;
let requests = app.views[2].feed_requests.clone();
let mut categories = app.config.categories.clone();
categories.get_mut("Creators").unwrap().reverse();
app.apply_config(config::Config {
categories,
..Default::default()
});
assert_eq!(app.views[2].feed_requests, requests);
assert_eq!(app.views[2].feed_cursor.plan(2).unwrap(), requests);
assert!(app.views[2].prefetch_requested);
assert_eq!(app.views[2].videos[0].id, "jNQXAC9IVRw");
assert_eq!(app.views[2].selection.selected(), Some(0));
assert_eq!(
cached_description(&app.descriptions, "jNQXAC9IVRw"),
Some("Retained")
);
let mut categories = app.config.categories.clone();
categories
.get_mut("Creators")
.unwrap()
.push("https://www.youtube.com/@three".into());
app.apply_config(config::Config {
categories,
..Default::default()
});
assert!(!app.views[2].loaded);
assert!(app.views[2].videos.is_empty());
assert!(app.views[2].feed_requests.is_empty());
assert!(!app.views[2].prefetch_requested);
assert_eq!(app.views[2].next_start, 1);
assert!(
app.views[2]
.feed_cursor
.plan(3)
.unwrap()
.iter()
.all(|request| request.start == 1)
);
}
#[test]
@@ -7624,13 +7862,13 @@ tags.cancel = "z"
assert_eq!(app.views[0].next_start, PAGE_SIZE + 1);
assert!(!app.views[0].needs_page());
let view = &mut app.views[0];
view.pending =
youtube::search_results(r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Pending"}]}"#)
.unwrap()
.videos;
view.feed_requests = view.feed_cursor.plan(1).unwrap();
view.prefetch_requested = true;
view.reset();
assert!(view.videos.is_empty());
assert!(view.pending.is_empty());
assert!(view.feed_requests.is_empty());
assert!(!view.feed_cursor.has_more());
assert!(!view.prefetch_requested);
assert_eq!(view.selection.selected(), None);
assert_eq!(view.next_start, 1);
assert!(!view.loaded);
@@ -7649,6 +7887,8 @@ tags.cancel = "z"
app.views[0].accept_page(page, false).unwrap();
assert!(!app.can_prefetch());
app.mode = Mode::Normal;
assert!(!app.can_prefetch());
app.key(KeyEvent::new(KeyCode::Char('j'), KeyModifiers::NONE));
assert!(app.can_prefetch());
app.preview = Some(PreviewMode::Description);
assert!(!app.can_prefetch());
@@ -7945,10 +8185,10 @@ tags.cancel = "z"
pinned_categories: vec!["Rust".into()],
});
app.views[0].selection.select(Some(1));
app.views[1].videos = youtube::feed_results(videos).unwrap().videos;
app.views[1].videos = feed_fixture(videos).unwrap().videos;
app.views[1].loaded = true;
app.views[1].selection.select(Some(0));
app.views[2].videos = youtube::feed_results(videos).unwrap().videos;
app.views[2].videos = feed_fixture(videos).unwrap().videos;
app.views[2].loaded = true;
app.views[2].selection.select(Some(0));
app.key(KeyEvent::new(KeyCode::Tab, KeyModifiers::NONE));
@@ -8018,31 +8258,25 @@ tags.cancel = "z"
);
let videos = r#"{"entries":[{"id":"l-GaaivVJCE","title":"New","timestamp":200},{"id":"rQ_J9WH6CGk","title":"Old","timestamp":100}]}"#;
assert_eq!(app.feed_channels, vec![direct, registered]);
app.views[1].videos = youtube::feed_results(videos).unwrap().videos;
app.views[1].videos = feed_fixture(videos).unwrap().videos;
app.views[1].selection.select(Some(1));
app.views[1].loaded = true;
app.views[1].next_start = 41;
app.views[1].has_more = true;
app.views[1].pending = youtube::feed_results(
r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Buffered","timestamp":50}]}"#,
)
.unwrap()
.videos;
app.views[1].feed_requests = app.views[1].feed_cursor.plan(2).unwrap();
app.views[1].prefetch_requested = true;
app.views[1].page_error = Some("Retained page error".into());
app.remember_description(
app.views[1].videos[1].id.clone(),
"Retained description".into(),
);
app.views[3].videos = youtube::feed_results(videos).unwrap().videos;
app.views[3].videos = feed_fixture(videos).unwrap().videos;
app.views[3].selection.select(Some(0));
app.views[3].loaded = true;
app.views[3].next_start = 61;
app.views[3].has_more = true;
app.views[3].pending = youtube::feed_results(
r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Tag pending","timestamp":50}]}"#,
)
.unwrap()
.videos;
app.views[3].feed_requests = app.views[3].feed_cursor.plan(1).unwrap();
app.views[3].prefetch_requested = true;
app.views[3].page_error = Some("Tag page failed".into());
app.switch_view(1);
@@ -8063,7 +8297,8 @@ tags.cancel = "z"
assert_eq!(app.views[1].selection.selected(), Some(1));
assert_eq!(app.views[1].next_start, 41);
assert!(app.views[1].has_more);
assert_eq!(app.views[1].pending[0].id, "jNQXAC9IVRw");
assert_eq!(app.views[1].feed_requests.len(), 2);
assert!(app.views[1].prefetch_requested);
assert_eq!(
app.views[1].page_error.as_deref(),
Some("Retained page error")
@@ -8077,7 +8312,8 @@ tags.cancel = "z"
assert_eq!(app.views[4].selection.selected(), None);
assert_eq!(app.views[4].next_start, 1);
assert!(!app.views[4].has_more);
assert!(app.views[4].pending.is_empty());
assert!(app.views[4].feed_requests.is_empty());
assert!(!app.views[4].prefetch_requested);
assert!(app.views[4].page_error.is_none());
app.apply_config(config::Config {
@@ -8097,7 +8333,8 @@ tags.cancel = "z"
assert_eq!(app.views[1].selection.selected(), Some(1));
assert_eq!(app.views[1].next_start, 41);
assert!(app.views[1].has_more);
assert_eq!(app.views[1].pending.len(), 1);
assert_eq!(app.views[1].feed_requests.len(), 2);
assert!(app.views[1].prefetch_requested);
assert!(app.views[1].page_error.is_some());
let mut channels = app.config.channels.clone();
@@ -8114,12 +8351,13 @@ tags.cancel = "z"
assert_eq!(app.views[1].selection.selected(), None);
assert_eq!(app.views[1].next_start, 1);
assert!(!app.views[1].has_more);
assert!(app.views[1].pending.is_empty());
assert!(app.views[1].feed_requests.is_empty());
assert!(!app.views[1].prefetch_requested);
assert!(app.views[1].page_error.is_none());
assert_eq!(app.view_index, 1);
assert!(app.job.is_none());
app.views[1].videos = youtube::feed_results(videos).unwrap().videos;
app.views[1].videos = feed_fixture(videos).unwrap().videos;
app.views[1].selection.select(Some(1));
app.views[1].loaded = true;
let mut channels = app.config.channels.clone();
@@ -8153,7 +8391,7 @@ tags.cancel = "z"
});
let videos = r#"{"entries":[{"id":"l-GaaivVJCE","title":"First"},{"id":"rQ_J9WH6CGk","title":"Second"}]}"#;
for (index, view) in app.views.iter_mut().enumerate() {
view.videos = youtube::feed_results(videos).unwrap().videos;
view.videos = feed_fixture(videos).unwrap().videos;
view.loaded = true;
view.selection.select(Some(usize::from(index % 2 == 0)));
}
src/youtube.rs
+781 -150
View File
File diff suppressed because it is too large Load Diff