feat: paginate search and subscription feeds while scrolling
This commit is contained in:
+516
-77
@@ -27,7 +27,13 @@
|
||||
//! Tab/Shift-Tab switch Search, the combined Feed, then independent tag views; r reloads/refreshes.
|
||||
//! Feed includes saved `[channels]` entries, even untagged, plus direct tag entries, without duplicates.
|
||||
//! Untagging is not unsubscribing: remove mappings and tag references from the file to unsubscribe.
|
||||
//! Feeds merge up to 20 videos per channel's /videos tab, ordered by approximate upload dates.
|
||||
//! Search pages contain 20 results; Feed/tag pages request 20 uploads per channel's /videos tab.
|
||||
//! In normal mode, approaching the loaded end fetches the next page without replacing rows/selection.
|
||||
//! Feed rows use approximate upload dates; older rows wait until newer channel pages catch up.
|
||||
//! Undated feed rows appear at exhaustion. Cached views retain their page position across switches.
|
||||
//! Switching views cancels in-flight extraction; returning resumes the same pending range.
|
||||
//! Errors keep loaded rows and stop automatic retries; r reloads config and restarts the current view.
|
||||
//! Deep pages may slow down because yt-dlp offsets re-traverse earlier continuation pages.
|
||||
//! IINA receives `--stdin`; other players must read standard input when passed `-`.
|
||||
//! Audio and video are streamed through a pipe, never saved to a video file.
|
||||
//! See `--help` for controls; replacing playback or quitting cancels its processes.
|
||||
@@ -36,7 +42,13 @@ mod config;
|
||||
mod thumbnail;
|
||||
mod youtube;
|
||||
|
||||
use std::{collections::BTreeMap, ffi::OsString, io, path::PathBuf, time::Duration};
|
||||
use std::{
|
||||
collections::{BTreeMap, HashSet},
|
||||
ffi::OsString,
|
||||
io,
|
||||
path::PathBuf,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use clap::Parser;
|
||||
use ratatui::{
|
||||
@@ -51,13 +63,13 @@ use ratatui::{
|
||||
text::Line,
|
||||
widgets::{Block, Clear, List, ListState, Paragraph, Row, Table, TableState, Tabs, Wrap},
|
||||
};
|
||||
use youtube::{Job, Playback, Video};
|
||||
use youtube::{Job, PAGE_SIZE, Page, Playback, Video};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
version,
|
||||
about = "Search YouTube and stream videos to your chosen player",
|
||||
after_help = "Requires yt-dlp, FFmpeg, and a player on PATH; thumbnails also use curl.\nDefaults to IINA on macOS and mpv elsewhere; --player overrides the default.\nIINA receives '--stdin'; other players receive '-' to read audio/video from standard input.\nWithout a CLI query, start in INSERT; a CLI query is searched automatically in NORMAL.\nINSERT: type to edit; Enter submits if changed and returns to NORMAL; Esc returns without submitting.\nTEXT EDITING (search and new tag): Option/Alt+Backspace or Ctrl-W deletes the previous whitespace-delimited word, including trailing whitespace; Cmd/Super+Backspace or Ctrl-U clears the entire line. Plain Backspace removes one character.\nThe terminal must transmit modified Backspace. If it intercepts Option/Cmd, map those shortcuts to Ctrl-W/Ctrl-U respectively.\nNORMAL: j/k or arrows select; Enter streams; i edits the query; d toggles description; t toggles thumbnail; c edits channel tags; q quits.\nTab/Shift-Tab switch Search, Feed, then tag views; r reloads config and refreshes the current view.\nFeed sits immediately right of Search and combines every saved [channels] registration, including untagged channels, plus channels entered directly in tags.\nChannel URLs and videos are deduplicated; Feed keeps its cached selection when only tag assignments change.\nSubscriptions: $XDG_CONFIG_HOME/tubular/config.toml, or ~/.config/tubular/config.toml when XDG_CONFIG_HOME is unset/empty/relative.\nExample TOML:\n[categories]\nCreators = [\"@jawed\"]\nFavorites = [\"@jawed\"]\n[channels]\n\"@jawed\" = \"https://www.youtube.com/channel/UC4QobU6STFB0P71PMvOGN5A\"\n[categories] defines flat, overlapping tags, each with its own feed; there is no hierarchy or inheritance.\nEntries accept @handles, HTTPS YouTube channel URLs, or canonical UC channel IDs.\n[channels] registers saved handles with canonical channel URLs/IDs; feeds use these stable identities.\nChecked tags save the handle plus its mapping; existing URLs/aliases are replaced with the handle, preserving comments.\nUnchecked tags remove only that channel's tag assignment; other creators, omitted tags, empty tag arrays, and saved registrations are retained.\nUntagging does not unsubscribe from Feed. To unsubscribe, remove all of that channel's [channels] mappings and tag entries, then press r.\nUnmapped handles work directly. Existing URL entries still load.\nSaving requires a handle and canonical ID; unavailable metadata or conflicting mappings are reported without changing the file.\nAll feed views combine up to 20 videos per channel's /videos tab, newest-first using approximate upload dates.\nTAGS: existing assignments are checked; j/k or arrows move; Space toggles; Enter saves all staged memberships; n prompts for a new tag; Esc cancels without writing.\nNEW TAG: type a name; Enter stages it checked and returns to the list; Esc abandons the name; Ctrl-U clears.\nNew tags and membership changes reach disk only on Enter in the main tag list.\nOpening the description fetches full video metadata, preserving paragraphs and caching it for these results.\nPreview windows use up to 70% width and 60% height, capped at 96 by 20 cells.\nIn descriptions, Up/Down scroll lines and PgUp/PgDn scroll pages; j/k still selects videos.\nEsc closes the description; i closes it to edit the query. Scrolling stays within the text.\nTHUMBNAIL: t or Esc closes; j/k or arrows load the selected video's image; d switches to description.\nThumbnails are fetched lazily into memory and fit proportionally inside the popup on resize.\nTerminal graphics are detected at startup, with colored half-block rendering when image protocols are unavailable.\nDownloading, JPEG decoding, and image resizing/encoding do not block navigation.\ni, c, view changes, and refresh dismiss the thumbnail; typing t in INSERT or a tag name edits text.\nCtrl-C quits in any mode or popup.\nSelecting another video replaces playback. Quitting stops playback.\nExample: tubular --player mpv --player-arg=--volume=50 \"rust programming\""
|
||||
after_help = "Requires yt-dlp, FFmpeg, and a player on PATH; thumbnails also use curl.\nDefaults to IINA on macOS and mpv elsewhere; --player overrides the default.\nIINA receives '--stdin'; other players receive '-' to read audio/video from standard input.\nWithout a CLI query, start in INSERT; a CLI query is searched automatically in NORMAL.\nINSERT: type to edit; Enter submits if changed and returns to NORMAL; Esc returns without submitting.\nTEXT EDITING (search and new tag): Option/Alt+Backspace or Ctrl-W deletes the previous whitespace-delimited word, including trailing whitespace; Cmd/Super+Backspace or Ctrl-U clears the entire line. Plain Backspace removes one character.\nThe terminal must transmit modified Backspace. If it intercepts Option/Cmd, map those shortcuts to Ctrl-W/Ctrl-U respectively.\nNORMAL: j/k or arrows select; Enter streams; i edits the query; d toggles description; t toggles thumbnail; c edits channel tags; q quits.\nTab/Shift-Tab switch Search, Feed, then tag views; r reloads config and refreshes the current view from its first page.\nFeed sits immediately right of Search and combines every saved [channels] registration, including untagged channels, plus channels entered directly in tags.\nChannel URLs and videos are deduplicated; Feed keeps its cached selection and page position when only tag assignments change.\nSubscriptions: $XDG_CONFIG_HOME/tubular/config.toml, or ~/.config/tubular/config.toml when XDG_CONFIG_HOME is unset/empty/relative.\nExample TOML:\n[categories]\nCreators = [\"@jawed\"]\nFavorites = [\"@jawed\"]\n[channels]\n\"@jawed\" = \"https://www.youtube.com/channel/UC4QobU6STFB0P71PMvOGN5A\"\n[categories] defines flat, overlapping tags, each with its own feed; there is no hierarchy or inheritance.\nEntries accept @handles, HTTPS YouTube channel URLs, or canonical UC channel IDs.\n[channels] registers saved handles with canonical channel URLs/IDs; feeds use these stable identities.\nChecked tags save the handle plus its mapping; existing URLs/aliases are replaced with the handle, preserving comments.\nUnchecked tags remove only that channel's tag assignment; other creators, omitted tags, empty tag arrays, and saved registrations are retained.\nUntagging does not unsubscribe from Feed. To unsubscribe, remove all of that channel's [channels] mappings and tag entries, then press r.\nUnmapped handles work directly. Existing URL entries still load.\nSaving requires a handle and canonical ID; unavailable metadata or conflicting mappings are reported without changing the file.\nPAGINATION: Search loads 20 results per page; Feed and tags request 20 uploads per channel's /videos tab per page.\nMoving near the loaded end in NORMAL fetches more asynchronously, preserving existing rows, selection, and previews. Insert mode and open popups pause automatic prefetch.\nFeed views merge newest-first using approximate upload dates. Older uploads wait while newer channel pages catch up; undated uploads appear when the feed is exhausted.\nStatus shows the page and whether more is available or the end is reached. Switching views cancels pending extraction; returning resumes the same range with cached rows intact.\nPage errors retain loaded rows and stop automatic retries; r reloads config and restarts the current view from page one.\nDeep pages may take longer because yt-dlp offsets re-traverse earlier continuation pages.\nTAGS: existing assignments are checked; j/k or arrows move; Space toggles; Enter saves all staged memberships; n prompts for a new tag; Esc cancels without writing.\nNEW TAG: type a name; Enter stages it checked and returns to the list; Esc abandons the name; Ctrl-U clears.\nNew tags and membership changes reach disk only on Enter in the main tag list.\nOpening the description fetches full video metadata, preserving paragraphs and caching it for these results.\nPreview windows use up to 70% width and 60% height, capped at 96 by 20 cells.\nIn descriptions, Up/Down scroll lines and PgUp/PgDn scroll pages; j/k still selects videos.\nEsc closes the description; i closes it to edit the query. Scrolling stays within the text.\nTHUMBNAIL: t or Esc closes; j/k or arrows load the selected video's image; d switches to description.\nThumbnails are fetched lazily into memory and fit proportionally inside the popup on resize.\nTerminal graphics are detected at startup, with colored half-block rendering when image protocols are unavailable.\nDownloading, JPEG decoding, and image resizing/encoding do not block navigation.\ni, c, view changes, and refresh dismiss the thumbnail; typing t in INSERT or a tag name edits text.\nCtrl-C quits in any mode or popup.\nSelecting another video replaces playback. Quitting stops playback.\nExample: tubular --player mpv --player-arg=--volume=50 \"rust programming\""
|
||||
)]
|
||||
struct Options {
|
||||
/// Player executable (IINA or a player supporting '-' for standard input).
|
||||
@@ -84,6 +96,10 @@ struct View {
|
||||
videos: Vec<Video>,
|
||||
selection: TableState,
|
||||
loaded: bool,
|
||||
next_start: usize,
|
||||
has_more: bool,
|
||||
pending: Vec<Video>,
|
||||
page_error: Option<String>,
|
||||
}
|
||||
|
||||
impl View {
|
||||
@@ -94,8 +110,81 @@ impl View {
|
||||
videos: Vec::new(),
|
||||
selection: TableState::default(),
|
||||
loaded: false,
|
||||
next_start: 1,
|
||||
has_more: false,
|
||||
pending: Vec::new(),
|
||||
page_error: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn reset(&mut self) {
|
||||
self.videos.clear();
|
||||
self.selection = TableState::default();
|
||||
self.loaded = false;
|
||||
self.next_start = 1;
|
||||
self.has_more = false;
|
||||
self.pending.clear();
|
||||
self.page_error = None;
|
||||
}
|
||||
|
||||
fn accept_page(&mut self, mut page: Page, feed: bool) -> Result<(), String> {
|
||||
let next_start = self
|
||||
.next_start
|
||||
.checked_add(PAGE_SIZE)
|
||||
.ok_or("Page offset exceeds the supported range")?;
|
||||
let mut videos = if feed {
|
||||
self.pending.append(&mut page.videos);
|
||||
std::mem::take(&mut self.pending)
|
||||
} else {
|
||||
page.videos
|
||||
};
|
||||
if feed {
|
||||
videos.sort_by(|a, b| b.timestamp.cmp(&a.timestamp));
|
||||
}
|
||||
if !videos.is_empty() {
|
||||
// Borrow IDs before moving rows, retaining existing rows and their description caches.
|
||||
let keep: Vec<_> = {
|
||||
let mut ids = HashSet::with_capacity(self.videos.len() + videos.len());
|
||||
ids.extend(self.videos.iter().map(|video| video.id.as_str()));
|
||||
videos
|
||||
.iter()
|
||||
.map(|video| ids.insert(video.id.as_str()))
|
||||
.collect()
|
||||
};
|
||||
let mut keep = keep.into_iter();
|
||||
videos.retain(|_| keep.next().expect("one deduplication flag per video"));
|
||||
}
|
||||
let ready = if feed && page.has_more {
|
||||
page.cutoff.map_or(0, |cutoff| {
|
||||
videos.partition_point(|video| video.timestamp.is_some_and(|date| date >= cutoff))
|
||||
})
|
||||
} else {
|
||||
videos.len()
|
||||
};
|
||||
self.videos.extend(videos.drain(..ready));
|
||||
if feed {
|
||||
self.pending = videos;
|
||||
}
|
||||
if self.selection.selected().is_none() && !self.videos.is_empty() {
|
||||
self.selection.select(Some(0));
|
||||
}
|
||||
self.loaded = true;
|
||||
self.next_start = next_start;
|
||||
self.has_more = page.has_more;
|
||||
self.page_error = None;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn needs_page(&self) -> bool {
|
||||
self.loaded
|
||||
&& self.has_more
|
||||
&& self.page_error.is_none()
|
||||
&& (self.videos.is_empty()
|
||||
|| self
|
||||
.selection
|
||||
.selected()
|
||||
.is_some_and(|index| index.saturating_add(5) >= self.videos.len()))
|
||||
}
|
||||
}
|
||||
|
||||
struct CategoryDialog {
|
||||
@@ -176,7 +265,7 @@ impl App {
|
||||
}
|
||||
if query != self.submitted_query {
|
||||
self.submitted_query = query.to_owned();
|
||||
self.views[0].loaded = false;
|
||||
self.views[0].reset();
|
||||
self.job = None;
|
||||
}
|
||||
self.switch_view(0);
|
||||
@@ -191,20 +280,12 @@ impl App {
|
||||
self.description_open = false;
|
||||
self.description_job = None;
|
||||
self.description_viewport = None;
|
||||
if !self.views[index].loaded {
|
||||
if !self.views[index].loaded && self.views[index].page_error.is_none() {
|
||||
if self.job.is_none() {
|
||||
self.load_view();
|
||||
self.load_page();
|
||||
}
|
||||
} else {
|
||||
self.status = if index == 1 && self.feed_channels.is_empty() {
|
||||
"Feed has no channels. Press c on a search result to add one.".into()
|
||||
} else {
|
||||
format!(
|
||||
"{} videos in {}. Select a video to stream.",
|
||||
self.views[index].videos.len(),
|
||||
self.views[index].name
|
||||
)
|
||||
};
|
||||
} else if self.job.is_none() {
|
||||
self.status = self.view_status();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -213,17 +294,23 @@ impl App {
|
||||
self.description_open = false;
|
||||
self.description_job = None;
|
||||
self.close_thumbnail();
|
||||
self.views[self.view_index].reset();
|
||||
self.load_page();
|
||||
}
|
||||
|
||||
fn load_page(&mut self) {
|
||||
if self.job.is_some() {
|
||||
return;
|
||||
}
|
||||
let view = &mut self.views[self.view_index];
|
||||
view.videos.clear();
|
||||
view.selection = TableState::default();
|
||||
view.loaded = false;
|
||||
let page = (view.next_start - 1) / PAGE_SIZE + 1;
|
||||
let result = if self.view_index == 0 {
|
||||
if self.submitted_query.is_empty() {
|
||||
self.status = "Type a query, then press Enter.".into();
|
||||
return;
|
||||
}
|
||||
self.status = format!("Searching for {}...", self.submitted_query);
|
||||
Job::search(&self.submitted_query)
|
||||
self.status = format!("Searching for {} (page {page})...", self.submitted_query);
|
||||
Job::search(&self.submitted_query, view.next_start)
|
||||
} else {
|
||||
let channels = if self.view_index == 1 {
|
||||
&self.feed_channels
|
||||
@@ -232,20 +319,94 @@ impl App {
|
||||
};
|
||||
if channels.is_empty() {
|
||||
view.loaded = true;
|
||||
view.has_more = false;
|
||||
self.status = format!(
|
||||
"{} has no channels. Press c on a search result to add one.",
|
||||
view.name
|
||||
);
|
||||
return;
|
||||
}
|
||||
self.status = format!("Loading {}...", view.name);
|
||||
Job::feed(channels)
|
||||
self.status = format!(
|
||||
"Loading {} page {page} ({} videos kept)...",
|
||||
view.name,
|
||||
view.videos.len()
|
||||
);
|
||||
Job::feed(channels, view.next_start)
|
||||
};
|
||||
match result {
|
||||
Ok(job) => self.job = Some(job),
|
||||
Err(error) => self.status = error.to_string(),
|
||||
Err(error) => {
|
||||
view.page_error = Some(error.to_string());
|
||||
self.status = self.view_status();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn view_status(&self) -> String {
|
||||
let view = &self.views[self.view_index];
|
||||
if let Some(error) = &view.page_error {
|
||||
return format!(
|
||||
"{} videos kept in {}. Page {} failed: {error}. Press r to refresh.",
|
||||
view.videos.len(),
|
||||
view.name,
|
||||
(view.next_start - 1) / PAGE_SIZE + 1
|
||||
);
|
||||
}
|
||||
if self.view_index != 0
|
||||
&& (if self.view_index == 1 {
|
||||
&self.feed_channels
|
||||
} else {
|
||||
&self.config.categories[&view.name]
|
||||
})
|
||||
.is_empty()
|
||||
{
|
||||
return format!(
|
||||
"{} has no channels. Press c on a search result to add one.",
|
||||
view.name
|
||||
);
|
||||
}
|
||||
let continuation = if view.has_more {
|
||||
if view.videos.is_empty() && !view.pending.is_empty() {
|
||||
"newer pages pending"
|
||||
} else {
|
||||
"more available"
|
||||
}
|
||||
} else {
|
||||
"end"
|
||||
};
|
||||
format!(
|
||||
"{} videos in {} (page {}; {continuation}). Select a video to stream.",
|
||||
view.videos.len(),
|
||||
view.name,
|
||||
(view.next_start - 1) / PAGE_SIZE
|
||||
)
|
||||
}
|
||||
|
||||
fn finish_page(&mut self, result: Result<String, String>) {
|
||||
self.job = None;
|
||||
let parsed = result.and_then(|output| {
|
||||
if self.view_index == 0 {
|
||||
youtube::search_results(&output)
|
||||
} else {
|
||||
youtube::feed_results(&output)
|
||||
}
|
||||
});
|
||||
if let Err(error) = parsed
|
||||
.and_then(|page| self.views[self.view_index].accept_page(page, self.view_index != 0))
|
||||
{
|
||||
self.views[self.view_index].page_error = Some(error);
|
||||
}
|
||||
self.status = self.view_status();
|
||||
}
|
||||
|
||||
fn can_prefetch(&self) -> bool {
|
||||
self.job.is_none()
|
||||
&& self.mode == Mode::Normal
|
||||
&& self.category_dialog.is_none()
|
||||
&& !self.description_open
|
||||
&& !self.thumbnail_open
|
||||
&& self.views[self.view_index].needs_page()
|
||||
}
|
||||
|
||||
fn apply_config(&mut self, config: config::Config) {
|
||||
if config.categories == self.config.categories && config.channels == self.config.channels {
|
||||
@@ -258,9 +419,7 @@ impl App {
|
||||
self.views.push(old.next().expect("search view exists"));
|
||||
let mut feed = old.next().expect("feed view exists");
|
||||
if feed_channels != self.feed_channels {
|
||||
feed.loaded = false;
|
||||
feed.videos.clear();
|
||||
feed.selection = TableState::default();
|
||||
feed.reset();
|
||||
}
|
||||
self.views.push(feed);
|
||||
let mut old: Vec<_> = old.collect();
|
||||
@@ -272,9 +431,7 @@ impl App {
|
||||
View::new(name.clone())
|
||||
};
|
||||
if config.categories.get(name) != self.config.categories.get(name) {
|
||||
view.loaded = false;
|
||||
view.videos.clear();
|
||||
view.selection = TableState::default();
|
||||
view.reset();
|
||||
}
|
||||
self.views.push(view);
|
||||
}
|
||||
@@ -494,9 +651,6 @@ impl App {
|
||||
}
|
||||
|
||||
fn play(&mut self) {
|
||||
if self.job.is_some() {
|
||||
return;
|
||||
}
|
||||
let view = &self.views[self.view_index];
|
||||
let Some(video) = view
|
||||
.selection
|
||||
@@ -520,34 +674,7 @@ impl App {
|
||||
|
||||
fn update(&mut self) {
|
||||
if let Some(result) = self.job.as_mut().and_then(Job::poll) {
|
||||
self.job = None;
|
||||
let parsed = result.and_then(|output| {
|
||||
if self.view_index == 0 {
|
||||
youtube::search_results(&output)
|
||||
} else {
|
||||
youtube::feed_results(&output)
|
||||
}
|
||||
});
|
||||
self.status = match parsed {
|
||||
Ok(videos) => {
|
||||
let view = &mut self.views[self.view_index];
|
||||
view.videos = videos;
|
||||
view.loaded = true;
|
||||
view.selection = TableState::default();
|
||||
view.selection
|
||||
.select((!view.videos.is_empty()).then_some(0));
|
||||
if self.view_index == 0 {
|
||||
format!("{} results. Select a video to stream.", view.videos.len())
|
||||
} else {
|
||||
format!(
|
||||
"{} videos in {}. Select a video to stream.",
|
||||
view.videos.len(),
|
||||
view.name
|
||||
)
|
||||
}
|
||||
}
|
||||
Err(error) => error,
|
||||
};
|
||||
self.finish_page(result);
|
||||
}
|
||||
if let Some(result) = self
|
||||
.description_job
|
||||
@@ -573,6 +700,9 @@ impl App {
|
||||
Err(error) => error,
|
||||
};
|
||||
}
|
||||
if self.can_prefetch() {
|
||||
self.load_page();
|
||||
}
|
||||
}
|
||||
|
||||
fn key(&mut self, key: KeyEvent) -> bool {
|
||||
@@ -685,11 +815,9 @@ impl App {
|
||||
}
|
||||
}
|
||||
let view = &mut self.views[self.view_index];
|
||||
if let Some(index) = view
|
||||
.selection
|
||||
.selected()
|
||||
.filter(|_| !view.videos.is_empty())
|
||||
{
|
||||
if view.videos.is_empty() {
|
||||
view.selection.select(None);
|
||||
} else if let Some(index) = view.selection.selected() {
|
||||
view.selection
|
||||
.select(Some(index.min(view.videos.len() - 1)));
|
||||
}
|
||||
@@ -1041,6 +1169,278 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_pages_preserve_relevance_selection_and_cached_descriptions() {
|
||||
let mut view = View::new("Search".into());
|
||||
let mut first = youtube::search_results(
|
||||
r#"{"entries":[{"id":"jNQXAC9IVRw","title":"First","timestamp":100},{"id":"rQ_J9WH6CGk","title":"Second","timestamp":200}]}"#,
|
||||
)
|
||||
.unwrap();
|
||||
first.has_more = true;
|
||||
view.accept_page(first, false).unwrap();
|
||||
view.selection.select(Some(1));
|
||||
view.videos[1].description = Some("Cached full description.".into());
|
||||
let mut next = youtube::search_results(
|
||||
r#"{"entries":[{"id":"rQ_J9WH6CGk","title":"Duplicate"},{"id":"l-GaaivVJCE","title":"Relevant","timestamp":50},{"id":"l-GaaivVJCE","title":"Repeated"},{"id":"aaaaaaaaaaa","title":"Later","timestamp":300}]}"#,
|
||||
)
|
||||
.unwrap();
|
||||
next.has_more = true;
|
||||
view.accept_page(next, false).unwrap();
|
||||
assert_eq!(
|
||||
view.videos
|
||||
.iter()
|
||||
.map(|video| video.id.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["jNQXAC9IVRw", "rQ_J9WH6CGk", "l-GaaivVJCE", "aaaaaaaaaaa"]
|
||||
);
|
||||
assert_eq!(view.videos[1].title, "Second");
|
||||
assert_eq!(
|
||||
view.videos[1].description.as_deref(),
|
||||
Some("Cached full description.")
|
||||
);
|
||||
assert_eq!(view.selection.selected(), Some(1));
|
||||
assert_eq!(view.next_start, 41);
|
||||
assert!(view.has_more);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn feed_pages_hold_short_channel_tails_until_newer_long_histories_finish() {
|
||||
let mut view = View::new("Feed".into());
|
||||
let long_page = |base: i64| {
|
||||
serde_json::json!({
|
||||
"entries": (0..PAGE_SIZE).map(|index| serde_json::json!({
|
||||
"id": format!("long{:07}", base + index as i64),
|
||||
"title": "Long channel",
|
||||
"timestamp": base + index as i64,
|
||||
})).collect::<Vec<_>>()
|
||||
})
|
||||
.to_string()
|
||||
};
|
||||
let short = r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Short old tail","timestamp":100},{"id":"rQ_J9WH6CGk","title":"Unknown date"}]}"#;
|
||||
let first = youtube::feed_results(&format!("{}\n{short}", long_page(200))).unwrap();
|
||||
view.accept_page(first, true).unwrap();
|
||||
assert_eq!(view.videos.len(), PAGE_SIZE);
|
||||
assert_eq!(view.videos.last().unwrap().timestamp, Some(200));
|
||||
assert_eq!(view.pending.len(), 2);
|
||||
view.selection.select(Some(PAGE_SIZE - 1));
|
||||
view.videos[PAGE_SIZE - 1].description = Some("Keep this cache.".into());
|
||||
let second = youtube::feed_results(&format!("{}\n{short}", long_page(180))).unwrap();
|
||||
view.accept_page(second, true).unwrap();
|
||||
assert_eq!(view.videos.len(), PAGE_SIZE * 2);
|
||||
assert_eq!(view.videos.last().unwrap().timestamp, Some(180));
|
||||
assert_eq!(view.pending.len(), 2);
|
||||
assert_eq!(view.selection.selected(), Some(PAGE_SIZE - 1));
|
||||
assert_eq!(
|
||||
view.videos[PAGE_SIZE - 1].description.as_deref(),
|
||||
Some("Keep this cache.")
|
||||
);
|
||||
view.accept_page(
|
||||
youtube::feed_results(
|
||||
r#"{"entries":[{"id":"l-GaaivVJCE","title":"Long last row","timestamp":179}]} {"entries":[]}"#,
|
||||
).unwrap(),
|
||||
true,
|
||||
).unwrap();
|
||||
assert_eq!(
|
||||
view.videos[PAGE_SIZE * 2..]
|
||||
.iter()
|
||||
.map(|video| video.id.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
["l-GaaivVJCE", "jNQXAC9IVRw", "rQ_J9WH6CGk"]
|
||||
);
|
||||
assert!(view.pending.is_empty());
|
||||
assert!(!view.has_more);
|
||||
assert!(!view.needs_page());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unavailable_full_channel_pages_do_not_release_other_channels_ahead_of_newer_uploads() {
|
||||
let mut view = View::new("Feed".into());
|
||||
let dated = |base: i64| {
|
||||
serde_json::json!({
|
||||
"entries": (0..PAGE_SIZE).map(|index| serde_json::json!({
|
||||
"id": format!("dated{:06}", base + index as i64),
|
||||
"title": "Dated upload",
|
||||
"timestamp": base + index as i64,
|
||||
})).collect::<Vec<_>>()
|
||||
})
|
||||
};
|
||||
let unavailable = serde_json::json!({
|
||||
"entries": vec![serde_json::Value::Null; PAGE_SIZE]
|
||||
});
|
||||
let page = youtube::feed_results(&format!("{}\n{unavailable}", dated(100))).unwrap();
|
||||
view.accept_page(page, true).unwrap();
|
||||
assert!(view.videos.is_empty());
|
||||
assert_eq!(view.selection.selected(), None);
|
||||
assert!(view.needs_page());
|
||||
let page = youtube::feed_results(&format!("{{\"entries\":[]}}\n{}", dated(200))).unwrap();
|
||||
view.accept_page(page, true).unwrap();
|
||||
assert_eq!(view.videos.first().unwrap().timestamp, Some(219));
|
||||
assert_eq!(view.videos.last().unwrap().timestamp, Some(200));
|
||||
view.accept_page(youtube::feed_results(r#"{"entries":[]}"#).unwrap(), true)
|
||||
.unwrap();
|
||||
assert_eq!(view.videos[PAGE_SIZE].timestamp, Some(119));
|
||||
assert_eq!(view.videos.last().unwrap().timestamp, Some(100));
|
||||
assert!(!view.has_more);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn undated_full_feed_pages_stay_pending_and_do_not_look_exhausted() {
|
||||
let mut app = app(&["tubular", "rust"]);
|
||||
app.view_index = 1;
|
||||
app.feed_channels
|
||||
.push("https://www.youtube.com/@jawed".into());
|
||||
app.config
|
||||
.channels
|
||||
.insert("@jawed".into(), app.feed_channels[0].clone());
|
||||
let entries: Vec<_> = (0..PAGE_SIZE)
|
||||
.map(|index| {
|
||||
serde_json::json!({
|
||||
"id": format!("unknown{index:04}"),
|
||||
"title": "Undated",
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
app.finish_page(Ok(serde_json::json!({"entries": entries}).to_string()));
|
||||
assert!(app.views[1].videos.is_empty());
|
||||
assert_eq!(app.views[1].pending.len(), PAGE_SIZE);
|
||||
assert_eq!(app.views[1].next_start, 21);
|
||||
assert!(app.views[1].loaded);
|
||||
assert!(app.can_prefetch());
|
||||
app.key(KeyEvent::new(KeyCode::Char('j'), KeyModifiers::NONE));
|
||||
assert_eq!(app.views[1].selection.selected(), None);
|
||||
assert!(app.job.is_none());
|
||||
app.finish_page(Err("Continuation failed".into()));
|
||||
assert_eq!(app.views[1].pending.len(), PAGE_SIZE);
|
||||
assert_eq!(app.views[1].next_start, 21);
|
||||
assert!(!app.can_prefetch());
|
||||
app.finish_page(Ok(r#"{"entries":[]}"#.into()));
|
||||
assert_eq!(app.views[1].videos.len(), PAGE_SIZE);
|
||||
assert!(app.views[1].pending.is_empty());
|
||||
assert_eq!(app.views[1].selection.selected(), Some(0));
|
||||
assert!(!app.can_prefetch());
|
||||
app.views[1].next_start = 81;
|
||||
app.views[1].has_more = true;
|
||||
app.views[1].page_error = Some("Refresh resets this".into());
|
||||
app.key(KeyEvent::new(KeyCode::Char('r'), KeyModifiers::NONE));
|
||||
assert_eq!(app.views[1].next_start, 1);
|
||||
assert!(app.views[1].videos.is_empty());
|
||||
assert!(app.views[1].pending.is_empty());
|
||||
assert!(!app.views[1].has_more);
|
||||
assert!(app.views[1].page_error.is_none());
|
||||
assert!(app.job.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_failures_and_view_switches_preserve_the_successful_cursor_until_reset() {
|
||||
let mut app = app(&["tubular", "rust"]);
|
||||
app.submitted_query = "rust".into();
|
||||
app.finish_page(Err("First request failed".into()));
|
||||
assert_eq!(app.views[0].next_start, 1);
|
||||
assert!(!app.views[0].loaded);
|
||||
assert!(app.views[0].page_error.is_some());
|
||||
app.switch_view(0);
|
||||
app.update();
|
||||
assert!(app.job.is_none());
|
||||
app.views[0].reset();
|
||||
let entries: Vec<_> = (0..PAGE_SIZE)
|
||||
.map(|index| {
|
||||
serde_json::json!({
|
||||
"id": format!("search{index:05}"),
|
||||
"title": "Result",
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
app.finish_page(Ok(serde_json::json!({"entries": entries}).to_string()));
|
||||
assert_eq!(app.views[0].next_start, 21);
|
||||
app.views[0].selection.select(Some(17));
|
||||
app.views[0].videos[17].description = Some("Cached.".into());
|
||||
app.switch_view(1);
|
||||
app.switch_view(0);
|
||||
assert_eq!(app.views[0].next_start, 21);
|
||||
assert_eq!(app.views[0].selection.selected(), Some(17));
|
||||
app.finish_page(Ok("not JSON".into()));
|
||||
app.update();
|
||||
assert!(app.job.is_none());
|
||||
assert_eq!(app.views[0].videos.len(), PAGE_SIZE);
|
||||
assert_eq!(
|
||||
app.views[0].videos[17].description.as_deref(),
|
||||
Some("Cached.")
|
||||
);
|
||||
assert_eq!(app.views[0].next_start, 21);
|
||||
assert!(!app.views[0].needs_page());
|
||||
let view = &mut app.views[0];
|
||||
view.pending =
|
||||
youtube::search_results(r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Pending"}]}"#)
|
||||
.unwrap()
|
||||
.videos;
|
||||
view.reset();
|
||||
assert!(view.videos.is_empty());
|
||||
assert!(view.pending.is_empty());
|
||||
assert_eq!(view.selection.selected(), None);
|
||||
assert_eq!(view.next_start, 1);
|
||||
assert!(!view.loaded);
|
||||
assert!(!view.has_more);
|
||||
assert!(view.page_error.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prefetch_only_runs_near_the_end_in_an_unobstructed_normal_view() {
|
||||
let mut app = app(&["tubular"]);
|
||||
let mut page =
|
||||
youtube::search_results(r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Result"}]}"#)
|
||||
.unwrap();
|
||||
page.has_more = true;
|
||||
app.views[0].accept_page(page, false).unwrap();
|
||||
assert!(!app.can_prefetch());
|
||||
app.mode = Mode::Normal;
|
||||
assert!(app.can_prefetch());
|
||||
app.description_open = true;
|
||||
assert!(!app.can_prefetch());
|
||||
app.description_open = false;
|
||||
app.thumbnail_open = true;
|
||||
assert!(!app.can_prefetch());
|
||||
app.thumbnail_open = false;
|
||||
app.category_dialog = Some(CategoryDialog {
|
||||
channel: String::new(),
|
||||
handle: String::new(),
|
||||
label: String::new(),
|
||||
tags: BTreeMap::new(),
|
||||
selection: ListState::default(),
|
||||
name: None,
|
||||
error: None,
|
||||
});
|
||||
assert!(!app.can_prefetch());
|
||||
app.category_dialog = None;
|
||||
let entries: Vec<_> = (0..PAGE_SIZE)
|
||||
.map(|index| {
|
||||
serde_json::json!({
|
||||
"id": format!("search{index:05}"),
|
||||
"title": "Next result",
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
app.views[0]
|
||||
.accept_page(
|
||||
youtube::search_results(&serde_json::json!({"entries": entries}).to_string())
|
||||
.unwrap(),
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(app.views[0].selection.selected(), Some(0));
|
||||
assert!(!app.can_prefetch());
|
||||
app.views[0].selection.select(Some(15));
|
||||
assert!(!app.can_prefetch());
|
||||
app.key(KeyEvent::new(KeyCode::Char('j'), KeyModifiers::NONE));
|
||||
assert!(app.can_prefetch());
|
||||
assert!(app.job.is_none());
|
||||
app.views[0].page_error = Some("Stop retrying".into());
|
||||
assert!(!app.can_prefetch());
|
||||
app.views[0].page_error = None;
|
||||
app.views[0].videos.clear();
|
||||
app.views[0].selection.select(None);
|
||||
assert!(app.can_prefetch());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn input_cursor_tracks_display_width_and_stays_inside_scrolled_fields() {
|
||||
use ratatui::{Terminal, backend::TestBackend};
|
||||
@@ -1219,7 +1619,7 @@ mod tests {
|
||||
fn category_dialog_is_modal_and_views_keep_independent_selections() {
|
||||
let mut app = app(&["tubular", "rust"]);
|
||||
let videos = r#"{"entries":[{"id":"l-GaaivVJCE","title":"Rust video","channel":"Rust","channel_id":"UCaYhcUwRBNscFNUKTjgPFiA","uploader_id":"@RustVideos"},{"id":"rQ_J9WH6CGk","title":"Another video"}]}"#;
|
||||
app.views[0].videos = youtube::search_results(videos).unwrap();
|
||||
app.views[0].videos = youtube::search_results(videos).unwrap().videos;
|
||||
app.views[0].loaded = true;
|
||||
app.views[0].selection.select(Some(1));
|
||||
// The selected channel is unavailable; do not save a guessed display-name identity.
|
||||
@@ -1267,10 +1667,10 @@ mod tests {
|
||||
channels: BTreeMap::new(),
|
||||
});
|
||||
app.views[0].selection.select(Some(1));
|
||||
app.views[1].videos = youtube::feed_results(videos).unwrap();
|
||||
app.views[1].videos = youtube::feed_results(videos).unwrap().videos;
|
||||
app.views[1].loaded = true;
|
||||
app.views[1].selection.select(Some(0));
|
||||
app.views[2].videos = youtube::feed_results(videos).unwrap();
|
||||
app.views[2].videos = youtube::feed_results(videos).unwrap().videos;
|
||||
app.views[2].loaded = true;
|
||||
app.views[2].selection.select(Some(0));
|
||||
app.key(KeyEvent::new(KeyCode::Tab, KeyModifiers::NONE));
|
||||
@@ -1334,12 +1734,29 @@ mod tests {
|
||||
);
|
||||
let videos = r#"{"entries":[{"id":"l-GaaivVJCE","title":"New","timestamp":200},{"id":"rQ_J9WH6CGk","title":"Old","timestamp":100}]}"#;
|
||||
assert_eq!(app.feed_channels, vec![direct, registered]);
|
||||
app.views[1].videos = youtube::feed_results(videos).unwrap();
|
||||
app.views[1].videos = youtube::feed_results(videos).unwrap().videos;
|
||||
app.views[1].selection.select(Some(1));
|
||||
app.views[1].loaded = true;
|
||||
app.views[3].videos = youtube::feed_results(videos).unwrap();
|
||||
app.views[1].next_start = 41;
|
||||
app.views[1].has_more = true;
|
||||
app.views[1].pending = youtube::feed_results(
|
||||
r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Buffered","timestamp":50}]}"#,
|
||||
)
|
||||
.unwrap()
|
||||
.videos;
|
||||
app.views[1].page_error = Some("Retained page error".into());
|
||||
app.views[1].videos[1].description = Some("Retained description".into());
|
||||
app.views[3].videos = youtube::feed_results(videos).unwrap().videos;
|
||||
app.views[3].selection.select(Some(0));
|
||||
app.views[3].loaded = true;
|
||||
app.views[3].next_start = 61;
|
||||
app.views[3].has_more = true;
|
||||
app.views[3].pending = youtube::feed_results(
|
||||
r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Tag pending","timestamp":50}]}"#,
|
||||
)
|
||||
.unwrap()
|
||||
.videos;
|
||||
app.views[3].page_error = Some("Tag page failed".into());
|
||||
app.switch_view(1);
|
||||
|
||||
app.apply_config(config::Config {
|
||||
@@ -1355,9 +1772,24 @@ mod tests {
|
||||
assert!(app.views[1].loaded);
|
||||
assert_eq!(app.views[1].videos[1].id, "rQ_J9WH6CGk");
|
||||
assert_eq!(app.views[1].selection.selected(), Some(1));
|
||||
assert_eq!(app.views[1].next_start, 41);
|
||||
assert!(app.views[1].has_more);
|
||||
assert_eq!(app.views[1].pending[0].id, "jNQXAC9IVRw");
|
||||
assert_eq!(
|
||||
app.views[1].page_error.as_deref(),
|
||||
Some("Retained page error")
|
||||
);
|
||||
assert_eq!(
|
||||
app.views[1].videos[1].description.as_deref(),
|
||||
Some("Retained description")
|
||||
);
|
||||
assert!(!app.views[4].loaded);
|
||||
assert!(app.views[4].videos.is_empty());
|
||||
assert_eq!(app.views[4].selection.selected(), None);
|
||||
assert_eq!(app.views[4].next_start, 1);
|
||||
assert!(!app.views[4].has_more);
|
||||
assert!(app.views[4].pending.is_empty());
|
||||
assert!(app.views[4].page_error.is_none());
|
||||
|
||||
app.apply_config(config::Config {
|
||||
channels: app.config.channels.clone(),
|
||||
@@ -1372,6 +1804,10 @@ mod tests {
|
||||
assert!(app.views[1].loaded);
|
||||
assert_eq!(app.views[1].videos.len(), 2);
|
||||
assert_eq!(app.views[1].selection.selected(), Some(1));
|
||||
assert_eq!(app.views[1].next_start, 41);
|
||||
assert!(app.views[1].has_more);
|
||||
assert_eq!(app.views[1].pending.len(), 1);
|
||||
assert!(app.views[1].page_error.is_some());
|
||||
|
||||
let mut channels = app.config.channels.clone();
|
||||
channels.insert("@YouTube".into(), added.into());
|
||||
@@ -1383,10 +1819,14 @@ mod tests {
|
||||
assert!(!app.views[1].loaded);
|
||||
assert!(app.views[1].videos.is_empty());
|
||||
assert_eq!(app.views[1].selection.selected(), None);
|
||||
assert_eq!(app.views[1].next_start, 1);
|
||||
assert!(!app.views[1].has_more);
|
||||
assert!(app.views[1].pending.is_empty());
|
||||
assert!(app.views[1].page_error.is_none());
|
||||
assert_eq!(app.view_index, 1);
|
||||
assert!(app.job.is_none());
|
||||
|
||||
app.views[1].videos = youtube::feed_results(videos).unwrap();
|
||||
app.views[1].videos = youtube::feed_results(videos).unwrap().videos;
|
||||
app.views[1].selection.select(Some(1));
|
||||
app.views[1].loaded = true;
|
||||
let mut channels = app.config.channels.clone();
|
||||
@@ -1416,7 +1856,7 @@ mod tests {
|
||||
});
|
||||
let videos = r#"{"entries":[{"id":"l-GaaivVJCE","title":"First"},{"id":"rQ_J9WH6CGk","title":"Second"}]}"#;
|
||||
for (index, view) in app.views.iter_mut().enumerate() {
|
||||
view.videos = youtube::feed_results(videos).unwrap();
|
||||
view.videos = youtube::feed_results(videos).unwrap().videos;
|
||||
view.loaded = true;
|
||||
view.selection.select(Some(usize::from(index % 2 == 0)));
|
||||
}
|
||||
@@ -1513,7 +1953,6 @@ mod tests {
|
||||
assert_eq!(app.view_index, 0);
|
||||
app.key(KeyEvent::new(KeyCode::BackTab, KeyModifiers::SHIFT));
|
||||
assert_eq!(app.view_index, 1);
|
||||
assert!(app.status.contains("Feed has no channels"));
|
||||
assert!(app.job.is_none());
|
||||
}
|
||||
|
||||
@@ -1542,7 +1981,7 @@ mod tests {
|
||||
app.views[0].videos = youtube::search_results(
|
||||
r#"{"entries":[{"id":"l-GaaivVJCE","title":"Rust video","channel_id":"UCaYhcUwRBNscFNUKTjgPFiA","uploader_id":"@RustVideos"}]}"#,
|
||||
)
|
||||
.unwrap();
|
||||
.unwrap().videos;
|
||||
app.views[0].loaded = true;
|
||||
app.views[0].selection.select(Some(0));
|
||||
app.key(KeyEvent::new(KeyCode::Char('c'), KeyModifiers::NONE));
|
||||
@@ -1625,7 +2064,7 @@ mod tests {
|
||||
|
||||
app.views[0].videos = youtube::search_results(
|
||||
r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Me at the zoo","channel_id":"UC4QobU6STFB0P71PMvOGN5A","uploader_id":"@jawed"}]}"#,
|
||||
).unwrap();
|
||||
).unwrap().videos;
|
||||
app.views[0].videos[0].description = Some("A cached full description.".into());
|
||||
app.views[0].selection.select(Some(0));
|
||||
app.views[0].loaded = true;
|
||||
|
||||
+208
-13
@@ -8,6 +8,14 @@ use std::{
|
||||
|
||||
use serde::Deserialize;
|
||||
|
||||
pub const PAGE_SIZE: usize = 20;
|
||||
|
||||
pub struct Page {
|
||||
pub videos: Vec<Video>,
|
||||
pub has_more: bool,
|
||||
pub cutoff: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct Video {
|
||||
pub id: String,
|
||||
@@ -29,14 +37,19 @@ struct SearchResults {
|
||||
uploader_id: Option<String>,
|
||||
}
|
||||
|
||||
pub fn search_results(output: &str) -> Result<Vec<Video>, String> {
|
||||
pub fn search_results(output: &str) -> Result<Page, String> {
|
||||
let results: SearchResults = serde_json::from_str(output)
|
||||
.map_err(|error| format!("Invalid search response: {error}"))?;
|
||||
let has_more = results.entries.len() >= PAGE_SIZE;
|
||||
let mut videos: Vec<Video> = results.entries.into_iter().flatten().collect();
|
||||
for video in &mut videos {
|
||||
validate_video(video).map_err(|error| format!("Search returned {error}"))?;
|
||||
}
|
||||
Ok(videos)
|
||||
Ok(Page {
|
||||
videos,
|
||||
has_more,
|
||||
cutoff: None,
|
||||
})
|
||||
}
|
||||
|
||||
fn valid_channel_id(id: &str) -> bool {
|
||||
@@ -79,9 +92,12 @@ fn validate_video(video: &mut Video) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn feed_results(output: &str) -> Result<Vec<Video>, String> {
|
||||
pub fn feed_results(output: &str) -> Result<Page, String> {
|
||||
let mut videos = Vec::new();
|
||||
let mut playlists = 0;
|
||||
let mut has_more = false;
|
||||
let mut cutoff = None;
|
||||
let mut unknown_frontier = false;
|
||||
// Multiple yt-dlp URLs produce one playlist object per line, not one combined object.
|
||||
for result in serde_json::Deserializer::from_str(output).into_iter::<SearchResults>() {
|
||||
let playlist = result.map_err(|error| format!("Invalid feed response: {error}"))?;
|
||||
@@ -93,6 +109,19 @@ pub fn feed_results(output: &str) -> Result<Vec<Video>, String> {
|
||||
{
|
||||
return Err("Feed returned an invalid YouTube channel ID".into());
|
||||
}
|
||||
// Raw rows include unavailable videos; filtering them must not end pagination early.
|
||||
if playlist.entries.len() >= PAGE_SIZE {
|
||||
has_more = true;
|
||||
let oldest = playlist
|
||||
.entries
|
||||
.iter()
|
||||
.flatten()
|
||||
.filter_map(|video| video.timestamp)
|
||||
.min();
|
||||
// A full undated/unavailable page may hide newer uploads on its next page.
|
||||
unknown_frontier |= oldest.is_none();
|
||||
cutoff = cutoff.max(oldest);
|
||||
}
|
||||
for mut video in playlist.entries.into_iter().flatten() {
|
||||
if video.channel_id.is_none() {
|
||||
video.channel_id = playlist.channel_id.clone();
|
||||
@@ -114,7 +143,11 @@ pub fn feed_results(output: &str) -> Result<Vec<Video>, String> {
|
||||
videos.sort_unstable_by(|a, b| a.id.cmp(&b.id).then_with(|| b.timestamp.cmp(&a.timestamp)));
|
||||
videos.dedup_by(|a, b| a.id == b.id);
|
||||
videos.sort_by(|a, b| b.timestamp.cmp(&a.timestamp));
|
||||
Ok(videos)
|
||||
Ok(Page {
|
||||
videos,
|
||||
has_more,
|
||||
cutoff: if unknown_frontier { None } else { cutoff },
|
||||
})
|
||||
}
|
||||
|
||||
pub fn full_description(output: &str, expected_id: &str) -> Result<String, String> {
|
||||
@@ -133,6 +166,13 @@ pub fn full_description(output: &str, expected_id: &str) -> Result<String, Strin
|
||||
Ok(description)
|
||||
}
|
||||
|
||||
fn page_end(start: usize) -> io::Result<usize> {
|
||||
start
|
||||
.checked_add(PAGE_SIZE - 1)
|
||||
.filter(|_| start != 0)
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Invalid playlist page offset"))
|
||||
}
|
||||
|
||||
// Own the extractor so replacing a request or quitting also cancels its process.
|
||||
pub struct Job {
|
||||
child: Child,
|
||||
@@ -140,28 +180,37 @@ pub struct Job {
|
||||
}
|
||||
|
||||
impl Job {
|
||||
pub fn search(query: &str) -> io::Result<Self> {
|
||||
pub fn search(query: &str, start: usize) -> io::Result<Self> {
|
||||
let end = page_end(start)?;
|
||||
Self::spawn(extractor().args([
|
||||
"--flat-playlist",
|
||||
"--playlist-start",
|
||||
&start.to_string(),
|
||||
"--playlist-end",
|
||||
&end.to_string(),
|
||||
"--dump-single-json",
|
||||
&format!("ytsearch20:{query}"),
|
||||
&format!("ytsearchall:{query}"),
|
||||
]))
|
||||
}
|
||||
|
||||
pub fn feed(channels: &[String]) -> io::Result<Self> {
|
||||
pub fn feed(channels: &[String], start: usize) -> io::Result<Self> {
|
||||
if channels.is_empty() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
"No channels in this category",
|
||||
));
|
||||
}
|
||||
// ponytail: 20 entries per channel's /videos tab; add pagination if older uploads are needed.
|
||||
let end = page_end(start)?;
|
||||
// ponytail: offsets traverse prior continuations; use direct tokens/per-channel exhaustion
|
||||
// tracking if deep archives become slow.
|
||||
Self::spawn(
|
||||
extractor()
|
||||
.args([
|
||||
"--flat-playlist",
|
||||
"--playlist-start",
|
||||
&start.to_string(),
|
||||
"--playlist-end",
|
||||
"20",
|
||||
&end.to_string(),
|
||||
"--extractor-args",
|
||||
"youtubetab:approximate_date",
|
||||
"--dump-single-json",
|
||||
@@ -499,7 +548,8 @@ mod tests {
|
||||
let videos = search_results(
|
||||
r#"{"entries":[null,{"id":"rQ_J9WH6CGk","title":"Rust\u001b","channel":null,"uploader_id":"@RustVideos"}]}"#,
|
||||
)
|
||||
.unwrap();
|
||||
.unwrap()
|
||||
.videos;
|
||||
assert_eq!(videos[0].title, "Rust");
|
||||
assert_eq!(videos[0].channel, None);
|
||||
assert_eq!(videos[0].channel_id, None);
|
||||
@@ -521,7 +571,10 @@ mod tests {
|
||||
fn feeds_merge_newest_first_deduplicate_and_inherit_channel_identity() {
|
||||
let output = r#"{"channel_id":"UCaYhcUwRBNscFNUKTjgPFiA","uploader_id":"@RustVideos","channel":"Rust\u001b","entries":[null,{"id":"l-GaaivVJCE","title":"Older","timestamp":100},{"id":"jNQXAC9IVRw","title":"Unknown","timestamp":null}]}
|
||||
{"channel_id":"UCaYhcUwRBNscFNUKTjgPFiA","uploader_id":"@RustVideos","channel":"Rust","entries":[{"id":"SLV9AJxnb1U","title":"Newest","timestamp":300},{"id":"l-GaaivVJCE","title":"Newer","timestamp":200},{"id":"rQ_J9WH6CGk","title":"No metadata","channel_id":null}]}"#;
|
||||
let videos = feed_results(output).unwrap();
|
||||
let page = feed_results(output).unwrap();
|
||||
assert!(!page.has_more);
|
||||
assert_eq!(page.cutoff, None);
|
||||
let videos = page.videos;
|
||||
assert_eq!(
|
||||
videos
|
||||
.iter()
|
||||
@@ -546,7 +599,8 @@ mod tests {
|
||||
let unavailable = feed_results(
|
||||
r#"{"entries":[null,{"id":"jNQXAC9IVRw","title":"Unknown","channel_id":null}]}"#,
|
||||
)
|
||||
.unwrap();
|
||||
.unwrap()
|
||||
.videos;
|
||||
assert_eq!(unavailable[0].channel_id, None);
|
||||
assert!(feed_results(r#"{"channel_id":"malformed","entries":[]}"#).is_err());
|
||||
assert!(
|
||||
@@ -559,7 +613,148 @@ mod tests {
|
||||
assert!(feed_results("").is_err());
|
||||
assert!(feed_results("{}").is_err());
|
||||
assert!(feed_results("{not JSON}").is_err());
|
||||
assert!(feed_results(r#"{"entries":[]}"#).unwrap().is_empty());
|
||||
let exhausted = feed_results(r#"{"entries":[]}"#).unwrap();
|
||||
assert!(exhausted.videos.is_empty());
|
||||
assert!(!exhausted.has_more);
|
||||
assert_eq!(exhausted.cutoff, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_offsets_reject_zero_and_overflow_before_spawning() {
|
||||
assert_eq!(page_end(1).unwrap(), PAGE_SIZE);
|
||||
assert_eq!(page_end(PAGE_SIZE + 1).unwrap(), PAGE_SIZE * 2);
|
||||
assert_eq!(page_end(usize::MAX - (PAGE_SIZE - 1)).unwrap(), usize::MAX);
|
||||
let channels = vec!["https://www.youtube.com/@RustVideos".to_owned()];
|
||||
for start in [0, usize::MAX, usize::MAX - (PAGE_SIZE - 2)] {
|
||||
assert!(matches!(
|
||||
Job::search("rust", start),
|
||||
Err(error) if error.kind() == io::ErrorKind::InvalidInput
|
||||
));
|
||||
assert!(matches!(
|
||||
Job::feed(&channels, start),
|
||||
Err(error) if error.kind() == io::ErrorKind::InvalidInput
|
||||
));
|
||||
}
|
||||
assert!(matches!(
|
||||
Job::feed(&[], 1),
|
||||
Err(error) if error.kind() == io::ErrorKind::InvalidInput
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn search_pagination_counts_raw_rows_and_preserves_relevance_order() {
|
||||
for count in [0, PAGE_SIZE - 1, PAGE_SIZE, PAGE_SIZE + 1] {
|
||||
let output = serde_json::json!({
|
||||
"entries": vec![serde_json::Value::Null; count]
|
||||
})
|
||||
.to_string();
|
||||
let page = search_results(&output).unwrap();
|
||||
assert_eq!(page.has_more, count >= PAGE_SIZE);
|
||||
assert!(page.videos.is_empty());
|
||||
assert_eq!(page.cutoff, None);
|
||||
}
|
||||
let mut entries = vec![serde_json::Value::Null; PAGE_SIZE];
|
||||
entries[0] = serde_json::json!({
|
||||
"id": "aaaaaaaaaaa", "title": "Relevant older video", "timestamp": 100
|
||||
});
|
||||
entries[PAGE_SIZE - 1] = serde_json::json!({
|
||||
"id": "bbbbbbbbbbb", "title": "Less relevant newer video", "timestamp": 300
|
||||
});
|
||||
let page = search_results(&serde_json::json!({"entries": entries}).to_string()).unwrap();
|
||||
assert!(page.has_more);
|
||||
assert_eq!(page.cutoff, None);
|
||||
assert_eq!(page.videos.len(), 2);
|
||||
assert_eq!(page.videos[0].id, "aaaaaaaaaaa");
|
||||
assert_eq!(page.videos[1].id, "bbbbbbbbbbb");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn feed_pagination_counts_nulls_and_duplicates_before_filtering() {
|
||||
for count in [0, PAGE_SIZE - 1, PAGE_SIZE, PAGE_SIZE + 1] {
|
||||
let output = serde_json::json!({
|
||||
"entries": vec![serde_json::Value::Null; count]
|
||||
})
|
||||
.to_string();
|
||||
let page = feed_results(&output).unwrap();
|
||||
assert_eq!(page.has_more, count >= PAGE_SIZE);
|
||||
assert!(page.videos.is_empty());
|
||||
assert_eq!(page.cutoff, None);
|
||||
}
|
||||
let entry = serde_json::json!({
|
||||
"id": "aaaaaaaaaaa", "title": "Repeated", "timestamp": 123
|
||||
});
|
||||
let page =
|
||||
feed_results(&serde_json::json!({"entries": vec![entry; PAGE_SIZE]}).to_string())
|
||||
.unwrap();
|
||||
assert!(page.has_more);
|
||||
assert_eq!(page.cutoff, Some(123));
|
||||
assert_eq!(page.videos.len(), 1);
|
||||
|
||||
let mut entries = vec![serde_json::Value::Null; PAGE_SIZE];
|
||||
entries[0] = serde_json::json!({"id": "bbbbbbbbbbb", "title": "Unknown"});
|
||||
let page = feed_results(&serde_json::json!({"entries": entries}).to_string()).unwrap();
|
||||
assert!(page.has_more);
|
||||
assert_eq!(page.cutoff, None);
|
||||
assert_eq!(page.videos.len(), 1);
|
||||
assert_eq!(page.videos[0].timestamp, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn feed_frontier_uses_only_full_channel_minima_and_retains_all_rows() {
|
||||
let mut first = vec![serde_json::Value::Null; PAGE_SIZE];
|
||||
first[0] =
|
||||
serde_json::json!({"id":"aaaaaaaaaaa","title":"Older duplicate","timestamp":500});
|
||||
first[1] = serde_json::json!({"id":"bbbbbbbbbbb","title":"First frontier","timestamp":300});
|
||||
first[2] = serde_json::json!({"id":"ccccccccccc","title":"Unknown"});
|
||||
let mut second = vec![serde_json::Value::Null; PAGE_SIZE];
|
||||
second[0] =
|
||||
serde_json::json!({"id":"aaaaaaaaaaa","title":"Newest duplicate","timestamp":600});
|
||||
second[1] =
|
||||
serde_json::json!({"id":"ddddddddddd","title":"Second frontier","timestamp":400});
|
||||
let short = serde_json::json!({"entries":[
|
||||
{"id":"eeeeeeeeeee","title":"Short newer","timestamp":999}
|
||||
]});
|
||||
let older_short = serde_json::json!({"entries":[
|
||||
{"id":"fffffffffff","title":"Short older","timestamp":50}
|
||||
]});
|
||||
let output = format!(
|
||||
"{}\n{}\n{}\n{}\n{}",
|
||||
serde_json::json!({"entries": first}),
|
||||
serde_json::json!({"entries": second}),
|
||||
short,
|
||||
older_short,
|
||||
serde_json::json!({"entries":[]})
|
||||
);
|
||||
let page = feed_results(&output).unwrap();
|
||||
assert!(page.has_more);
|
||||
assert_eq!(page.cutoff, Some(400));
|
||||
assert_eq!(
|
||||
page.videos
|
||||
.iter()
|
||||
.map(|video| video.id.as_str())
|
||||
.collect::<Vec<_>>(),
|
||||
[
|
||||
"eeeeeeeeeee",
|
||||
"aaaaaaaaaaa",
|
||||
"ddddddddddd",
|
||||
"bbbbbbbbbbb",
|
||||
"fffffffffff",
|
||||
"ccccccccccc"
|
||||
]
|
||||
);
|
||||
assert_eq!(page.videos[1].title, "Newest duplicate");
|
||||
assert_eq!(page.videos.last().unwrap().timestamp, None);
|
||||
|
||||
let exhausted = feed_results(&format!(
|
||||
"{}\n{}\n{}",
|
||||
short,
|
||||
older_short,
|
||||
serde_json::json!({"entries":[]})
|
||||
))
|
||||
.unwrap();
|
||||
assert!(!exhausted.has_more);
|
||||
assert_eq!(exhausted.cutoff, None);
|
||||
assert_eq!(exhausted.videos.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user