feat: paginate search and subscription feeds while scrolling

src/youtube.rs

@@ -8,6 +8,14 @@ use std::{
 
 use serde::Deserialize;
 
+pub const PAGE_SIZE: usize = 20;
+
+pub struct Page {
+    pub videos: Vec<Video>,
+    pub has_more: bool,
+    pub cutoff: Option<i64>,
+}
+
 #[derive(Deserialize)]
 pub struct Video {
     pub id: String,
@@ -29,14 +37,19 @@ struct SearchResults {
     uploader_id: Option<String>,
 }
 
-pub fn search_results(output: &str) -> Result<Vec<Video>, String> {
+pub fn search_results(output: &str) -> Result<Page, String> {
     let results: SearchResults = serde_json::from_str(output)
         .map_err(|error| format!("Invalid search response: {error}"))?;
+    let has_more = results.entries.len() >= PAGE_SIZE;
     let mut videos: Vec<Video> = results.entries.into_iter().flatten().collect();
     for video in &mut videos {
         validate_video(video).map_err(|error| format!("Search returned {error}"))?;
     }
-    Ok(videos)
+    Ok(Page {
+        videos,
+        has_more,
+        cutoff: None,
+    })
 }
 
 fn valid_channel_id(id: &str) -> bool {
@@ -79,9 +92,12 @@ fn validate_video(video: &mut Video) -> Result<(), String> {
     Ok(())
 }
 
-pub fn feed_results(output: &str) -> Result<Vec<Video>, String> {
+pub fn feed_results(output: &str) -> Result<Page, String> {
     let mut videos = Vec::new();
     let mut playlists = 0;
+    let mut has_more = false;
+    let mut cutoff = None;
+    let mut unknown_frontier = false;
     // Multiple yt-dlp URLs produce one playlist object per line, not one combined object.
     for result in serde_json::Deserializer::from_str(output).into_iter::<SearchResults>() {
         let playlist = result.map_err(|error| format!("Invalid feed response: {error}"))?;
@@ -93,6 +109,19 @@ pub fn feed_results(output: &str) -> Result<Vec<Video>, String> {
         {
             return Err("Feed returned an invalid YouTube channel ID".into());
         }
+        // Raw rows include unavailable videos; filtering them must not end pagination early.
+        if playlist.entries.len() >= PAGE_SIZE {
+            has_more = true;
+            let oldest = playlist
+                .entries
+                .iter()
+                .flatten()
+                .filter_map(|video| video.timestamp)
+                .min();
+            // A full undated/unavailable page may hide newer uploads on its next page.
+            unknown_frontier |= oldest.is_none();
+            cutoff = cutoff.max(oldest);
+        }
         for mut video in playlist.entries.into_iter().flatten() {
             if video.channel_id.is_none() {
                 video.channel_id = playlist.channel_id.clone();
@@ -114,7 +143,11 @@ pub fn feed_results(output: &str) -> Result<Vec<Video>, String> {
     videos.sort_unstable_by(|a, b| a.id.cmp(&b.id).then_with(|| b.timestamp.cmp(&a.timestamp)));
     videos.dedup_by(|a, b| a.id == b.id);
     videos.sort_by(|a, b| b.timestamp.cmp(&a.timestamp));
-    Ok(videos)
+    Ok(Page {
+        videos,
+        has_more,
+        cutoff: if unknown_frontier { None } else { cutoff },
+    })
 }
 
 pub fn full_description(output: &str, expected_id: &str) -> Result<String, String> {
@@ -133,6 +166,13 @@ pub fn full_description(output: &str, expected_id: &str) -> Result<String, Strin
     Ok(description)
 }
 
+fn page_end(start: usize) -> io::Result<usize> {
+    start
+        .checked_add(PAGE_SIZE - 1)
+        .filter(|_| start != 0)
+        .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Invalid playlist page offset"))
+}
+
 // Own the extractor so replacing a request or quitting also cancels its process.
 pub struct Job {
     child: Child,
@@ -140,28 +180,37 @@ pub struct Job {
 }
 
 impl Job {
-    pub fn search(query: &str) -> io::Result<Self> {
+    pub fn search(query: &str, start: usize) -> io::Result<Self> {
+        let end = page_end(start)?;
         Self::spawn(extractor().args([
             "--flat-playlist",
+            "--playlist-start",
+            &start.to_string(),
+            "--playlist-end",
+            &end.to_string(),
             "--dump-single-json",
-            &format!("ytsearch20:{query}"),
+            &format!("ytsearchall:{query}"),
         ]))
     }
 
-    pub fn feed(channels: &[String]) -> io::Result<Self> {
+    pub fn feed(channels: &[String], start: usize) -> io::Result<Self> {
         if channels.is_empty() {
             return Err(io::Error::new(
                 io::ErrorKind::InvalidInput,
                 "No channels in this category",
             ));
         }
-        // ponytail: 20 entries per channel's /videos tab; add pagination if older uploads are needed.
+        let end = page_end(start)?;
+        // ponytail: offsets traverse prior continuations; use direct tokens/per-channel exhaustion
+        // tracking if deep archives become slow.
         Self::spawn(
             extractor()
                 .args([
                     "--flat-playlist",
+                    "--playlist-start",
+                    &start.to_string(),
                     "--playlist-end",
-                    "20",
+                    &end.to_string(),
                     "--extractor-args",
                     "youtubetab:approximate_date",
                     "--dump-single-json",
@@ -499,7 +548,8 @@ mod tests {
         let videos = search_results(
             r#"{"entries":[null,{"id":"rQ_J9WH6CGk","title":"Rust\u001b","channel":null,"uploader_id":"@RustVideos"}]}"#,
         )
-        .unwrap();
+        .unwrap()
+        .videos;
         assert_eq!(videos[0].title, "Rust");
         assert_eq!(videos[0].channel, None);
         assert_eq!(videos[0].channel_id, None);
@@ -521,7 +571,10 @@ mod tests {
     fn feeds_merge_newest_first_deduplicate_and_inherit_channel_identity() {
         let output = r#"{"channel_id":"UCaYhcUwRBNscFNUKTjgPFiA","uploader_id":"@RustVideos","channel":"Rust\u001b","entries":[null,{"id":"l-GaaivVJCE","title":"Older","timestamp":100},{"id":"jNQXAC9IVRw","title":"Unknown","timestamp":null}]}
 {"channel_id":"UCaYhcUwRBNscFNUKTjgPFiA","uploader_id":"@RustVideos","channel":"Rust","entries":[{"id":"SLV9AJxnb1U","title":"Newest","timestamp":300},{"id":"l-GaaivVJCE","title":"Newer","timestamp":200},{"id":"rQ_J9WH6CGk","title":"No metadata","channel_id":null}]}"#;
-        let videos = feed_results(output).unwrap();
+        let page = feed_results(output).unwrap();
+        assert!(!page.has_more);
+        assert_eq!(page.cutoff, None);
+        let videos = page.videos;
         assert_eq!(
             videos
                 .iter()
@@ -546,7 +599,8 @@ mod tests {
         let unavailable = feed_results(
             r#"{"entries":[null,{"id":"jNQXAC9IVRw","title":"Unknown","channel_id":null}]}"#,
         )
-        .unwrap();
+        .unwrap()
+        .videos;
         assert_eq!(unavailable[0].channel_id, None);
         assert!(feed_results(r#"{"channel_id":"malformed","entries":[]}"#).is_err());
         assert!(
@@ -559,7 +613,148 @@ mod tests {
         assert!(feed_results("").is_err());
         assert!(feed_results("{}").is_err());
         assert!(feed_results("{not JSON}").is_err());
-        assert!(feed_results(r#"{"entries":[]}"#).unwrap().is_empty());
+        let exhausted = feed_results(r#"{"entries":[]}"#).unwrap();
+        assert!(exhausted.videos.is_empty());
+        assert!(!exhausted.has_more);
+        assert_eq!(exhausted.cutoff, None);
+    }
+
+    #[test]
+    fn page_offsets_reject_zero_and_overflow_before_spawning() {
+        assert_eq!(page_end(1).unwrap(), PAGE_SIZE);
+        assert_eq!(page_end(PAGE_SIZE + 1).unwrap(), PAGE_SIZE * 2);
+        assert_eq!(page_end(usize::MAX - (PAGE_SIZE - 1)).unwrap(), usize::MAX);
+        let channels = vec!["https://www.youtube.com/@RustVideos".to_owned()];
+        for start in [0, usize::MAX, usize::MAX - (PAGE_SIZE - 2)] {
+            assert!(matches!(
+                Job::search("rust", start),
+                Err(error) if error.kind() == io::ErrorKind::InvalidInput
+            ));
+            assert!(matches!(
+                Job::feed(&channels, start),
+                Err(error) if error.kind() == io::ErrorKind::InvalidInput
+            ));
+        }
+        assert!(matches!(
+            Job::feed(&[], 1),
+            Err(error) if error.kind() == io::ErrorKind::InvalidInput
+        ));
+    }
+
+    #[test]
+    fn search_pagination_counts_raw_rows_and_preserves_relevance_order() {
+        for count in [0, PAGE_SIZE - 1, PAGE_SIZE, PAGE_SIZE + 1] {
+            let output = serde_json::json!({
+                "entries": vec![serde_json::Value::Null; count]
+            })
+            .to_string();
+            let page = search_results(&output).unwrap();
+            assert_eq!(page.has_more, count >= PAGE_SIZE);
+            assert!(page.videos.is_empty());
+            assert_eq!(page.cutoff, None);
+        }
+        let mut entries = vec![serde_json::Value::Null; PAGE_SIZE];
+        entries[0] = serde_json::json!({
+            "id": "aaaaaaaaaaa", "title": "Relevant older video", "timestamp": 100
+        });
+        entries[PAGE_SIZE - 1] = serde_json::json!({
+            "id": "bbbbbbbbbbb", "title": "Less relevant newer video", "timestamp": 300
+        });
+        let page = search_results(&serde_json::json!({"entries": entries}).to_string()).unwrap();
+        assert!(page.has_more);
+        assert_eq!(page.cutoff, None);
+        assert_eq!(page.videos.len(), 2);
+        assert_eq!(page.videos[0].id, "aaaaaaaaaaa");
+        assert_eq!(page.videos[1].id, "bbbbbbbbbbb");
+    }
+
+    #[test]
+    fn feed_pagination_counts_nulls_and_duplicates_before_filtering() {
+        for count in [0, PAGE_SIZE - 1, PAGE_SIZE, PAGE_SIZE + 1] {
+            let output = serde_json::json!({
+                "entries": vec![serde_json::Value::Null; count]
+            })
+            .to_string();
+            let page = feed_results(&output).unwrap();
+            assert_eq!(page.has_more, count >= PAGE_SIZE);
+            assert!(page.videos.is_empty());
+            assert_eq!(page.cutoff, None);
+        }
+        let entry = serde_json::json!({
+            "id": "aaaaaaaaaaa", "title": "Repeated", "timestamp": 123
+        });
+        let page =
+            feed_results(&serde_json::json!({"entries": vec![entry; PAGE_SIZE]}).to_string())
+                .unwrap();
+        assert!(page.has_more);
+        assert_eq!(page.cutoff, Some(123));
+        assert_eq!(page.videos.len(), 1);
+
+        let mut entries = vec![serde_json::Value::Null; PAGE_SIZE];
+        entries[0] = serde_json::json!({"id": "bbbbbbbbbbb", "title": "Unknown"});
+        let page = feed_results(&serde_json::json!({"entries": entries}).to_string()).unwrap();
+        assert!(page.has_more);
+        assert_eq!(page.cutoff, None);
+        assert_eq!(page.videos.len(), 1);
+        assert_eq!(page.videos[0].timestamp, None);
+    }
+
+    #[test]
+    fn feed_frontier_uses_only_full_channel_minima_and_retains_all_rows() {
+        let mut first = vec![serde_json::Value::Null; PAGE_SIZE];
+        first[0] =
+            serde_json::json!({"id":"aaaaaaaaaaa","title":"Older duplicate","timestamp":500});
+        first[1] = serde_json::json!({"id":"bbbbbbbbbbb","title":"First frontier","timestamp":300});
+        first[2] = serde_json::json!({"id":"ccccccccccc","title":"Unknown"});
+        let mut second = vec![serde_json::Value::Null; PAGE_SIZE];
+        second[0] =
+            serde_json::json!({"id":"aaaaaaaaaaa","title":"Newest duplicate","timestamp":600});
+        second[1] =
+            serde_json::json!({"id":"ddddddddddd","title":"Second frontier","timestamp":400});
+        let short = serde_json::json!({"entries":[
+            {"id":"eeeeeeeeeee","title":"Short newer","timestamp":999}
+        ]});
+        let older_short = serde_json::json!({"entries":[
+            {"id":"fffffffffff","title":"Short older","timestamp":50}
+        ]});
+        let output = format!(
+            "{}\n{}\n{}\n{}\n{}",
+            serde_json::json!({"entries": first}),
+            serde_json::json!({"entries": second}),
+            short,
+            older_short,
+            serde_json::json!({"entries":[]})
+        );
+        let page = feed_results(&output).unwrap();
+        assert!(page.has_more);
+        assert_eq!(page.cutoff, Some(400));
+        assert_eq!(
+            page.videos
+                .iter()
+                .map(|video| video.id.as_str())
+                .collect::<Vec<_>>(),
+            [
+                "eeeeeeeeeee",
+                "aaaaaaaaaaa",
+                "ddddddddddd",
+                "bbbbbbbbbbb",
+                "fffffffffff",
+                "ccccccccccc"
+            ]
+        );
+        assert_eq!(page.videos[1].title, "Newest duplicate");
+        assert_eq!(page.videos.last().unwrap().timestamp, None);
+
+        let exhausted = feed_results(&format!(
+            "{}\n{}\n{}",
+            short,
+            older_short,
+            serde_json::json!({"entries":[]})
+        ))
+        .unwrap();
+        assert!(!exhausted.has_more);
+        assert_eq!(exhausted.cutoff, None);
+        assert_eq!(exhausted.videos.len(), 2);
     }
 
     #[test]