From 2f11bdd40afea961aa81e444ac148f86934691da Mon Sep 17 00:00:00 2001
From: hl-valdemar <valdemar.lorenzen@icloud.com>
Date: Sun, 11 Oct 2026 19:49:11 +0200
Subject: [PATCH] fix: discard malformed video entries without failing pages

---
 Cargo.toml     |   2 +-
 README.md      |   2 +
 src/main.rs    |   2 +
 src/youtube.rs | 141 +++++++++++++++++++++++++++++++++++--------------
 4 files changed, 106 insertions(+), 41 deletions(-)

diff --git a/Cargo.toml b/Cargo.toml
index 0c05c5a..64d95ed 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -10,7 +10,7 @@ image = { version = "0.25.10", default-features = false, features = ["jpeg"] }
 ratatui = { version = "0.30.2", default-features = false, features = ["crossterm", "unstable-rendered-line-info"] }
 ratatui-image = { version = "11.1.0", default-features = false, features = ["crossterm"] }
 serde = { version = "1.0.229", features = ["derive"] }
-serde_json = "1.0.151"
+serde_json = { version = "1.0.151", features = ["raw_value"] }
 toml_edit = "0.25.17"
 time = "0.3.55"
 
diff --git a/README.md b/README.md
index 3f61348..16dadee 100644
--- a/README.md
+++ b/README.md
@@ -191,6 +191,8 @@ Using the same binding for overview and description preserves the default cycle.
 
 Search loads 20 results per page; Feed and category views request 20 uploads per channel per page. Individual channel views request only that channel's uploads, using the same pagination. Moving near the loaded end in normal mode fetches more asynchronously, preserving selection and existing rows. Insert mode and open popups pause automatic prefetch. Deep channel archives can get slower because yt-dlp offsets traverse earlier pages again.
 
+Malformed individual entries—including invalid video IDs, missing required fields, and wrong field types—are discarded without losing valid videos from the same page. Skipped entries still count toward pagination, so a full page of bad or unavailable entries does not hide later results. Invalid JSON, malformed playlist-level metadata, and extractor failures still fail the page.
+
 Feed views merge newest-first using available upload dates. Older uploads wait while newer channel pages catch up; undated uploads appear when the feed is exhausted. Publication labels use `YYYY-MM-DD`; `~` marks an estimated UTC date, and `Unknown` means unavailable metadata.
 
 Pane titles show animated spinners while content loads. Errors retain loaded rows and stop automatic retries; `r` refreshes the current view.
diff --git a/src/main.rs b/src/main.rs
index e696de7..dfa9a32 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -73,6 +73,8 @@
 //! Feed includes saved `[channels]` entries, even untagged, plus direct tag entries, without duplicates.
 //! Untagging is not unsubscribing: remove mappings and tag references from the file to unsubscribe.
 //! Search pages contain 20 results; Feed/tag pages request 20 uploads per channel's /videos tab.
+//! Invalid individual video entries are discarded; raw entry counts still determine pagination.
+//! Invalid JSON, playlist-level metadata, and extractor failures remain page errors.
 //! In normal mode, approaching the loaded end fetches the next page without replacing rows/selection.
 //! Loading result/preview panes animate a spinner after their title, including additional pages.
 //! Info spinners cover image download/decode/resize and description fetching; errors stop them.
diff --git a/src/youtube.rs b/src/youtube.rs
index 6580e5d..8831d47 100644
--- a/src/youtube.rs
+++ b/src/youtube.rs
@@ -30,8 +30,10 @@ pub struct Video {
 }
 
 #[derive(Deserialize)]
-struct SearchResults {
-    entries: Vec<Option<Video>>,
+struct SearchResults<'a> {
+    // Borrow raw rows so one schema error cannot abort siblings or copy unused metadata.
+    #[serde(borrow)]
+    entries: Vec<&'a serde_json::value::RawValue>,
     channel: Option<String>,
     channel_id: Option<String>,
     uploader_id: Option<String>,
@@ -41,10 +43,15 @@ pub fn search_results(output: &str) -> Result<Page, String> {
     let results: SearchResults = serde_json::from_str(output)
         .map_err(|error| format!("Invalid search response: {error}"))?;
     let has_more = results.entries.len() >= PAGE_SIZE;
-    let mut videos: Vec<Video> = results.entries.into_iter().flatten().collect();
-    for video in &mut videos {
-        validate_video(video).map_err(|error| format!("Search returned {error}"))?;
-    }
+    let videos = results
+        .entries
+        .into_iter()
+        .filter_map(|entry| {
+            let mut video = serde_json::from_str(entry.get()).ok()?;
+            validate_video(&mut video).ok()?;
+            Some(video)
+        })
+        .collect();
     Ok(Page {
         videos,
         has_more,
@@ -128,19 +135,12 @@ pub fn feed_results(output: &str) -> Result<Page, String> {
             return Err("Feed returned an invalid YouTube channel ID".into());
         }
         // Raw rows include unavailable videos; filtering them must not end pagination early.
-        if playlist.entries.len() >= PAGE_SIZE {
-            has_more = true;
-            let oldest = playlist
-                .entries
-                .iter()
-                .flatten()
-                .filter_map(|video| video.timestamp)
-                .min();
-            // A full undated/unavailable page may hide newer uploads on its next page.
-            unknown_frontier |= oldest.is_none();
-            cutoff = cutoff.max(oldest);
-        }
-        for mut video in playlist.entries.into_iter().flatten() {
+        let full_page = playlist.entries.len() >= PAGE_SIZE;
+        let first_video = videos.len();
+        for entry in playlist.entries {
+            let Ok(mut video) = serde_json::from_str::<Video>(entry.get()) else {
+                continue;
+            };
             if video.channel_id.is_none() {
                 video.channel_id = playlist.channel_id.clone();
             }
@@ -150,8 +150,19 @@ pub fn feed_results(output: &str) -> Result<Page, String> {
             if video.channel.is_none() {
                 video.channel = playlist.channel.clone();
             }
-            validate_video(&mut video).map_err(|error| format!("Feed returned {error}"))?;
-            videos.push(video);
+            if validate_video(&mut video).is_ok() {
+                videos.push(video);
+            }
+        }
+        if full_page {
+            has_more = true;
+            let oldest = videos[first_video..]
+                .iter()
+                .filter_map(|video| video.timestamp)
+                .min();
+            // A full undated/unavailable page may hide newer uploads on its next page.
+            unknown_frontier |= oldest.is_none();
+            cutoff = cutoff.max(oldest);
         }
     }
     if playlists == 0 {
@@ -561,7 +572,66 @@ mod tests {
     }
 
     #[test]
-    fn search_handles_unavailable_entries_and_rejects_invalid_ids() {
+    fn bad_video_entries_do_not_discard_valid_results() {
+        for parse in [search_results, feed_results] {
+            for bad in [
+                r#"{"id":"UC4QobU6STFB0P71PMvOGN5A","title":"Channel result"}"#,
+                r#"{"id":"--bad","title":"Unsafe ID"}"#,
+                r#"{"id":"jNQXAC9IVRw","title":"Bad channel","channel_id":"malformed"}"#,
+                r#"{"id":"jNQXAC9IVRw","title":"Bad handle","uploader_id":"@jawed\u001b"}"#,
+                r#"{"id":42,"title":"Wrong ID type"}"#,
+                r#"{"id":"jNQXAC9IVRw","title":null}"#,
+                r#"{"id":"jNQXAC9IVRw","title":"Bad date type","timestamp":"yesterday"}"#,
+                r#"{"id":"jNQXAC9IVRw","title":"Bad channel type","channel":[]}"#,
+                r#"{"id":"jNQXAC9IVRw"}"#,
+                "null",
+                "42",
+                "[]",
+                "{}",
+            ] {
+                let output = format!(
+                    r#"{{"entries":[{{"id":"jNQXAC9IVRw","title":"Before"}},{bad},{{"id":"rQ_J9WH6CGk","title":"After"}}]}}"#,
+                );
+                let page = parse(&output).unwrap_or_else(|error| {
+                    panic!("bad entry {bad} discarded the entire page: {error}")
+                });
+                assert_eq!(
+                    page.videos
+                        .iter()
+                        .map(|video| video.title.as_str())
+                        .collect::<Vec<_>>(),
+                    ["Before", "After"]
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn discarded_entries_still_count_for_pagination_but_not_feed_frontiers() {
+        let bad = serde_json::json!({"id":"invalid", "title":"Bad", "timestamp":-500});
+        let mut entries = vec![bad; PAGE_SIZE];
+        for parse in [search_results, feed_results] {
+            let page = parse(&serde_json::json!({"entries":entries}).to_string()).unwrap();
+            assert!(page.videos.is_empty());
+            assert!(
+                page.has_more,
+                "an all-invalid full page must not stop pagination"
+            );
+            assert_eq!(page.cutoff, None);
+        }
+        entries[0] = serde_json::json!({"id":"jNQXAC9IVRw","title":"Valid","timestamp":100});
+        let page = feed_results(&serde_json::json!({"entries":entries}).to_string()).unwrap();
+        assert!(page.has_more);
+        assert_eq!(
+            page.cutoff,
+            Some(100),
+            "invalid dates must not release old feed rows"
+        );
+        assert_eq!(page.videos[0].id, "jNQXAC9IVRw");
+    }
+
+    #[test]
+    fn search_sanitizes_display_metadata_and_rejects_invalid_envelopes() {
         let videos = search_results(
             r#"{"entries":[null,{"id":"rQ_J9WH6CGk","title":"Rust\u001b","channel":null,"uploader_id":"@RustVideos"}]}"#,
         )
@@ -571,17 +641,15 @@ mod tests {
         assert_eq!(videos[0].channel, None);
         assert_eq!(videos[0].channel_id, None);
         assert_eq!(videos[0].uploader_id.as_deref(), Some("@RustVideos"));
-        assert!(
-            search_results(
-                r#"{"entries":[{"id":"rQ_J9WH6CGk","title":"Bad","channel_id":"not-a-channel"}]}"#
-            )
-            .is_err()
-        );
-        assert!(search_results(
-            r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Unsafe","uploader_id":"@jawed\u001b"}]}"#
-        ).is_err());
-        assert!(search_results(r#"{"entries":[{"id":"--bad","title":"Bad"}]}"#).is_err());
-        assert!(search_results("not JSON").is_err());
+        for output in [
+            "not JSON",
+            "{}",
+            r#"{"entries":null}"#,
+            r#"{"entries":{}}"#,
+            r#"{"entries":[{"id":} ]}"#,
+        ] {
+            assert!(search_results(output).is_err());
+        }
     }
 
     #[test]
@@ -620,13 +688,6 @@ mod tests {
         .videos;
         assert_eq!(unavailable[0].channel_id, None);
         assert!(feed_results(r#"{"channel_id":"malformed","entries":[]}"#).is_err());
-        assert!(
-            feed_results(
-                r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Bad","channel_id":"malformed"}]}"#
-            )
-            .is_err()
-        );
-        assert!(feed_results(r#"{"entries":[{"id":"bad","title":"Bad"}]}"#).is_err());
         assert!(feed_results("").is_err());
         assert!(feed_results("{}").is_err());
         assert!(feed_results("{not JSON}").is_err());