perf: batch channel fetching and prefetch earlier

src/main.rs

@@ -79,10 +79,11 @@
 //! r reloads config and refreshes the current view.
 //! Feed includes saved `[channels]` entries, even untagged, plus direct tag entries, without duplicates.
 //! Untagging is not unsubscribing: remove mappings and tag references from the file to unsubscribe.
-//! Search pages contain 20 results; Feed/tag pages request 20 uploads per channel's /videos tab.
+//! Search pages contain 60 results; Feed/tag pages request 60 uploads per channel's /videos tab.
+//! Search uses one extractor; feeds batch disjoint channels across up to four concurrent extractors.
 //! Invalid individual video entries are discarded; raw entry counts still determine pagination.
 //! Invalid JSON, playlist-level metadata, and extractor failures remain page errors.
-//! In normal mode, approaching the loaded end fetches the next page without replacing rows/selection.
+//! In unobstructed normal mode, 20 rows before the loaded end fetches the next page without replacing rows/selection.
 //! Loading result/preview panes animate a spinner after their title, including additional pages.
 //! Info spinners cover image download/decode/resize and description fetching; errors stop them.
 //! Feed rows use approximate upload dates; older rows wait until newer channel pages catch up.
@@ -397,7 +398,7 @@ impl View {
                 || self
                     .selection
                     .selected()
-                    .is_some_and(|index| index.saturating_add(5) >= self.videos.len()))
+                    .is_some_and(|index| index.saturating_add(PAGE_SIZE / 3) >= self.videos.len()))
     }
 }
 
@@ -7447,13 +7448,15 @@ tags.cancel = "z"
         );
         assert_eq!(view.videos[1].title, "Second");
         assert_eq!(view.selection.selected(), Some(1));
-        assert_eq!(view.next_start, 41);
+        assert_eq!(view.next_start, PAGE_SIZE * 2 + 1);
         assert!(view.has_more);
     }
 
     #[test]
     fn feed_pages_hold_short_channel_tails_until_newer_long_histories_finish() {
         let mut view = View::new("Feed".into(), ViewSource::Feed);
+        let second_base = 101 + PAGE_SIZE as i64;
+        let first_base = second_base + PAGE_SIZE as i64;
         let long_page = |base: i64| {
             serde_json::json!({
                 "entries": (0..PAGE_SIZE).map(|index| serde_json::json!({
@@ -7465,22 +7468,25 @@ tags.cancel = "z"
             .to_string()
         };
         let short = r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Short old tail","timestamp":100},{"id":"rQ_J9WH6CGk","title":"Unknown date"}]}"#;
-        let first = youtube::feed_results(&format!("{}\n{short}", long_page(200))).unwrap();
+        let first = youtube::feed_results(&format!("{}\n{short}", long_page(first_base))).unwrap();
         view.accept_page(first, true).unwrap();
         assert_eq!(view.videos.len(), PAGE_SIZE);
-        assert_eq!(view.videos.last().unwrap().timestamp, Some(200));
+        assert_eq!(view.videos.last().unwrap().timestamp, Some(first_base));
         assert_eq!(view.pending.len(), 2);
         view.selection.select(Some(PAGE_SIZE - 1));
-        let second = youtube::feed_results(&format!("{}\n{short}", long_page(180))).unwrap();
+        let second =
+            youtube::feed_results(&format!("{}\n{short}", long_page(second_base))).unwrap();
         view.accept_page(second, true).unwrap();
         assert_eq!(view.videos.len(), PAGE_SIZE * 2);
-        assert_eq!(view.videos.last().unwrap().timestamp, Some(180));
+        assert_eq!(view.videos.last().unwrap().timestamp, Some(second_base));
         assert_eq!(view.pending.len(), 2);
         assert_eq!(view.selection.selected(), Some(PAGE_SIZE - 1));
         view.accept_page(
-            youtube::feed_results(
-                r#"{"entries":[{"id":"l-GaaivVJCE","title":"Long last row","timestamp":179}]} {"entries":[]}"#,
-            ).unwrap(),
+            youtube::feed_results(&format!(
+                r#"{{"entries":[{{"id":"l-GaaivVJCE","title":"Long last row","timestamp":{}}}]}} {{"entries":[]}}"#,
+                second_base - 1,
+            ))
+            .unwrap(),
             true,
         ).unwrap();
         assert_eq!(
@@ -7517,11 +7523,17 @@ tags.cancel = "z"
         assert!(view.needs_page());
         let page = youtube::feed_results(&format!("{{\"entries\":[]}}\n{}", dated(200))).unwrap();
         view.accept_page(page, true).unwrap();
-        assert_eq!(view.videos.first().unwrap().timestamp, Some(219));
+        assert_eq!(
+            view.videos.first().unwrap().timestamp,
+            Some(200 + PAGE_SIZE as i64 - 1)
+        );
         assert_eq!(view.videos.last().unwrap().timestamp, Some(200));
         view.accept_page(youtube::feed_results(r#"{"entries":[]}"#).unwrap(), true)
             .unwrap();
-        assert_eq!(view.videos[PAGE_SIZE].timestamp, Some(119));
+        assert_eq!(
+            view.videos[PAGE_SIZE].timestamp,
+            Some(100 + PAGE_SIZE as i64 - 1)
+        );
         assert_eq!(view.videos.last().unwrap().timestamp, Some(100));
         assert!(!view.has_more);
     }
@@ -7546,7 +7558,7 @@ tags.cancel = "z"
         app.finish_page(Ok(serde_json::json!({"entries": entries}).to_string()));
         assert!(app.views[1].videos.is_empty());
         assert_eq!(app.views[1].pending.len(), PAGE_SIZE);
-        assert_eq!(app.views[1].next_start, 21);
+        assert_eq!(app.views[1].next_start, PAGE_SIZE + 1);
         assert!(app.views[1].loaded);
         assert!(app.can_prefetch());
         app.key(KeyEvent::new(KeyCode::Char('j'), KeyModifiers::NONE));
@@ -7554,7 +7566,7 @@ tags.cancel = "z"
         assert!(app.job.is_none());
         app.finish_page(Err("Continuation failed".into()));
         assert_eq!(app.views[1].pending.len(), PAGE_SIZE);
-        assert_eq!(app.views[1].next_start, 21);
+        assert_eq!(app.views[1].next_start, PAGE_SIZE + 1);
         assert!(!app.can_prefetch());
         app.finish_page(Ok(r#"{"entries":[]}"#.into()));
         assert_eq!(app.views[1].videos.len(), PAGE_SIZE);
@@ -7594,12 +7606,12 @@ tags.cancel = "z"
             })
             .collect();
         app.finish_page(Ok(serde_json::json!({"entries": entries}).to_string()));
-        assert_eq!(app.views[0].next_start, 21);
+        assert_eq!(app.views[0].next_start, PAGE_SIZE + 1);
         app.views[0].selection.select(Some(17));
         app.remember_description(app.views[0].videos[17].id.clone(), "Cached.".into());
         app.switch_view(1);
         app.switch_view(0);
-        assert_eq!(app.views[0].next_start, 21);
+        assert_eq!(app.views[0].next_start, PAGE_SIZE + 1);
         assert_eq!(app.views[0].selection.selected(), Some(17));
         app.finish_page(Ok("not JSON".into()));
         app.update();
@@ -7609,7 +7621,7 @@ tags.cancel = "z"
             cached_description(&app.descriptions, &app.views[0].videos[17].id),
             Some("Cached.")
         );
-        assert_eq!(app.views[0].next_start, 21);
+        assert_eq!(app.views[0].next_start, PAGE_SIZE + 1);
         assert!(!app.views[0].needs_page());
         let view = &mut app.views[0];
         view.pending =
@@ -7664,6 +7676,7 @@ tags.cancel = "z"
                 })
             })
             .collect();
+        app.views[0].reset();
         app.views[0]
             .accept_page(
                 youtube::search_results(&serde_json::json!({"entries": entries}).to_string())
@@ -7673,11 +7686,25 @@ tags.cancel = "z"
             .unwrap();
         assert_eq!(app.views[0].selection.selected(), Some(0));
         assert!(!app.can_prefetch());
-        app.views[0].selection.select(Some(15));
+        let threshold = app.views[0].videos.len() - PAGE_SIZE / 3;
+        app.views[0].selection.select(Some(threshold - 1));
         assert!(!app.can_prefetch());
         app.key(KeyEvent::new(KeyCode::Char('j'), KeyModifiers::NONE));
         assert!(app.can_prefetch());
         assert!(app.job.is_none());
+        assert_eq!(app.views[0].selection.selected(), Some(threshold));
+        app.views[0].loaded = false;
+        assert!(!app.can_prefetch());
+        app.views[0].loaded = true;
+        app.views[0].has_more = false;
+        assert!(!app.can_prefetch());
+        app.views[0].has_more = true;
+        app.help_popup = Some(HelpPopup::default());
+        assert!(!app.can_prefetch());
+        app.help_popup = None;
+        app.views[0].selection.select(None);
+        assert!(!app.can_prefetch());
+        app.views[0].selection.select(Some(threshold));
         app.views[0].page_error = Some("Stop retrying".into());
         assert!(!app.can_prefetch());
         app.views[0].page_error = None;

src/youtube.rs

@@ -9,7 +9,7 @@ use std::{
 
 use serde::{Deserialize, Serialize};
 
-pub const PAGE_SIZE: usize = 20;
+pub const PAGE_SIZE: usize = 60;
 
 pub struct Page {
     pub videos: Vec<Video>,
@@ -222,10 +222,23 @@ fn page_end(start: usize) -> io::Result<usize> {
         .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "Invalid playlist page offset"))
 }
 
-// Own the extractor so replacing a request or quitting also cancels its process.
+// Own every extractor so replacing a request or quitting cancels the whole page.
 pub struct Job {
-    child: Child,
+    children: Vec<Extraction>,
+}
+
+struct Extraction {
+    child: Option<Child>,
     output: Receiver<io::Result<(String, String)>>,
+    status: Option<std::process::ExitStatus>,
+    completed: Option<String>,
+}
+
+fn channel_batches(channels: &[String]) -> impl Iterator<Item = &[String]> {
+    let count = channels.len().min(4);
+    (0..count).map(move |index| {
+        &channels[index * channels.len() / count..(index + 1) * channels.len() / count]
+    })
 }
 
 impl Job {
@@ -252,22 +265,26 @@ impl Job {
             ));
         }
         let end = page_end(start)?;
+        let start = start.to_string();
+        let end = end.to_string();
         // ponytail: offsets traverse prior continuations; use direct tokens/per-channel exhaustion
         // tracking if deep archives become slow.
-        Self::spawn(
-            extractor()
+        Self::spawn_all(channel_batches(channels).map(|batch| {
+            let mut command = extractor();
+            command
                 .args([
                     "--flat-playlist",
                     "--playlist-start",
-                    &start.to_string(),
+                    &start,
                     "--playlist-end",
-                    &end.to_string(),
+                    &end,
                     "--extractor-args",
                     "youtubetab:approximate_date",
                     "--dump-single-json",
                 ])
-                .args(channels.iter().map(|channel| format!("{channel}/videos"))),
-        )
+                .args(batch.iter().map(|channel| format!("{channel}/videos")));
+            command
+        }))
     }
 
     pub fn channel(id: &str) -> io::Result<Self> {
@@ -295,6 +312,64 @@ impl Job {
         ]))
     }
 
+    fn spawn(command: &mut Command) -> io::Result<Self> {
+        Ok(Self {
+            children: vec![Extraction::spawn(command)?],
+        })
+    }
+
+    fn spawn_all(commands: impl IntoIterator<Item = Command>) -> io::Result<Self> {
+        // Collect drops already-started children if a later spawn fails.
+        let children = commands
+            .into_iter()
+            .map(|mut command| Extraction::spawn(&mut command))
+            .collect::<io::Result<Vec<_>>>()?;
+        Ok(Self { children })
+    }
+
+    pub fn poll(&mut self) -> Option<Result<String, String>> {
+        if self.children.is_empty() {
+            return None;
+        }
+        let mut pending = false;
+        for child in &mut self.children {
+            if child.completed.is_some() {
+                continue;
+            }
+            match child.poll() {
+                None => pending = true,
+                Some(Ok(output)) => child.completed = Some(output),
+                Some(Err(error)) => {
+                    self.children.clear();
+                    return Some(Err(error));
+                }
+            }
+        }
+        if pending {
+            return None;
+        }
+        let mut children = self.children.drain(..);
+        // Keep the first buffer; only subsequent batches need to be appended.
+        let mut output = children.next()?.completed.take().unwrap();
+        output.reserve(
+            children
+                .as_slice()
+                .iter()
+                .map(|child| child.completed.as_ref().unwrap().len() + 1)
+                .sum(),
+        );
+        for mut child in children {
+            let next = child.completed.take().unwrap();
+            if !output.is_empty() && !next.is_empty() {
+                output.push('\n');
+            }
+            output.push_str(&next);
+        }
+        Some(Ok(output))
+    }
+}
+
+impl Extraction {
     fn spawn(command: &mut Command) -> io::Result<Self> {
         let mut child = command
             .stdin(Stdio::null())
@@ -321,16 +396,27 @@ impl Job {
                 .and_then(|text| stderr_result.map(|_| (text, errors)));
             let _ = sender.send(result);
         });
-        Ok(Self { child, output })
+        Ok(Self {
+            child: Some(child),
+            output,
+            status: None,
+            completed: None,
+        })
     }
 
-    pub fn poll(&mut self) -> Option<Result<String, String>> {
-        let status = match self.child.try_wait() {
-            Ok(Some(status)) => status,
-            Ok(None) => return None,
-            Err(error) => return Some(Err(format!("Cannot wait for yt-dlp: {error}"))),
-        };
-        match self.output.try_recv() {
+    fn poll(&mut self) -> Option<Result<String, String>> {
+        if self.status.is_none() {
+            self.status = match self.child.as_mut()?.try_wait() {
+                Ok(status) => status,
+                Err(error) => return Some(Err(format!("Cannot wait for yt-dlp: {error}"))),
+            };
+        }
+        let status = self.status?;
+        if !status.success() {
+            // Do not wait for inherited pipes held open by a failed child's descendants.
+            self.cancel();
+        }
+        let result = match self.output.try_recv() {
             Ok(Ok((output, _))) if status.success() => Some(Ok(output)),
             Ok(Ok((_, errors))) => Some(Err(if errors.trim().is_empty() {
                 format!("yt-dlp exited with {status}")
@@ -342,13 +428,29 @@ impl Job {
             Err(TryRecvError::Disconnected) => {
                 Some(Err("yt-dlp output reader disconnected".into()))
             }
+        };
+        if result.is_some() {
+            self.cancel();
+        }
+        result
+    }
+
+    fn cancel(&mut self) {
+        if let Some(mut child) = self.child.take() {
+            // The leader may have exited while descendants still own its output pipes.
+            // SAFETY: every extraction owns an isolated process group.
+            #[cfg(unix)]
+            unsafe {
+                libc::kill(-(child.id() as i32), libc::SIGKILL);
+            }
+            stop(&mut child);
         }
     }
 }
 
-impl Drop for Job {
+impl Drop for Extraction {
     fn drop(&mut self) {
-        stop(&mut self.child);
+        self.cancel();
     }
 }
 
@@ -554,6 +656,243 @@ impl Drop for PlayerLaunch {
 mod tests {
     use super::*;
 
+    #[test]
+    fn channel_batches_are_bounded_balanced_and_preserve_every_requested_url() {
+        for count in 0..=17 {
+            let channels: Vec<_> = (0..count).map(|index| format!("channel-{index}")).collect();
+            let batches: Vec<_> = channel_batches(&channels).collect();
+            assert_eq!(batches.len(), count.min(4));
+            assert!(batches.iter().all(|batch| !batch.is_empty()));
+            assert_eq!(batches.concat(), channels);
+            if !batches.is_empty() {
+                let smallest = batches.iter().map(|batch| batch.len()).min().unwrap();
+                let largest = batches.iter().map(|batch| batch.len()).max().unwrap();
+                assert!(largest - smallest <= 1);
+            }
+        }
+    }
+
+    #[cfg(unix)]
+    mod subprocesses {
+        use super::*;
+
+        fn controlled(script: &str) -> Command {
+            let mut command = isolated_command(OsStr::new("/bin/sh"));
+            command.args(["-c", &format!("kill -STOP $$; {script}")]);
+            command
+        }
+
+        fn wait_stopped(extraction: &Extraction) {
+            wait_stopped_pid(extraction.child.as_ref().unwrap().id());
+        }
+
+        fn wait_stopped_pid(pid: u32) {
+            let pid = pid as i32;
+            let mut status = 0;
+            loop {
+                // SAFETY: wait only for this test's owned child, without reaping its exit.
+                let result = unsafe { libc::waitpid(pid, &mut status, libc::WUNTRACED) };
+                if result == -1 && io::Error::last_os_error().kind() == io::ErrorKind::Interrupted {
+                    continue;
+                }
+                assert_eq!(result, pid);
+                assert!(libc::WIFSTOPPED(status));
+                break;
+            }
+        }
+
+        fn release(extraction: &mut Extraction) {
+            let child = extraction.child.as_mut().unwrap();
+            // SAFETY: this child is stopped at the deterministic test barrier.
+            assert_eq!(unsafe { libc::kill(child.id() as i32, libc::SIGCONT) }, 0);
+            child.wait().unwrap();
+            // Synchronize pipe draining too, then leave the real result for Job::poll.
+            // Blocking is confined to tests; production polling never waits for output.
+            let result = extraction.output.recv().unwrap();
+            let (sender, receiver) = mpsc::channel();
+            sender.send(result).unwrap();
+            extraction.output = receiver;
+        }
+
+        fn take_output(extraction: &mut Extraction) -> Receiver<io::Result<(String, String)>> {
+            let (_, receiver) = mpsc::channel();
+            std::mem::replace(&mut extraction.output, receiver)
+        }
+
+        fn assert_reaped(pid: u32) {
+            let mut status = 0;
+            // SAFETY: query only this test's child; ECHILD proves cancellation reaped it.
+            assert_eq!(
+                unsafe { libc::waitpid(pid as i32, &mut status, libc::WNOHANG) },
+                -1
+            );
+            assert_eq!(
+                io::Error::last_os_error().raw_os_error(),
+                Some(libc::ECHILD)
+            );
+        }
+
+        #[test]
+        fn a_later_spawn_failure_cancels_and_reaps_already_started_batches() {
+            static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
+            let directory = std::env::temp_dir().join(format!(
+                "tubular-extractor-spawn-{}-{}",
+                std::process::id(),
+                NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed)
+            ));
+            std::fs::create_dir(&directory).unwrap();
+            let ready = directory.join("ready");
+            let mut first = isolated_command(OsStr::new("/bin/sh"));
+            first
+                .args(["-c", "printf '%s' $$ > \"$1\"; kill -STOP $$", "test"])
+                .arg(&ready);
+            let mut started_pid = None;
+            let commands = std::iter::once(first).chain(std::iter::once_with(|| {
+                // The file only communicates identity; SIGSTOP is the completion barrier.
+                let pid = loop {
+                    if let Ok(text) = std::fs::read_to_string(&ready)
+                        && let Ok(pid) = text.parse::<u32>()
+                    {
+                        break pid;
+                    }
+                    thread::yield_now();
+                };
+                wait_stopped_pid(pid);
+                started_pid = Some(pid);
+                isolated_command(directory.join("missing-extractor").as_os_str())
+            }));
+            let error = Job::spawn_all(commands).err().unwrap();
+            assert_eq!(error.kind(), io::ErrorKind::NotFound);
+            assert_reaped(started_pid.unwrap());
+            std::fs::remove_dir_all(directory).unwrap();
+        }
+
+        #[test]
+        fn aggregate_waits_for_all_batches_and_merges_in_request_not_completion_order() {
+            let mut job = Job::spawn_all([
+                controlled("printf '%s' '{\"batch\":1}'"),
+                controlled("printf '%s' '{\"batch\":2}'"),
+                controlled("printf '%s' '{\"batch\":3}'"),
+            ])
+            .unwrap();
+            for child in &job.children {
+                wait_stopped(child);
+            }
+            assert!(job.poll().is_none());
+            release(&mut job.children[2]);
+            assert!(
+                job.poll().is_none(),
+                "the last batch cannot complete a page"
+            );
+            release(&mut job.children[1]);
+            assert!(job.poll().is_none(), "the first batch is still outstanding");
+            release(&mut job.children[0]);
+            let output = job.poll().unwrap().unwrap();
+            let batches: Vec<_> = serde_json::Deserializer::from_str(&output)
+                .into_iter::<serde_json::Value>()
+                .map(Result::unwrap)
+                .collect();
+            assert_eq!(
+                batches,
+                [
+                    serde_json::json!({"batch":1}),
+                    serde_json::json!({"batch":2}),
+                    serde_json::json!({"batch":3}),
+                ]
+            );
+            assert!(job.poll().is_none());
+        }
+
+        #[test]
+        fn a_failed_batch_discards_completed_output_and_cancels_pending_siblings() {
+            let mut job = Job::spawn_all([
+                controlled("printf '%s' '{\"complete\":true}'"),
+                controlled(
+                    "printf '%s' '{\"partial\":true}'; printf '%s' 'batch failed' >&2; exit 7",
+                ),
+                controlled("/bin/sh -c 'kill -STOP $$' & wait"),
+            ])
+            .unwrap();
+            for child in &job.children {
+                wait_stopped(child);
+            }
+            let pending_pid = job.children[2].child.as_ref().unwrap().id();
+            let pending_output = take_output(&mut job.children[2]);
+            release(&mut job.children[0]);
+            assert!(job.poll().is_none());
+            release(&mut job.children[1]);
+            assert_eq!(job.poll().unwrap().unwrap_err(), "batch failed");
+            assert!(pending_output.recv().unwrap().is_ok());
+            assert_reaped(pending_pid);
+            assert!(job.poll().is_none());
+        }
+
+        #[test]
+        fn unreadable_batch_output_fails_the_whole_page_and_cancels_siblings() {
+            let mut job = Job::spawn_all([
+                controlled("printf '\\377'"),
+                controlled("printf '%s' 'must not finish'"),
+            ])
+            .unwrap();
+            for child in &job.children {
+                wait_stopped(child);
+            }
+            let pending_pid = job.children[1].child.as_ref().unwrap().id();
+            let pending_output = take_output(&mut job.children[1]);
+            release(&mut job.children[0]);
+            assert!(
+                job.poll()
+                    .unwrap()
+                    .unwrap_err()
+                    .starts_with("Cannot read yt-dlp output:")
+            );
+            assert!(pending_output.recv().unwrap().is_ok());
+            assert_reaped(pending_pid);
+        }
+
+        #[test]
+        fn dropping_a_page_cancels_and_reaps_every_batch() {
+            let mut job = Job::spawn_all([
+                controlled("/bin/sh -c 'kill -STOP $$' & wait"),
+                controlled("printf '%s' 'must not finish'"),
+            ])
+            .unwrap();
+            for child in &job.children {
+                wait_stopped(child);
+            }
+            let pids: Vec<_> = job
+                .children
+                .iter()
+                .map(|child| child.child.as_ref().unwrap().id())
+                .collect();
+            let outputs: Vec<_> = job.children.iter_mut().map(take_output).collect();
+            drop(job);
+            for output in outputs {
+                assert!(output.recv().unwrap().is_ok());
+            }
+            for pid in pids {
+                assert_reaped(pid);
+            }
+        }
+
+        #[test]
+        fn dropping_after_the_leader_exits_still_cancels_inherited_pipe_owners() {
+            let mut job =
+                Job::spawn_all([controlled("/bin/sh -c 'kill -STOP $$' & exit 0")]).unwrap();
+            wait_stopped(&job.children[0]);
+            let child = job.children[0].child.as_mut().unwrap();
+            let pid = child.id();
+            // SAFETY: resume this test's stopped leader, which exits without its descendant.
+            assert_eq!(unsafe { libc::kill(pid as i32, libc::SIGCONT) }, 0);
+            assert!(child.wait().unwrap().success());
+            let output = take_output(&mut job.children[0]);
+            drop(job);
+            // EOF proves the descendant that inherited stdout/stderr was also canceled.
+            assert!(output.recv().unwrap().is_ok());
+            assert_reaped(pid);
+        }
+    }
+
     #[test]
     fn player_launch_rejects_unsafe_video_ids_before_spawning() {
         for id in ["", "--help", "jNQXAC9IVRw&list=other", "日本語"] {