fix: include music releases in channel feeds

Channel feeds previously queried only the Videos tab, excluding artist release tracks visible in general Search. Flatten Releases albums after uploads while retaining raw-row pagination, deduplication, cancellation, and extractor errors.

src/main.rs

@@ -85,19 +85,21 @@
 //! Choices include untagged registrations and category-only channels, deduplicated with readable handle labels.
 //! Channels, combined feeds and unpinned categories share one temporary tab; hidden views retain history.
 //! Channel views use the same pagination but extract only their stored channel identity; S stays category-only.
+//! Unfiltered channel/feed/tag catalogs traverse uploads, then flatten Releases album tracks.
+//! Release tracks share the raw-row budget; absent Releases tabs are normal, other failures are page errors.
 //! Config changes preserve unsaved browsing, relabel saved identities, and drop explicitly removed subscriptions.
 //! r reloads config and refreshes the current view.
 //! Feed includes saved `[channels]` entries, even untagged, plus direct tag entries, without duplicates.
 //! Untagging is not unsubscribing: remove mappings and tag references from the file to unsubscribe.
-//! Search pages contain 60 results; Feed/tag pages share a total request budget of 60 uploads or query matches.
-//! Search uses one extractor; feeds batch matching channel ranges across up to four concurrent extractors.
+//! Search pages contain 60 results; Feed/tag pages share a total budget of 60 catalog rows or query matches.
+//! Upload/search ranges are batched; release discovery/albums also stay within four concurrent extractors.
 //! Invalid individual video entries are discarded; raw entry counts still determine pagination.
 //! Invalid JSON, playlist-level metadata, and extractor failures remain page errors.
 //! Forward video navigation authorizes prefetch within 20 rows of the loaded end in unobstructed normal mode.
 //! Each accepted page clears that intent, so short or invalid batches never auto-chain on startup.
 //! Loading result/preview panes animate a spinner after their title, including additional pages.
 //! Info spinners cover image download/decode/resize and description fetching; errors stop them.
-//! Feed rows, including undated rows, are globally newest-first among loaded uploads; selection follows video ID.
+//! Feed rows, including undated rows, are globally newest-first among loaded catalog entries; selection follows video ID.
 //! Switching views cancels in-flight extraction; returning retains the unaccepted request and per-channel offsets.
 //! Errors keep loaded rows and stop automatic retries; r reloads config and restarts the current view.
 //! Deep pages may slow down because yt-dlp offsets re-traverse earlier continuation pages.

src/youtube.rs

@@ -10,6 +10,8 @@ use std::{
 
 use serde::{Deserialize, Serialize};
 
+mod catalog;
+
 pub const PAGE_SIZE: usize = 60;
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -387,6 +389,7 @@ const MAX_EXTRACTORS: usize = 4;
 pub struct Job {
     children: Vec<Extraction>,
     queued: VecDeque<Command>,
+    feed: Option<catalog::FeedExpansion>,
 }
 
 struct Extraction {
@@ -486,7 +489,11 @@ impl Job {
     }
 
     pub fn feed(channels: &[String], requests: &[FeedRequest], query: &str) -> io::Result<Self> {
-        Self::spawn_all(feed_commands(channels, requests, query)?)
+        let mut job = Self::spawn_all(feed_commands(channels, requests, query)?)?;
+        if query.is_empty() {
+            job.feed = Some(catalog::FeedExpansion::new(channels, requests));
+        }
+        Ok(job)
     }
 
     pub fn channel(id: &str) -> io::Result<Self> {
@@ -518,6 +525,7 @@ impl Job {
         Ok(Self {
             children: vec![Extraction::spawn(command)?],
             queued: VecDeque::new(),
+            feed: None,
         })
     }
 
@@ -525,6 +533,7 @@ impl Job {
         let mut job = Self {
             children: Vec::new(),
             queued: VecDeque::new(),
+            feed: None,
         };
         // Dropping job also cancels started children if any initial spawn fails.
         for mut command in commands {
@@ -538,6 +547,17 @@ impl Job {
     }
 
     pub fn poll(&mut self) -> Option<Result<String, String>> {
+        if let Some(mut feed) = self.feed.take() {
+            let result = feed.poll(self);
+            if result.is_none() {
+                self.feed = Some(feed);
+            }
+            return result;
+        }
+        self.poll_extractions()
+    }
+
+    fn poll_extractions(&mut self) -> Option<Result<String, String>> {
         if self.children.is_empty() {
             return None;
         }

src/youtube/catalog.rs

@@ -0,0 +1,343 @@
+use super::*;
+use serde_json::value::RawValue;
+
+#[derive(Deserialize, Serialize)]
+struct Playlist<T> {
+    entries: Vec<T>,
+    channel: Option<String>,
+    channel_id: Option<String>,
+    uploader_id: Option<String>,
+    #[serde(skip_serializing)]
+    playlist_count: Option<usize>,
+    #[serde(rename = "_type", skip_serializing)]
+    kind: Option<String>,
+}
+
+impl<T> Playlist<T> {
+    fn validate(&self, start: usize, count: usize) -> Result<(), String> {
+        if self.kind.as_deref().is_some_and(|kind| kind != "playlist") {
+            return Err("Invalid catalog playlist type".into());
+        }
+        if self
+            .channel_id
+            .as_deref()
+            .is_some_and(|id| !valid_channel_id(id))
+        {
+            return Err("Catalog returned an invalid YouTube channel ID".into());
+        }
+        if self.entries.len() > count {
+            return Err("Catalog returned more rows than requested".into());
+        }
+        match self.playlist_count {
+            Some(total) if total.saturating_sub(start - 1).min(count) != self.entries.len() => {
+                Err("Catalog playlist count did not match its rows".into())
+            }
+            _ => Ok(()),
+        }
+    }
+
+    fn exhausted_total(&self, start: usize) -> Result<usize, String> {
+        match self.playlist_count {
+            Some(total) => Ok(total),
+            None if start == 1 || !self.entries.is_empty() => (start - 1)
+                .checked_add(self.entries.len())
+                .ok_or_else(|| "Catalog playlist count overflow".into()),
+            None => Err("Exhausted catalog playlist has no total count".into()),
+        }
+    }
+}
+
+fn playlist(output: &str, start: usize, count: usize) -> Result<Playlist<Box<RawValue>>, String> {
+    let playlist: Playlist<Box<RawValue>> = serde_json::from_str(output)
+        .map_err(|error| format!("Invalid catalog response: {error}"))?;
+    playlist.validate(start, count)?;
+    Ok(playlist)
+}
+
+struct FeedChannel {
+    channel: String,
+    request: FeedRequest,
+    playlist: Option<Playlist<Box<RawValue>>>,
+    releases: Option<ChannelReleases>,
+    job: Option<Job>,
+}
+
+pub(super) struct FeedExpansion {
+    channels: Vec<FeedChannel>,
+    uploads_complete: bool,
+}
+
+impl FeedExpansion {
+    pub(super) fn new(channels: &[String], requests: &[FeedRequest]) -> Self {
+        Self {
+            channels: requests
+                .iter()
+                .map(|&request| FeedChannel {
+                    channel: channels[request.channel].clone(),
+                    request,
+                    playlist: None,
+                    releases: None,
+                    job: None,
+                })
+                .collect(),
+            uploads_complete: false,
+        }
+    }
+
+    fn full_uploads(&self, output: &str) -> Result<bool, String> {
+        // Borrow raw rows on the ordinary uploads-only path; return its output unchanged.
+        let mut playlists =
+            serde_json::Deserializer::from_str(output).into_iter::<Playlist<&RawValue>>();
+        for channel in &self.channels {
+            let playlist = playlists
+                .next()
+                .ok_or_else(|| "Feed returned fewer playlists than requested".to_owned())?
+                .map_err(|error| format!("Invalid catalog response: {error}"))?;
+            playlist.validate(channel.request.start, channel.request.count)?;
+            if playlist.entries.len() < channel.request.count {
+                return Ok(false);
+            }
+        }
+        if let Some(extra) = playlists.next() {
+            extra.map_err(|error| format!("Invalid catalog response: {error}"))?;
+            return Err("Feed returned more playlists than requested".into());
+        }
+        Ok(true)
+    }
+
+    fn accept_uploads(&mut self, output: &str) -> Result<(), String> {
+        let mut playlists =
+            serde_json::Deserializer::from_str(output).into_iter::<Playlist<Box<RawValue>>>();
+        for channel in &mut self.channels {
+            let playlist = playlists
+                .next()
+                .ok_or_else(|| "Feed returned fewer playlists than requested".to_owned())?
+                .map_err(|error| format!("Invalid catalog response: {error}"))?;
+            playlist.validate(channel.request.start, channel.request.count)?;
+            let remaining = channel.request.count - playlist.entries.len();
+            if remaining != 0 {
+                // Beyond-end requests need the extractor's total to locate release tracks.
+                let skip = (channel.request.start - 1)
+                    .saturating_sub(playlist.exhausted_total(channel.request.start)?);
+                channel.releases = Some(ChannelReleases::new(
+                    channel.channel.clone(),
+                    skip,
+                    remaining,
+                ));
+            }
+            channel.playlist = Some(playlist);
+        }
+        if let Some(extra) = playlists.next() {
+            extra.map_err(|error| format!("Invalid catalog response: {error}"))?;
+            return Err("Feed returned more playlists than requested".into());
+        }
+        self.uploads_complete = true;
+        Ok(())
+    }
+
+    pub(super) fn poll(&mut self, uploads: &mut Job) -> Option<Result<String, String>> {
+        if !self.uploads_complete {
+            let output = match uploads.poll_extractions()? {
+                Ok(output) => output,
+                Err(error) => return Some(Err(error)),
+            };
+            match self.full_uploads(&output) {
+                Ok(true) => return Some(Ok(output)),
+                Ok(false) => {}
+                Err(error) => return Some(Err(error)),
+            }
+            if let Err(error) = self.accept_uploads(&output) {
+                return Some(Err(error));
+            }
+        }
+        match self.poll_releases() {
+            Ok(false) => None,
+            Err(error) => Some(Err(error)),
+            Ok(true) => {
+                let mut output = Vec::new();
+                for channel in &self.channels {
+                    if !output.is_empty() {
+                        output.push(b'\n');
+                    }
+                    if let Err(error) =
+                        serde_json::to_writer(&mut output, channel.playlist.as_ref().unwrap())
+                    {
+                        return Some(Err(format!("Cannot encode catalog response: {error}")));
+                    }
+                }
+                Some(String::from_utf8(output).map_err(|error| error.to_string()))
+            }
+        }
+    }
+
+    fn poll_releases(&mut self) -> Result<bool, String> {
+        let mut active = self
+            .channels
+            .iter()
+            .filter(|channel| channel.job.is_some())
+            .count();
+        let mut complete = true;
+        for channel in &mut self.channels {
+            let Some(releases) = &mut channel.releases else {
+                continue;
+            };
+            if let Some(job) = &mut channel.job {
+                if let Some(result) = job.poll_extractions() {
+                    channel.job = None;
+                    active -= 1;
+                    match result {
+                        Ok(output) => releases.accept(&output)?,
+                        Err(error) => releases.accept_error(error)?,
+                    }
+                }
+            }
+            if releases.done() {
+                let entries = channel.releases.take().unwrap().into_entries();
+                channel.playlist.as_mut().unwrap().entries.extend(entries);
+            } else {
+                complete = false;
+                if channel.job.is_none() && active < MAX_EXTRACTORS {
+                    channel.job = Some(
+                        Job::spawn(&mut releases.command().map_err(|error| error.to_string())?)
+                            .map_err(|error| error.to_string())?,
+                    );
+                    active += 1;
+                }
+            }
+        }
+        Ok(complete)
+    }
+}
+
+// ponytail: each Feed page revisits albums; index offsets if traversal becomes costly.
+struct ChannelReleases {
+    channel: String,
+    skip: usize,
+    count: usize,
+    entries: Vec<Box<RawValue>>,
+    albums: VecDeque<String>,
+    discovery_start: usize,
+    discovery_exhausted: bool,
+}
+
+impl ChannelReleases {
+    fn new(channel: String, skip: usize, count: usize) -> Self {
+        Self {
+            channel,
+            skip,
+            count,
+            entries: Vec::new(),
+            albums: VecDeque::new(),
+            discovery_start: 1,
+            discovery_exhausted: false,
+        }
+    }
+
+    fn command(&self) -> io::Result<Command> {
+        let (start, count, url) = match self.albums.front() {
+            Some(id) => (
+                self.skip.checked_add(1).ok_or_else(|| {
+                    io::Error::new(io::ErrorKind::InvalidInput, "Release track offset overflow")
+                })?,
+                self.count - self.entries.len(),
+                format!("https://www.youtube.com/playlist?list={id}"),
+            ),
+            None => (
+                self.discovery_start,
+                PAGE_SIZE,
+                format!("{}/releases", self.channel),
+            ),
+        };
+        let end = range_end(start, count)?;
+        let mut command = extractor();
+        command.args([
+            "--flat-playlist",
+            "--playlist-start",
+            &start.to_string(),
+            "--playlist-end",
+            &end.to_string(),
+            "--extractor-args",
+            "youtubetab:approximate_date",
+            "--dump-single-json",
+            &url,
+        ]);
+        Ok(command)
+    }
+
+    fn accept(&mut self, output: &str) -> Result<(), String> {
+        if self.albums.front().is_some() {
+            let remaining = self.count - self.entries.len();
+            let start = self
+                .skip
+                .checked_add(1)
+                .ok_or("Release track offset overflow")?;
+            let playlist = playlist(output, start, remaining)?;
+            if playlist.entries.len() < remaining {
+                self.skip = self.skip.saturating_sub(playlist.exhausted_total(start)?);
+            }
+            self.entries.extend(playlist.entries);
+            self.albums.pop_front();
+        } else {
+            let playlist = playlist(output, self.discovery_start, PAGE_SIZE)?;
+            let end =
+                range_end(self.discovery_start, PAGE_SIZE).map_err(|error| error.to_string())?;
+            self.discovery_exhausted = playlist.entries.len() < PAGE_SIZE
+                || playlist.playlist_count.is_some_and(|count| count <= end);
+            #[derive(Deserialize)]
+            struct Album {
+                id: String,
+            }
+            for entry in playlist.entries {
+                let album: Album = serde_json::from_str(entry.get())
+                    .map_err(|error| format!("Invalid release album: {error}"))?;
+                if album.id.is_empty()
+                    || album.id.len() > 128
+                    || !album
+                        .id
+                        .bytes()
+                        .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_'))
+                {
+                    return Err("Release album has an invalid YouTube playlist ID".into());
+                }
+                self.albums.push_back(album.id);
+            }
+            if !self.discovery_exhausted {
+                self.discovery_start = end
+                    .checked_add(1)
+                    .ok_or("Release discovery offset overflow")?;
+            }
+        }
+        Ok(())
+    }
+
+    fn accept_error(&mut self, error: String) -> Result<(), String> {
+        // Only the optional tab's exact extractor error is ordinary exhaustion.
+        if self.albums.is_empty()
+            && error
+                .trim_end()
+                .ends_with("This channel does not have a releases tab")
+            && !error
+                .lines()
+                .rev()
+                .skip(1)
+                .any(|line| line.trim_start().starts_with("ERROR:"))
+        {
+            self.discovery_exhausted = true;
+            Ok(())
+        } else {
+            Err(error)
+        }
+    }
+
+    fn done(&self) -> bool {
+        self.entries.len() == self.count || (self.discovery_exhausted && self.albums.is_empty())
+    }
+
+    fn into_entries(self) -> Vec<Box<RawValue>> {
+        self.entries
+    }
+}
+
+#[cfg(test)]
+#[path = "../../tests/unit/youtube_catalog.rs"]
+mod tests;