hl-valdemar/tubular
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

feat: add categorized channel subscription feeds

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-10-09 21:33:31 +02:00
parent 3954de092e
commit 656474df5b
5 changed files with 1107 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/youtube.rs
+148 -22
View File
@@ -13,6 +13,8 @@ pub struct Video {
pub id: String,
pub title: String,
pub channel: Option<String>,
pub channel_id: Option<String>,
pub timestamp: Option<i64>,
// Search descriptions are truncated; only full video metadata populates this cache.
#[serde(skip)]
pub description: Option<String>,
@@ -21,30 +23,86 @@ pub struct Video {
#[derive(Deserialize)]
struct SearchResults {
entries: Vec<Option<Video>>,
channel: Option<String>,
channel_id: Option<String>,
}
pub fn search_results(output: &str) -> Result<Vec<Video>, String> {
let results: SearchResults = serde_json::from_str(output)
.map_err(|error| format!("Invalid search response: {error}"))?;
let mut videos = Vec::new();
for mut video in results.entries.into_iter().flatten() {
if video.id.len() != 11
|| !video
.id
.bytes()
.all(|c| c.is_ascii_alphanumeric() || c == b'-' || c == b'_')
{
return Err("Search returned an invalid YouTube video ID".into());
}
video.title.retain(|c| !c.is_control());
if let Some(channel) = &mut video.channel {
channel.retain(|c| !c.is_control());
}
videos.push(video);
let mut videos: Vec<Video> = results.entries.into_iter().flatten().collect();
for video in &mut videos {
validate_video(video).map_err(|error| format!("Search returned {error}"))?;
}
Ok(videos)
}
fn valid_channel_id(id: &str) -> bool {
id.len() == 24
&& id.starts_with("UC")
&& id
.bytes()
.all(|c| c.is_ascii_alphanumeric() || c == b'-' || c == b'_')
}
fn validate_video(video: &mut Video) -> Result<(), String> {
if video.id.len() != 11
|| !video
.id
.bytes()
.all(|c| c.is_ascii_alphanumeric() || c == b'-' || c == b'_')
{
return Err("an invalid YouTube video ID".into());
}
if video
.channel_id
.as_deref()
.is_some_and(|id| !valid_channel_id(id))
{
return Err("an invalid YouTube channel ID".into());
}
video.title.retain(|c| !c.is_control());
if let Some(channel) = &mut video.channel {
channel.retain(|c| !c.is_control());
}
Ok(())
}
pub fn feed_results(output: &str) -> Result<Vec<Video>, String> {
let mut videos = Vec::new();
let mut playlists = 0;
// Multiple yt-dlp URLs produce one playlist object per line, not one combined object.
for result in serde_json::Deserializer::from_str(output).into_iter::<SearchResults>() {
let playlist = result.map_err(|error| format!("Invalid feed response: {error}"))?;
playlists += 1;
if playlist
.channel_id
.as_deref()
.is_some_and(|id| !valid_channel_id(id))
{
return Err("Feed returned an invalid YouTube channel ID".into());
}
for mut video in playlist.entries.into_iter().flatten() {
if video.channel_id.is_none() {
video.channel_id = playlist.channel_id.clone();
}
if video.channel.is_none() {
video.channel = playlist.channel.clone();
}
validate_video(&mut video).map_err(|error| format!("Feed returned {error}"))?;
videos.push(video);
}
}
if playlists == 0 {
return Err("Feed returned no playlist data".into());
}
// Keep the newest occurrence of each ID without copying IDs into a second collection.
videos.sort_unstable_by(|a, b| a.id.cmp(&b.id).then_with(|| b.timestamp.cmp(&a.timestamp)));
videos.dedup_by(|a, b| a.id == b.id);
videos.sort_by(|a, b| b.timestamp.cmp(&a.timestamp));
Ok(videos)
}
pub fn full_description(output: &str, expected_id: &str) -> Result<String, String> {
#[derive(Deserialize)]
struct Details {
@@ -69,25 +127,46 @@ pub struct Job {
impl Job {
pub fn search(query: &str) -> io::Result<Self> {
Self::spawn(&[
Self::spawn(extractor().args([
"--flat-playlist",
"--dump-single-json",
&format!("ytsearch20:{query}"),
])
]))
}
pub fn feed(channels: &[String]) -> io::Result<Self> {
if channels.is_empty() {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"No channels in this category",
));
}
// ponytail: 20 entries per channel's /videos tab; add pagination if older uploads are needed.
Self::spawn(
extractor()
.args([
"--flat-playlist",
"--playlist-end",
"20",
"--extractor-args",
"youtubetab:approximate_date",
"--dump-single-json",
])
.args(channels.iter().map(|channel| format!("{channel}/videos"))),
)
}
pub fn description(id: &str) -> io::Result<Self> {
Self::spawn(&[
Self::spawn(extractor().args([
"--no-playlist",
"--skip-download",
"--dump-single-json",
&format!("https://www.youtube.com/watch?v={id}"),
])
]))
}
fn spawn(args: &[&str]) -> io::Result<Self> {
let mut child = extractor()
.args(args)
fn spawn(command: &mut Command) -> io::Result<Self> {
let mut child = command
.stdin(Stdio::null())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
@@ -300,10 +379,57 @@ mod tests {
.unwrap();
assert_eq!(videos[0].title, "Rust");
assert_eq!(videos[0].channel, None);
assert_eq!(videos[0].channel_id, None);
assert!(
search_results(
r#"{"entries":[{"id":"rQ_J9WH6CGk","title":"Bad","channel_id":"not-a-channel"}]}"#
)
.is_err()
);
assert!(search_results(r#"{"entries":[{"id":"--bad","title":"Bad"}]}"#).is_err());
assert!(search_results("not JSON").is_err());
}
#[test]
fn feeds_merge_newest_first_deduplicate_and_inherit_channel_identity() {
let output = r#"{"channel_id":"UCaYhcUwRBNscFNUKTjgPFiA","channel":"Rust\u001b","entries":[null,{"id":"l-GaaivVJCE","title":"Older","timestamp":100},{"id":"jNQXAC9IVRw","title":"Unknown","timestamp":null}]}
{"channel_id":"UCaYhcUwRBNscFNUKTjgPFiA","channel":"Rust","entries":[{"id":"SLV9AJxnb1U","title":"Newest","timestamp":300},{"id":"l-GaaivVJCE","title":"Newer","timestamp":200},{"id":"rQ_J9WH6CGk","title":"No metadata","channel_id":null}]}"#;
let videos = feed_results(output).unwrap();
assert_eq!(
videos
.iter()
.map(|video| video.id.as_str())
.collect::<Vec<_>>(),
["SLV9AJxnb1U", "l-GaaivVJCE", "jNQXAC9IVRw", "rQ_J9WH6CGk"]
);
assert_eq!(videos[1].title, "Newer");
assert_eq!(videos[1].timestamp, Some(200));
assert_eq!(videos[2].timestamp, None);
assert_eq!(videos[2].channel.as_deref(), Some("Rust"));
assert!(
videos
.iter()
.all(|video| video.channel_id.as_deref() == Some("UCaYhcUwRBNscFNUKTjgPFiA"))
);
let unavailable = feed_results(
r#"{"entries":[null,{"id":"jNQXAC9IVRw","title":"Unknown","channel_id":null}]}"#,
)
.unwrap();
assert_eq!(unavailable[0].channel_id, None);
assert!(feed_results(r#"{"channel_id":"malformed","entries":[]}"#).is_err());
assert!(
feed_results(
r#"{"entries":[{"id":"jNQXAC9IVRw","title":"Bad","channel_id":"malformed"}]}"#
)
.is_err()
);
assert!(feed_results(r#"{"entries":[{"id":"bad","title":"Bad"}]}"#).is_err());
assert!(feed_results("").is_err());
assert!(feed_results("{}").is_err());
assert!(feed_results("{not JSON}").is_err());
assert!(feed_results(r#"{"entries":[]}"#).unwrap().is_empty());
}
#[test]
fn full_descriptions_preserve_paragraphs_and_reject_mismatched_videos() {
let metadata = r#"{"id":"jNQXAC9IVRw","description":"First paragraph.\n\n日本語\tsecond paragraph.\u001b\nLast line."}"#;