hl-valdemar
063177b7df
Channel feeds previously queried only the Videos tab, excluding artist release tracks visible in general Search. Flatten Releases albums after uploads while retaining raw-row pagination, deduplication, cancellation, and extractor errors.
272 lines
9.7 KiB
Rust
272 lines
9.7 KiB
Rust
use super::super::{FeedCursor, PAGE_SIZE, feed_results};
|
|
use super::*;
|
|
use serde_json::{Value, json};
|
|
|
|
const CHANNEL: &str = "https://www.youtube.com/channel/UCaaaaaaaaaaaaaaaaaaaaaa";
|
|
const CHANNEL_ID: &str = "UCaaaaaaaaaaaaaaaaaaaaaa";
|
|
|
|
fn album(index: usize) -> Value {
|
|
json!({"id": format!("PL{index:032}"), "_type": "url"})
|
|
}
|
|
|
|
fn track(id: &str, title: &str, timestamp: i64) -> Value {
|
|
json!({"id": id, "title": title, "timestamp": timestamp})
|
|
}
|
|
|
|
fn response(entries: Vec<Value>, total: Option<usize>) -> String {
|
|
json!({"entries": entries, "playlist_count": total}).to_string()
|
|
}
|
|
|
|
fn accept(releases: &mut ChannelReleases, entries: Vec<Value>, total: Option<usize>) {
|
|
// Construct, but never execute, the next extractor request.
|
|
releases.command().unwrap();
|
|
releases.accept(&response(entries, total)).unwrap();
|
|
}
|
|
|
|
fn range(releases: &ChannelReleases) -> (usize, usize) {
|
|
let command = releases.command().unwrap();
|
|
let args: Vec<_> = command.get_args().collect();
|
|
let value = |flag: &str| {
|
|
args.windows(2).find(|pair| pair[0] == flag).unwrap()[1]
|
|
.to_str()
|
|
.unwrap()
|
|
.parse::<usize>()
|
|
.unwrap()
|
|
};
|
|
(value("--playlist-start"), value("--playlist-end"))
|
|
}
|
|
|
|
fn feed_output(releases: ChannelReleases) -> String {
|
|
assert!(releases.done());
|
|
let entries = releases.into_entries();
|
|
json!({"channel_id": CHANNEL_ID, "entries": entries}).to_string()
|
|
}
|
|
|
|
#[test]
|
|
fn mixed_feed_merges_uploads_and_release_tracks_without_advancing_other_channels() {
|
|
let mut cursor = FeedCursor::default();
|
|
let requests = cursor.plan(2).unwrap();
|
|
let channels = vec![CHANNEL.into(), "https://www.youtube.com/@ordinary".into()];
|
|
let mut expansion = FeedExpansion::new(&channels, &requests);
|
|
let uploads = json!({
|
|
"channel_id": CHANNEL_ID,
|
|
"uploader_id": "@artist",
|
|
"playlist_count": 3,
|
|
"entries": [
|
|
track("aaaaaaaaaaa", "Upload", 100),
|
|
track("ddddddddddd", "Second upload", 100),
|
|
track("eeeeeeeeeee", "Third upload", 100)
|
|
]
|
|
});
|
|
let ordinary = json!({
|
|
"entries": (0..30).map(|index| {
|
|
track(&format!("b{index:010}"), "Ordinary upload", 50)
|
|
}).collect::<Vec<_>>()
|
|
});
|
|
expansion
|
|
.accept_uploads(&format!("{uploads}\n{ordinary}"))
|
|
.unwrap();
|
|
let releases = expansion.channels[0].releases.as_mut().unwrap();
|
|
accept(releases, vec![album(1)], Some(1));
|
|
let mut tracks = vec![
|
|
track("aaaaaaaaaaa", "Release duplicate", 200),
|
|
track("ccccccccccc", "peppermint parkway", 300),
|
|
];
|
|
tracks.resize(27, Value::Null);
|
|
accept(releases, tracks, Some(27));
|
|
|
|
let mut finished_uploads = Job::spawn_all(std::iter::empty::<Command>()).unwrap();
|
|
let output = expansion.poll(&mut finished_uploads).unwrap().unwrap();
|
|
let page = feed_results(&output, &requests).unwrap();
|
|
assert_eq!(
|
|
page.feed_progress
|
|
.iter()
|
|
.map(|p| p.count)
|
|
.collect::<Vec<_>>(),
|
|
[30, 30]
|
|
);
|
|
assert_eq!(page.videos.len(), 34);
|
|
assert_eq!(page.videos[0].id, "ccccccccccc");
|
|
assert_eq!(page.videos[0].channel_id.as_deref(), Some(CHANNEL_ID));
|
|
assert_eq!(page.videos[0].uploader_id.as_deref(), Some("@artist"));
|
|
assert_eq!(page.videos[1].title, "Release duplicate");
|
|
cursor.accept(&page.feed_progress).unwrap();
|
|
assert!(
|
|
cursor
|
|
.plan(2)
|
|
.unwrap()
|
|
.iter()
|
|
.all(|request| request.start == 31)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn release_tracks_reach_the_feed_without_refunding_filtered_or_duplicate_rows() {
|
|
let mut cursor = FeedCursor::default();
|
|
let requests = cursor.plan(1).unwrap();
|
|
let mut releases = ChannelReleases::new(CHANNEL.into(), 0, requests[0].count);
|
|
accept(&mut releases, vec![album(1), album(2)], Some(2));
|
|
accept(
|
|
&mut releases,
|
|
vec![
|
|
track("aaaaaaaaaaa", "Older upload duplicate", 100),
|
|
json!({"id": "bbbbbbbbbbb", "title": "Members", "availability": "subscriber_only"}),
|
|
json!({"id": "invalid", "title": "Invalid video ID"}),
|
|
],
|
|
Some(3),
|
|
);
|
|
assert!(!releases.done());
|
|
assert_eq!(range(&releases), (1, PAGE_SIZE - 3));
|
|
let mut second_album = vec![
|
|
track("aaaaaaaaaaa", "Newer release duplicate", 200),
|
|
track("ccccccccccc", "peppermint parkway", 300),
|
|
];
|
|
second_album.resize(PAGE_SIZE - 3, Value::Null);
|
|
accept(&mut releases, second_album, None);
|
|
|
|
let page = feed_results(&feed_output(releases), &requests).unwrap();
|
|
assert_eq!(page.feed_progress[0].count, PAGE_SIZE);
|
|
assert!(page.has_more);
|
|
assert_eq!(page.videos.len(), 2);
|
|
assert_eq!(page.videos[0].id, "ccccccccccc");
|
|
assert_eq!(page.videos[0].title, "peppermint parkway");
|
|
assert_eq!(page.videos[1].title, "Newer release duplicate");
|
|
assert!(
|
|
page.videos
|
|
.iter()
|
|
.all(|video| video.channel_id.as_deref() == Some(CHANNEL_ID))
|
|
);
|
|
cursor.accept(&page.feed_progress).unwrap();
|
|
assert_eq!(cursor.plan(1).unwrap()[0].start, PAGE_SIZE + 1);
|
|
}
|
|
|
|
#[test]
|
|
fn release_skip_subtracts_exhausted_album_totals_even_when_the_requested_start_is_past_end() {
|
|
for (skip, second_range) in [(3, (1, 2)), (5, (3, 4)), (8, (6, 7))] {
|
|
let mut releases = ChannelReleases::new(CHANNEL.into(), skip, 2);
|
|
accept(&mut releases, vec![album(1), album(2), album(3)], Some(3));
|
|
assert_eq!(range(&releases), (skip + 1, skip + 2));
|
|
// yt-dlp returns the exact album total even if the range starts beyond it.
|
|
accept(&mut releases, vec![], Some(3));
|
|
assert_eq!(range(&releases), second_range);
|
|
if skip == 8 {
|
|
accept(&mut releases, vec![], Some(4));
|
|
assert_eq!(range(&releases), (2, 3));
|
|
}
|
|
accept(
|
|
&mut releases,
|
|
vec![
|
|
track("ddddddddddd", "First selected track", 200),
|
|
track("eeeeeeeeeee", "Second selected track", 100),
|
|
],
|
|
None,
|
|
);
|
|
let request = super::super::FeedRequest {
|
|
channel: 0,
|
|
start: skip + 1,
|
|
count: 2,
|
|
};
|
|
let page = feed_results(&feed_output(releases), &[request]).unwrap();
|
|
assert_eq!(page.feed_progress[0].count, 2);
|
|
assert_eq!(
|
|
page.videos
|
|
.iter()
|
|
.map(|video| video.id.as_str())
|
|
.collect::<Vec<_>>(),
|
|
["ddddddddddd", "eeeeeeeeeee"]
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn discovery_continues_after_sixty_albums_and_keeps_later_tracks() {
|
|
let mut releases = ChannelReleases::new(CHANNEL.into(), 0, 2);
|
|
assert_eq!(range(&releases), (1, 60));
|
|
accept(&mut releases, (0..60).map(album).collect(), None);
|
|
for _ in 0..60 {
|
|
assert!(!releases.done());
|
|
accept(&mut releases, vec![], Some(0));
|
|
}
|
|
assert!(!releases.done());
|
|
assert_eq!(range(&releases), (61, 120));
|
|
accept(&mut releases, vec![album(60)], Some(61));
|
|
accept(
|
|
&mut releases,
|
|
vec![track("fffffffffff", "Track on album 61", 100)],
|
|
Some(1),
|
|
);
|
|
let request = super::super::FeedRequest {
|
|
channel: 0,
|
|
start: 1,
|
|
count: 2,
|
|
};
|
|
let page = feed_results(&feed_output(releases), &[request]).unwrap();
|
|
assert_eq!(page.videos.len(), 1);
|
|
assert_eq!(page.videos[0].title, "Track on album 61");
|
|
assert_eq!(page.feed_progress[0].count, 1);
|
|
assert!(!page.has_more);
|
|
}
|
|
|
|
#[test]
|
|
fn only_missing_releases_tab_during_discovery_is_normal_exhaustion() {
|
|
let missing = "ERROR: [youtube:tab] channel: This channel does not have a releases tab";
|
|
let mut ordinary = ChannelReleases::new(CHANNEL.into(), 0, 2);
|
|
ordinary.command().unwrap();
|
|
ordinary.accept_error(missing.into()).unwrap();
|
|
assert!(ordinary.done());
|
|
assert!(ordinary.into_entries().is_empty());
|
|
|
|
for error in [
|
|
"ERROR: network timeout",
|
|
"ERROR: This channel does not have a releases tab; retry later",
|
|
] {
|
|
let mut discovery = ChannelReleases::new(CHANNEL.into(), 0, 2);
|
|
discovery.command().unwrap();
|
|
assert_eq!(discovery.accept_error(error.into()).unwrap_err(), error);
|
|
}
|
|
for error in [missing, "ERROR: album is private"] {
|
|
let mut releases = ChannelReleases::new(CHANNEL.into(), 0, 2);
|
|
accept(&mut releases, vec![album(1)], Some(1));
|
|
releases.command().unwrap();
|
|
assert_eq!(releases.accept_error(error.into()).unwrap_err(), error);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn malformed_envelopes_and_exhaustion_counts_are_errors_not_empty_catalogs() {
|
|
for output in [
|
|
"not JSON",
|
|
"{}",
|
|
r#"{"entries":null,"playlist_count":0}"#,
|
|
r#"{"entries":[],"playlist_count":"0"}"#,
|
|
r#"{"entries":[],"playlist_count":-1}"#,
|
|
] {
|
|
let mut releases = ChannelReleases::new(CHANNEL.into(), 0, 2);
|
|
releases.command().unwrap();
|
|
assert!(
|
|
releases.accept(output).is_err(),
|
|
"accepted malformed discovery: {output}"
|
|
);
|
|
let mut releases = ChannelReleases::new(CHANNEL.into(), 0, 2);
|
|
accept(&mut releases, vec![album(1)], Some(1));
|
|
releases.command().unwrap();
|
|
assert!(
|
|
releases.accept(output).is_err(),
|
|
"accepted malformed album: {output}"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn empty_discovery_needs_no_total_but_skipping_an_empty_album_does() {
|
|
let mut discovery = ChannelReleases::new(CHANNEL.into(), 0, 2);
|
|
accept(&mut discovery, vec![], None);
|
|
assert!(discovery.done());
|
|
assert!(discovery.into_entries().is_empty());
|
|
|
|
let mut releases = ChannelReleases::new(CHANNEL.into(), 5, 2);
|
|
accept(&mut releases, vec![album(1), album(2)], Some(2));
|
|
releases.command().unwrap();
|
|
assert!(releases.accept(&response(vec![], None)).is_err());
|
|
}
|