use super::super::{FeedCursor, PAGE_SIZE, feed_results}; use super::*; use serde_json::{Value, json}; const CHANNEL: &str = "https://www.youtube.com/channel/UCaaaaaaaaaaaaaaaaaaaaaa"; const CHANNEL_ID: &str = "UCaaaaaaaaaaaaaaaaaaaaaa"; fn album(index: usize) -> Value { json!({"id": format!("PL{index:032}"), "_type": "url"}) } fn track(id: &str, title: &str, timestamp: i64) -> Value { json!({"id": id, "title": title, "timestamp": timestamp}) } fn response(entries: Vec, total: Option) -> String { json!({"entries": entries, "playlist_count": total}).to_string() } fn accept(releases: &mut ChannelReleases, entries: Vec, total: Option) { // Construct, but never execute, the next extractor request. releases.command().unwrap(); releases.accept(&response(entries, total)).unwrap(); } fn range(releases: &ChannelReleases) -> (usize, usize) { let command = releases.command().unwrap(); let args: Vec<_> = command.get_args().collect(); let value = |flag: &str| { args.windows(2).find(|pair| pair[0] == flag).unwrap()[1] .to_str() .unwrap() .parse::() .unwrap() }; (value("--playlist-start"), value("--playlist-end")) } fn feed_output(releases: ChannelReleases) -> String { assert!(releases.done()); let entries = releases.into_entries(); json!({"channel_id": CHANNEL_ID, "entries": entries}).to_string() } #[test] fn mixed_feed_merges_uploads_and_release_tracks_without_advancing_other_channels() { let mut cursor = FeedCursor::default(); let requests = cursor.plan(2).unwrap(); let channels = vec![CHANNEL.into(), "https://www.youtube.com/@ordinary".into()]; let mut expansion = FeedExpansion::new(&channels, &requests); let uploads = json!({ "channel_id": CHANNEL_ID, "uploader_id": "@artist", "playlist_count": 3, "entries": [ track("aaaaaaaaaaa", "Upload", 100), track("ddddddddddd", "Second upload", 100), track("eeeeeeeeeee", "Third upload", 100) ] }); let ordinary = json!({ "entries": (0..30).map(|index| { track(&format!("b{index:010}"), "Ordinary upload", 50) }).collect::>() }); expansion .accept_uploads(&format!("{uploads}\n{ordinary}")) .unwrap(); let releases = expansion.channels[0].releases.as_mut().unwrap(); accept(releases, vec![album(1)], Some(1)); let mut tracks = vec![ track("aaaaaaaaaaa", "Release duplicate", 200), track("ccccccccccc", "peppermint parkway", 300), ]; tracks.resize(27, Value::Null); accept(releases, tracks, Some(27)); let mut finished_uploads = Job::spawn_all(std::iter::empty::()).unwrap(); let output = expansion.poll(&mut finished_uploads).unwrap().unwrap(); let page = feed_results(&output, &requests).unwrap(); assert_eq!( page.feed_progress .iter() .map(|p| p.count) .collect::>(), [30, 30] ); assert_eq!(page.videos.len(), 34); assert_eq!(page.videos[0].id, "ccccccccccc"); assert_eq!(page.videos[0].channel_id.as_deref(), Some(CHANNEL_ID)); assert_eq!(page.videos[0].uploader_id.as_deref(), Some("@artist")); assert_eq!(page.videos[1].title, "Release duplicate"); cursor.accept(&page.feed_progress).unwrap(); assert!( cursor .plan(2) .unwrap() .iter() .all(|request| request.start == 31) ); } #[test] fn release_tracks_reach_the_feed_without_refunding_filtered_or_duplicate_rows() { let mut cursor = FeedCursor::default(); let requests = cursor.plan(1).unwrap(); let mut releases = ChannelReleases::new(CHANNEL.into(), 0, requests[0].count); accept(&mut releases, vec![album(1), album(2)], Some(2)); accept( &mut releases, vec![ track("aaaaaaaaaaa", "Older upload duplicate", 100), json!({"id": "bbbbbbbbbbb", "title": "Members", "availability": "subscriber_only"}), json!({"id": "invalid", "title": "Invalid video ID"}), ], Some(3), ); assert!(!releases.done()); assert_eq!(range(&releases), (1, PAGE_SIZE - 3)); let mut second_album = vec![ track("aaaaaaaaaaa", "Newer release duplicate", 200), track("ccccccccccc", "peppermint parkway", 300), ]; second_album.resize(PAGE_SIZE - 3, Value::Null); accept(&mut releases, second_album, None); let page = feed_results(&feed_output(releases), &requests).unwrap(); assert_eq!(page.feed_progress[0].count, PAGE_SIZE); assert!(page.has_more); assert_eq!(page.videos.len(), 2); assert_eq!(page.videos[0].id, "ccccccccccc"); assert_eq!(page.videos[0].title, "peppermint parkway"); assert_eq!(page.videos[1].title, "Newer release duplicate"); assert!( page.videos .iter() .all(|video| video.channel_id.as_deref() == Some(CHANNEL_ID)) ); cursor.accept(&page.feed_progress).unwrap(); assert_eq!(cursor.plan(1).unwrap()[0].start, PAGE_SIZE + 1); } #[test] fn release_skip_subtracts_exhausted_album_totals_even_when_the_requested_start_is_past_end() { for (skip, second_range) in [(3, (1, 2)), (5, (3, 4)), (8, (6, 7))] { let mut releases = ChannelReleases::new(CHANNEL.into(), skip, 2); accept(&mut releases, vec![album(1), album(2), album(3)], Some(3)); assert_eq!(range(&releases), (skip + 1, skip + 2)); // yt-dlp returns the exact album total even if the range starts beyond it. accept(&mut releases, vec![], Some(3)); assert_eq!(range(&releases), second_range); if skip == 8 { accept(&mut releases, vec![], Some(4)); assert_eq!(range(&releases), (2, 3)); } accept( &mut releases, vec![ track("ddddddddddd", "First selected track", 200), track("eeeeeeeeeee", "Second selected track", 100), ], None, ); let request = super::super::FeedRequest { channel: 0, start: skip + 1, count: 2, }; let page = feed_results(&feed_output(releases), &[request]).unwrap(); assert_eq!(page.feed_progress[0].count, 2); assert_eq!( page.videos .iter() .map(|video| video.id.as_str()) .collect::>(), ["ddddddddddd", "eeeeeeeeeee"] ); } } #[test] fn discovery_continues_after_sixty_albums_and_keeps_later_tracks() { let mut releases = ChannelReleases::new(CHANNEL.into(), 0, 2); assert_eq!(range(&releases), (1, 60)); accept(&mut releases, (0..60).map(album).collect(), None); for _ in 0..60 { assert!(!releases.done()); accept(&mut releases, vec![], Some(0)); } assert!(!releases.done()); assert_eq!(range(&releases), (61, 120)); accept(&mut releases, vec![album(60)], Some(61)); accept( &mut releases, vec![track("fffffffffff", "Track on album 61", 100)], Some(1), ); let request = super::super::FeedRequest { channel: 0, start: 1, count: 2, }; let page = feed_results(&feed_output(releases), &[request]).unwrap(); assert_eq!(page.videos.len(), 1); assert_eq!(page.videos[0].title, "Track on album 61"); assert_eq!(page.feed_progress[0].count, 1); assert!(!page.has_more); } #[test] fn only_missing_releases_tab_during_discovery_is_normal_exhaustion() { let missing = "ERROR: [youtube:tab] channel: This channel does not have a releases tab"; let mut ordinary = ChannelReleases::new(CHANNEL.into(), 0, 2); ordinary.command().unwrap(); ordinary.accept_error(missing.into()).unwrap(); assert!(ordinary.done()); assert!(ordinary.into_entries().is_empty()); for error in [ "ERROR: network timeout", "ERROR: This channel does not have a releases tab; retry later", ] { let mut discovery = ChannelReleases::new(CHANNEL.into(), 0, 2); discovery.command().unwrap(); assert_eq!(discovery.accept_error(error.into()).unwrap_err(), error); } for error in [missing, "ERROR: album is private"] { let mut releases = ChannelReleases::new(CHANNEL.into(), 0, 2); accept(&mut releases, vec![album(1)], Some(1)); releases.command().unwrap(); assert_eq!(releases.accept_error(error.into()).unwrap_err(), error); } } #[test] fn malformed_envelopes_and_exhaustion_counts_are_errors_not_empty_catalogs() { for output in [ "not JSON", "{}", r#"{"entries":null,"playlist_count":0}"#, r#"{"entries":[],"playlist_count":"0"}"#, r#"{"entries":[],"playlist_count":-1}"#, ] { let mut releases = ChannelReleases::new(CHANNEL.into(), 0, 2); releases.command().unwrap(); assert!( releases.accept(output).is_err(), "accepted malformed discovery: {output}" ); let mut releases = ChannelReleases::new(CHANNEL.into(), 0, 2); accept(&mut releases, vec![album(1)], Some(1)); releases.command().unwrap(); assert!( releases.accept(output).is_err(), "accepted malformed album: {output}" ); } } #[test] fn empty_discovery_needs_no_total_but_skipping_an_empty_album_does() { let mut discovery = ChannelReleases::new(CHANNEL.into(), 0, 2); accept(&mut discovery, vec![], None); assert!(discovery.done()); assert!(discovery.into_entries().is_empty()); let mut releases = ChannelReleases::new(CHANNEL.into(), 5, 2); accept(&mut releases, vec![album(1), album(2)], Some(2)); releases.command().unwrap(); assert!(releases.accept(&response(vec![], None)).is_err()); }