806 lines
47 KiB
Swift
806 lines
47 KiB
Swift
import Foundation
|
|
import Darwin
|
|
|
|
@main
|
|
struct YouTubeResolverChecks {
|
|
struct Failure: Error, CustomStringConvertible {
|
|
let description: String
|
|
}
|
|
|
|
static let page = URL(string: "https://www.youtube.com/watch?v=jNQXAC9IVRw")!
|
|
static let splitJSON = #"""
|
|
{
|
|
"title": "Separate streams", "duration": 19.25, "is_live": false,
|
|
"http_headers": {"Referer": "https://www.youtube.com/", "User-Agent": "metadata-agent"},
|
|
"requested_formats": [
|
|
{"url": "https://media.example/video?token=video", "vcodec": "av01.0.04M.08", "acodec": "none", "dynamic_range": "SDR", "http_headers": {"User-Agent": "video-agent", "X-Input": "video"}},
|
|
{"url": "https://media.example/audio?token=audio", "vcodec": "none", "acodec": "opus"}
|
|
]
|
|
}
|
|
"""#
|
|
static let combinedJSON = #"""
|
|
{"title": "Combined stream", "duration": 12, "is_live": false,
|
|
"url": "https://media.example/combined", "vcodec": "h264", "acodec": "aac",
|
|
"http_headers": {"User-Agent": "combined-agent"}}
|
|
"""#
|
|
|
|
static func require(_ condition: @autoclosure () throws -> Bool, _ message: String) throws {
|
|
if try condition() == false { throw Failure(description: "FAIL: \(message)") }
|
|
}
|
|
|
|
static func reject(_ message: String, _ operation: () throws -> Void) throws {
|
|
do { try operation() }
|
|
catch is Failure { throw Failure(description: "FAIL: \(message) assertion failed") }
|
|
catch { return }
|
|
throw Failure(description: "FAIL: \(message) was accepted")
|
|
}
|
|
|
|
static func decode(_ json: String) throws -> YouTubeVideo {
|
|
try YouTubeResolver.decode(Data(json.utf8), webpageURL: page)
|
|
}
|
|
|
|
static func decode(_ object: [String: Any]) throws -> YouTubeVideo {
|
|
try YouTubeResolver.decode(JSONSerialization.data(withJSONObject: object), webpageURL: page)
|
|
}
|
|
|
|
static func fixture(_ json: String) throws -> [String: Any] {
|
|
try JSONSerialization.jsonObject(with: Data(json.utf8)) as! [String: Any]
|
|
}
|
|
|
|
static func headerLines(_ text: String) throws -> Set<String> {
|
|
try require(text.isEmpty || text.hasSuffix("\r\n"), "headers terminate with CRLF")
|
|
let lines = text.components(separatedBy: "\r\n").filter { !$0.isEmpty }
|
|
try require(lines.allSatisfy { !$0.contains("\r") && !$0.contains("\n") },
|
|
"headers contain no bare CR or LF")
|
|
return Set(lines)
|
|
}
|
|
|
|
static func videoURLs() throws {
|
|
let valid = [
|
|
"https://youtube.com/watch?v=jNQXAC9IVRw",
|
|
"https://www.youtube.com/watch?v=jNQXAC9IVRw&list=PLignored&t=3",
|
|
"https://m.youtube.com/watch?v=jNQXAC9IVRw",
|
|
"https://music.youtube.com/watch?v=jNQXAC9IVRw",
|
|
"https://youtu.be/jNQXAC9IVRw?si=share",
|
|
"https://www.youtube.com/shorts/jNQXAC9IVRw",
|
|
"https://www.youtube.com/live/jNQXAC9IVRw",
|
|
"https://www.youtube.com/embed/jNQXAC9IVRw"
|
|
]
|
|
for input in valid {
|
|
let url = try YouTubeResolver.validateURL(input)
|
|
try require(url == page, "supported video URLs identify the same video")
|
|
}
|
|
for input in ["https://vimeo.com/76979871", "https://example.com/watch?v=jNQXAC9IVRw",
|
|
"https://youtube.com.evil.example/watch?v=jNQXAC9IVRw",
|
|
"https://evil.youtube.com/watch?v=jNQXAC9IVRw",
|
|
"http://127.0.0.1:8080/video.mp4?token=kept#fragment",
|
|
"https://www.youtube.com/playlist?list=PLexample"] {
|
|
try require(try YouTubeResolver.validateURL(input).absoluteString == input,
|
|
"HTTP(S) URLs outside canonical YouTube video forms reach yt-dlp unchanged")
|
|
}
|
|
let hostile = [
|
|
"", "not a URL", "javascript:alert(1)", "file:///watch?v=jNQXAC9IVRw",
|
|
"ftp://example.com/video", "https://", "https://example.com:0/video",
|
|
"https://example.com:65536/video", "https://example.com/video\ninjected",
|
|
"https://youtube.com@evil.example/watch?v=jNQXAC9IVRw",
|
|
"https://user:password@www.youtube.com/watch?v=jNQXAC9IVRw"
|
|
]
|
|
for input in hostile {
|
|
try reject("hostile or malformed video URL") { _ = try YouTubeResolver.validateURL(input) }
|
|
}
|
|
let asciiID = try YouTubeResolver.validateURL("https://youtu.be/Abc_123-XYZ")
|
|
try require(asciiID.absoluteString.hasSuffix("v=Abc_123-XYZ"), "video IDs retain case, underscore and hyphen")
|
|
}
|
|
|
|
static func metadata() throws {
|
|
let split = try decode(splitJSON)
|
|
try require(split.title == "Separate streams" && split.duration == 19.25 && !split.isLive,
|
|
"split streams preserve title, fractional duration and recorded status")
|
|
try require(split.webpageURL == page, "resolved media retains its webpage URL")
|
|
try require(split.video.url == "https://media.example/video?token=video"
|
|
&& split.audio?.url == "https://media.example/audio?token=audio",
|
|
"video and audio inputs remain distinct and preserve signed query strings")
|
|
let videoHeaders = try headerLines(split.video.headers)
|
|
let audioHeaders = try headerLines(split.audio!.headers)
|
|
try require(videoHeaders == ["User-Agent: video-agent", "X-Input: video"],
|
|
"selected video headers take precedence as a complete map")
|
|
try require(audioHeaders == ["Referer: https://www.youtube.com/", "User-Agent: metadata-agent"],
|
|
"audio without selected headers inherits metadata headers")
|
|
|
|
var splitObject = try fixture(splitJSON)
|
|
var formats = splitObject["requested_formats"] as! [[String: Any]]
|
|
formats.reverse()
|
|
formats[0]["http_headers"] = ["X-Input": "audio"]
|
|
formats[1]["http_headers"] = NSNull()
|
|
splitObject["requested_formats"] = formats
|
|
let reversed = try decode(splitObject)
|
|
let reversedVideoHeaders = try headerLines(reversed.video.headers)
|
|
let reversedAudioHeaders = try headerLines(reversed.audio!.headers)
|
|
try require(reversed.video.url == split.video.url && reversed.audio?.url == split.audio?.url,
|
|
"stream roles depend on codecs rather than array order")
|
|
try require(reversedVideoHeaders == audioHeaders && reversedAudioHeaders == ["X-Input: audio"],
|
|
"each input independently selects its own headers or metadata fallback")
|
|
|
|
let combined = try decode(combinedJSON)
|
|
try require(combined.video.url == "https://media.example/combined" && combined.audio == nil,
|
|
"combined fallback uses one input, not duplicated audio")
|
|
try require(combined.title == "Combined stream" && combined.duration == 12 && !combined.isLive,
|
|
"combined metadata is preserved")
|
|
let combinedHeaders = try headerLines(combined.video.headers)
|
|
try require(combinedHeaders == ["User-Agent: combined-agent"], "combined input carries headers")
|
|
|
|
var selectedCombined = try fixture(combinedJSON)
|
|
selectedCombined.removeValue(forKey: "url")
|
|
selectedCombined["requested_formats"] = [
|
|
["url": "https://media.example/selected-combined", "vcodec": "h264", "acodec": "aac",
|
|
"http_headers": ["X-Input": "combined"]]
|
|
]
|
|
let selected = try decode(selectedCombined)
|
|
let selectedHeaders = try headerLines(selected.video.headers)
|
|
try require(selected.audio == nil && selected.video.url == "https://media.example/selected-combined"
|
|
&& selectedHeaders == ["X-Input: combined"],
|
|
"a selected combined format is one playable input with its own headers")
|
|
|
|
var hls = try fixture(combinedJSON)
|
|
hls["url"] = "https://media.example/live.m3u8"
|
|
hls["protocol"] = "m3u8_native"
|
|
let playlist = try decode(hls)
|
|
try require(playlist.video.url == "https://media.example/live.m3u8" && playlist.audio == nil,
|
|
"native HLS remains a usable streaming playlist, not a downloaded file")
|
|
|
|
var live = try fixture(combinedJSON)
|
|
live["is_live"] = true
|
|
live["duration"] = NSNull()
|
|
let directLive = try decode(live)
|
|
try require(directLive.isLive && directLive.duration == 0, "live stream permits unknown duration")
|
|
live.removeValue(forKey: "is_live")
|
|
live.removeValue(forKey: "duration")
|
|
live["live_status"] = "is_live"
|
|
try require(try decode(live).isLive, "live_status identifies live broadcasts")
|
|
live["live_status"] = "was_live"
|
|
try require(!(try decode(live)).isLive, "completed live broadcasts are recorded media")
|
|
live["live_status"] = "is_upcoming"
|
|
try reject("upcoming broadcast with no playable live content") { _ = try decode(live) }
|
|
|
|
var emptyHeaders = try fixture(combinedJSON)
|
|
emptyHeaders.removeValue(forKey: "http_headers")
|
|
try require(try decode(emptyHeaders).video.headers.isEmpty, "absent headers serialize to an empty block")
|
|
}
|
|
|
|
static func subtitles() throws {
|
|
func vtt(_ url: String, name: Any = "English", headers: Any? = nil) -> [String: Any] {
|
|
var row: [String: Any] = ["ext": "vtt", "url": url, "name": name]
|
|
if let headers { row["http_headers"] = headers }
|
|
return row
|
|
}
|
|
let original = "https://www.youtube.com/api/timedtext?lang=en&kind=asr&fmt=vtt"
|
|
var object = try fixture(combinedJSON)
|
|
try require(try decode(object).subtitles.isEmpty, "absent captions default to Off with no optional tracks")
|
|
for payload in [NSNull(), [], true, 17, "captions"] as [Any] {
|
|
object["subtitles"] = payload
|
|
object["automatic_captions"] = payload
|
|
try require(try decode(object).subtitles.isEmpty,
|
|
"malformed optional caption containers never block playback")
|
|
}
|
|
object["subtitles"] = [
|
|
"broken": true,
|
|
"en": [NSNull(), 7, ["ext": false], ["ext": "vtt", "url": 9],
|
|
["ext": "srt", "url": "https://subs.example/convert-me"],
|
|
vtt("file:///unsafe.vtt"),
|
|
vtt("https://subs.example/unsafe.vtt", headers: ["X-Unsafe": "bad\nvalue"]),
|
|
vtt("https://subs.example/bad-headers.vtt", headers: ["X-Bad": 3]),
|
|
vtt("https://subs.example/en.vtt?token=first", headers: ["X-Track": "manual"]),
|
|
vtt("https://subs.example/duplicate.vtt")],
|
|
"fr": [vtt("https://subs.example/fr.vtt", name: ["malformed"])],
|
|
"srt-only": [["ext": "srt", "url": "https://subs.example/only.srt"]]
|
|
] as [String: Any]
|
|
object["automatic_captions"] = [
|
|
"en-orig": [vtt(original, headers: ["X-Track": "auto"])],
|
|
"en": [vtt(original)],
|
|
"fr": [vtt(original + "&tlang=fr", name: "French")],
|
|
"de": [vtt("https://subs.example/de.vtt", name: "German (Original)")],
|
|
"bad-orig": [NSNull(), vtt("ftp://subs.example/bad.vtt")]
|
|
] as [String: Any]
|
|
let tracks = try decode(object).subtitles
|
|
try require(tracks.count == 3 && tracks.map(\.language) == ["en", "fr", "en"],
|
|
"manual VTT and one original auto survive malformed siblings; translations and aliases do not")
|
|
try require(tracks.map(\.selectionID) == ["youtube:manual:en", "youtube:manual:fr", "youtube:auto:en"],
|
|
"manual and auto identities depend on language metadata, not signed URLs or labels")
|
|
try require(tracks[0].url == "https://subs.example/en.vtt?token=first"
|
|
&& tracks[0].label == "English" && tracks[1].label == "fr"
|
|
&& tracks[2].label == "English (auto-generated)",
|
|
"one valid supplied VTT per language preserves name and query; malformed name falls back to language")
|
|
try require(tracks.allSatisfy { !$0.isDefault } && ResolvedSource.validSubtitles(tracks),
|
|
"optional captions preserve transport safeguards and never enable themselves")
|
|
try require(tracks[0].headers == "X-Track: manual\r\n"
|
|
&& tracks[1].headers == "User-Agent: combined-agent\r\n"
|
|
&& tracks[2].headers == "X-Track: auto\r\n",
|
|
"variant headers override the complete root map; absent variant headers inherit root")
|
|
var changed = object
|
|
changed["automatic_captions"] = ["en": [vtt(original + "&token=renewed", name: "Renamed")]]
|
|
let alias = try decode(changed).subtitles.last!
|
|
try require(alias.selectionID == tracks.last?.selectionID && alias.url != tracks.last?.url,
|
|
"explicit untranslated ASR alias retains original identity after signed URL and label refresh")
|
|
changed["automatic_captions"] = [
|
|
"en-orig": [vtt(original)],
|
|
"en": [vtt(original + "&token=alias")],
|
|
"other-orig": [vtt(original)]
|
|
]
|
|
try require(try decode(changed).subtitles.filter { $0.selectionID?.hasPrefix("youtube:auto:") == true }.count == 1,
|
|
"original auto aliases deduplicate by language and by URL")
|
|
changed["automatic_captions"] = ["en-orig": [vtt(original)]]
|
|
changed["subtitles"] = ["en": [vtt("https://subs.example/empty.vtt", headers: [:])]]
|
|
try require(try decode(changed).subtitles.first?.headers == "",
|
|
"an explicitly empty caption header map does not inherit root headers")
|
|
|
|
for unsafe in ["https://user:pass@subs.example/c.vtt", "https://subs.example/c.vtt#fragment",
|
|
"https://subs.example:0/c.vtt", "https://subs.example/c vtt"] {
|
|
changed["subtitles"] = ["en": [vtt(unsafe)]]
|
|
try require(try decode(changed).subtitles.count == 1,
|
|
"unsafe optional subtitle URL is skipped without discarding a good auto sibling")
|
|
}
|
|
var fragmented = vtt("https://subs.example/chunks")
|
|
fragmented["protocol"] = "m3u8_native"
|
|
changed["subtitles"] = ["en": [fragmented]]
|
|
try require(try decode(changed).subtitles.count == 1,
|
|
"caption playlists are not treated as immutable VTT documents")
|
|
|
|
var many: [String: Any] = [:]
|
|
for index in 0..<40 {
|
|
let key = String(format: "lang-%02d", index)
|
|
many[key] = [vtt("https://subs.example/\(key).vtt", name: key)]
|
|
}
|
|
changed["subtitles"] = many
|
|
let capped = try decode(changed).subtitles
|
|
try require(capped.count == 32 && capped.last?.selectionID == "youtube:auto:en"
|
|
&& capped.first?.language == "lang-00" && capped[30].language == "lang-30",
|
|
"deterministic 32-track cap reserves original auto instead of crowding it out with manual languages")
|
|
changed["automatic_captions"] = [:]
|
|
try require(try decode(changed).subtitles.count == 32,
|
|
"manual-only metadata uses the entire track allowance")
|
|
changed["automatic_captions"] = ["en-orig": [vtt(original + "&tlang=fr")]]
|
|
let translated = try decode(changed).subtitles
|
|
try require(translated.count == 32
|
|
&& translated.allSatisfy { $0.selectionID?.hasPrefix("youtube:manual:") == true },
|
|
"translation query metadata is excluded even under an original-looking key")
|
|
for (key, value) in [("is_live", true), ("live_status", "is_live")] as [(String, Any)] {
|
|
var broadcast = object
|
|
broadcast[key] = value
|
|
try require(try decode(broadcast).subtitles.isEmpty,
|
|
"live captions are disabled instead of freezing a partial live transcript")
|
|
}
|
|
changed["url"] = "file:///unsafe-video"
|
|
try reject("optional caption tolerance does not weaken video validation") { _ = try decode(changed) }
|
|
}
|
|
|
|
static func chapters() throws {
|
|
var object = try fixture(combinedJSON)
|
|
try require(try decode(object).chapters.isEmpty, "absent chapters remain playable")
|
|
for payload in [NSNull(), [], true, 17, "chapters", [:],
|
|
[NSNull(), 17, "row", [:], ["start_time": 0],
|
|
["start_time": "0", "end_time": 2], ["start_time": 0, "end_time": false]]] as [Any] {
|
|
object["chapters"] = payload
|
|
let video = try decode(object)
|
|
try require(video.chapters.isEmpty && video.video.url == "https://media.example/combined",
|
|
"null, empty and malformed optional chapters never prevent playback")
|
|
}
|
|
object["chapters"] = [
|
|
NSNull(),
|
|
["start_time": 0, "end_time": 2.25, "title": "Opening"],
|
|
["start_time": 2.25, "end_time": 8.75],
|
|
["start_time": 8.75, "end_time": 12, "title": ["malformed"]],
|
|
["start_time": 2.25, "end_time": 8.75, "title": NSNull()]
|
|
] as [Any]
|
|
let decoded = try decode(object)
|
|
try require(decoded.chapters.count == 4 && decoded.chapters[0].title == "Opening"
|
|
&& decoded.chapters[0].end == 2.25 && decoded.chapters[1].start == 2.25,
|
|
"fractional chapters and titles survive malformed sibling rows")
|
|
try require(decoded.chapters.dropFirst().allSatisfy { $0.title == nil },
|
|
"missing, null and malformed optional titles do not discard chapters")
|
|
try require(decoded.chapterBoundaries(duration: 12) == [2.25, 8.75],
|
|
"chapter edges are sorted, deduplicated and strictly interior")
|
|
try require(decoded.chapterTitle(at: 0, duration: 12) == "Opening" &&
|
|
decoded.chapterTitle(at: 2.249, duration: 12) == "Opening" &&
|
|
decoded.chapterTitle(at: 2.25, duration: 12) == nil &&
|
|
decoded.chapterTitle(at: 12, duration: 12) == nil,
|
|
"hover titles include chapter start, exclude its end and never invent missing titles")
|
|
for duration in [0, -1, 1, Double.nan, .infinity] {
|
|
try require(decoded.chapterTitle(at: 0.5, duration: duration) == nil,
|
|
"unknown durations and chapters extending beyond media have no hover title")
|
|
}
|
|
|
|
object["duration"] = 0
|
|
object["chapters"] = [["start_time": 0, "end_time": 2.25],
|
|
["start_time": 2.25, "end_time": 20.5]]
|
|
let unknownDuration = try decode(object)
|
|
try require(unknownDuration.chapterBoundaries(duration: 20.5) == [2.25],
|
|
"actual playback duration, not extractor duration, validates chapters")
|
|
for duration in [0, -1, Double.nan, .infinity, -.infinity] {
|
|
try require(unknownDuration.chapterBoundaries(duration: duration).isEmpty,
|
|
"unknown and invalid playback durations never produce boundaries")
|
|
}
|
|
|
|
let video = YouTubeVideo(title: "Chapters",
|
|
video: .init(url: "https://media.example/combined", headers: ""), audio: nil,
|
|
duration: 0, isLive: false, webpageURL: page, height: nil, availableHeights: [],
|
|
chapters: [
|
|
.init(start: 8.75, end: 12, title: nil),
|
|
.init(start: 0, end: 2.25, title: nil),
|
|
.init(start: 2.25, end: 8.75, title: nil),
|
|
.init(start: 2.25, end: 8.75, title: nil),
|
|
.init(start: -1, end: 4, title: nil),
|
|
.init(start: 3, end: 13, title: nil),
|
|
.init(start: 6, end: 6, title: nil),
|
|
.init(start: 7, end: 5, title: nil),
|
|
.init(start: .nan, end: 4, title: nil),
|
|
.init(start: .infinity, end: 4, title: nil),
|
|
.init(start: -.infinity, end: 4, title: nil),
|
|
.init(start: 3, end: .nan, title: nil),
|
|
.init(start: 3, end: .infinity, title: nil),
|
|
.init(start: 3, end: -.infinity, title: nil)
|
|
])
|
|
try require(video.chapterBoundaries(duration: 12) == [2.25, 8.75],
|
|
"invalid, zero-length, reversed and over-range rows contribute neither edge")
|
|
object["chapters"] = "malformed optional data"
|
|
object["url"] = 17
|
|
try reject("chapter tolerance must not weaken required stream validation") { _ = try decode(object) }
|
|
}
|
|
|
|
static func qualities() throws {
|
|
func video(_ height: Int, audio: Bool = false, range: String = "SDR", transport: String = "https") -> [String: Any] {
|
|
["height": height, "vcodec": "h264", "acodec": audio ? "aac" : "none",
|
|
"dynamic_range": range, "protocol": transport, "url": "https://media.example/\(height)p"]
|
|
}
|
|
let audio: [String: Any] = ["vcodec": "none", "acodec": "aac", "url": "https://media.example/audio"]
|
|
var object: [String: Any] = [
|
|
"requested_formats": [video(1080), audio],
|
|
"formats": [video(2160), video(1440), video(1080), video(1080, audio: true), video(720),
|
|
video(480), video(4320, range: "HDR10"), video(360, transport: "http_dash_segments"),
|
|
video(0), audio]
|
|
]
|
|
let normal = try decode(object)
|
|
try require(normal.height == 1080 && normal.availableHeights == [2160, 1440, 1080, 720, 480],
|
|
"quality choices are unique playable SDR resolutions, including manual choices above the default cap")
|
|
object["requested_formats"] = [video(720), audio]
|
|
try require(try decode(object).height == 720, "a lower available resolution is accepted by the default policy")
|
|
object["requested_formats"] = [video(2160), audio]
|
|
try reject("4K stream selected by the default policy") { _ = try decode(object) }
|
|
let data = try JSONSerialization.data(withJSONObject: object)
|
|
let high = try YouTubeResolver.decode(data, webpageURL: page, height: 2160)
|
|
try require(high.height == 2160 && high.video.url == "https://media.example/2160p",
|
|
"explicit selection permits supported 4K video")
|
|
try reject("requested resolution silently replaced by another") {
|
|
_ = try YouTubeResolver.decode(data, webpageURL: page, height: 1440)
|
|
}
|
|
object["requested_formats"] = [video(720, audio: true)]
|
|
object["formats"] = [video(2160), video(720, audio: true)]
|
|
try require(try decode(object).availableHeights == [720],
|
|
"video-only resolutions without a playable separate audio stream are not offered")
|
|
object["requested_formats"] = [video(-720, audio: true)]
|
|
try reject("invalid selected video dimensions") { _ = try decode(object) }
|
|
}
|
|
|
|
static func malformedMetadata() throws {
|
|
var collection = try fixture(combinedJSON)
|
|
collection["_type"] = "playlist"
|
|
try reject("collection result instead of a single video") { _ = try decode(collection) }
|
|
try reject("non-JSON extractor output") { _ = try decode("not JSON") }
|
|
for json in ["null", "[]", "{}", #"{"requested_formats":[]}"#] {
|
|
try reject("missing playable stream metadata") { _ = try decode(json) }
|
|
}
|
|
try reject("nonfinite duration") {
|
|
_ = try decode(#"{"url":"https://media.example/video","vcodec":"h264","acodec":"aac","duration":1e999}"#)
|
|
}
|
|
let combined = try fixture(combinedJSON)
|
|
var missingURL = combined
|
|
missingURL.removeValue(forKey: "url")
|
|
try reject("missing combined stream URL") { _ = try decode(missingURL) }
|
|
for key in ["url", "vcodec", "acodec"] {
|
|
var object = combined
|
|
object[key] = 17
|
|
try reject("wrong type for combined \(key)") { _ = try decode(object) }
|
|
}
|
|
for url in ["", "https://", "file:///tmp/video", "ftp://media.example/video",
|
|
"data:video/mp4;base64,AA==", "https://media.example/video\r\nX-Injected: yes"] {
|
|
var object = combined
|
|
object["url"] = url
|
|
try reject("invalid or unsafe stream URL") { _ = try decode(object) }
|
|
}
|
|
for duration in [-1, "19", true] as [Any] {
|
|
var object = combined
|
|
object["duration"] = duration
|
|
try reject("invalid duration metadata") { _ = try decode(object) }
|
|
}
|
|
for key in ["is_live", "title"] {
|
|
var object = combined
|
|
object[key] = ["malformed"]
|
|
try reject("malformed \(key) metadata") { _ = try decode(object) }
|
|
}
|
|
var hdr = combined
|
|
hdr["dynamic_range"] = "HDR10"
|
|
try reject("HDR input unsupported by the SDR renderer") { _ = try decode(hdr) }
|
|
var audioOnly = combined
|
|
audioOnly["vcodec"] = "none"
|
|
try reject("explicitly audio-only result") { _ = try decode(audioOnly) }
|
|
for transport in ["http_dash_segments", "mhtml", "rtmp"] {
|
|
var object = combined
|
|
object["protocol"] = transport
|
|
try reject("fragment assembly or unsupported transport is not a whole playable input") {
|
|
_ = try decode(object)
|
|
}
|
|
}
|
|
|
|
let split = try fixture(splitJSON)
|
|
let formats = split["requested_formats"] as! [[String: Any]]
|
|
for malformed in ["bad", [formats[0]], [formats[1]], [formats[0], formats[0]],
|
|
[formats[0], formats[1], formats[1]]] as [Any] {
|
|
var object = split
|
|
object["requested_formats"] = malformed
|
|
try reject("malformed or ambiguous split stream selection") { _ = try decode(object) }
|
|
}
|
|
var missingAudio = split
|
|
var badFormats = formats
|
|
badFormats[1].removeValue(forKey: "url")
|
|
missingAudio["requested_formats"] = badFormats
|
|
try reject("missing separate audio URL is not silently downgraded") { _ = try decode(missingAudio) }
|
|
|
|
let badHeaders: [Any] = [
|
|
"not a header map", ["User-Agent": 5], ["Bad Name": "value"], ["Bad:Name": "value"],
|
|
["Bad\r\nName": "value"], ["": "value"], ["X-Header": "a\r\nInjected: yes"],
|
|
["X-Header": "a\nb"], ["X-Header": "a\rb"], ["X-Header": "a\u{0}b"],
|
|
["X-Header": "a\t b"], ["X-Header": "a\u{7f}b"]
|
|
]
|
|
for headers in badHeaders {
|
|
var object = combined
|
|
object["http_headers"] = headers
|
|
try reject("unsafe or malformed metadata headers") { _ = try decode(object) }
|
|
var perInput = split
|
|
var selected = formats
|
|
selected[1]["http_headers"] = headers
|
|
perInput["requested_formats"] = selected
|
|
try reject("unsafe or malformed selected audio headers") { _ = try decode(perInput) }
|
|
}
|
|
}
|
|
|
|
struct Extractor: Sendable {
|
|
let executable: URL
|
|
let directory: URL
|
|
|
|
init(root: URL, name: String, json: String = YouTubeResolverChecks.splitJSON,
|
|
body: String = #"/bin/cat "$base/response.json""#) throws {
|
|
directory = root.appendingPathComponent(name, isDirectory: true)
|
|
try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: false)
|
|
executable = directory.appendingPathComponent("fake-extractor")
|
|
let script = "#!/bin/sh\nset -eu\nbase=${0%/*}\n: > \"$base/started\"\n" + body + "\n"
|
|
try Data(script.utf8).write(to: executable)
|
|
try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: executable.path)
|
|
try Data(json.utf8).write(to: directory.appendingPathComponent("response.json"))
|
|
}
|
|
|
|
func marker(_ name: String) -> URL { directory.appendingPathComponent(name) }
|
|
}
|
|
|
|
static func awaitMarker(_ extractor: Extractor, _ name: String = "started") async throws {
|
|
for _ in 0..<400 {
|
|
if FileManager.default.fileExists(atPath: extractor.marker(name).path) { return }
|
|
try await Task.sleep(for: .milliseconds(5))
|
|
}
|
|
throw Failure(description: "FAIL: extractor did not reach \(name) within the isolated check deadline")
|
|
}
|
|
|
|
static func cancelled<T: Sendable>(_ task: Task<T, Error>, _ message: String) async throws {
|
|
do { _ = try await task.value }
|
|
catch is CancellationError { return }
|
|
catch { throw Failure(description: "FAIL: \(message) returned an extraction error instead of cancellation") }
|
|
throw Failure(description: "FAIL: \(message) returned media instead of cancellation")
|
|
}
|
|
|
|
final class TitleProtocol: URLProtocol, @unchecked Sendable {
|
|
static let stalled = AsyncStream<Void>.makeStream()
|
|
override class func canInit(with request: URLRequest) -> Bool { true }
|
|
override class func canonicalRequest(for request: URLRequest) -> URLRequest { request }
|
|
override func startLoading() {
|
|
let parts = URLComponents(url: request.url!, resolvingAgainstBaseURL: false)!
|
|
precondition(parts.scheme == "https" && parts.host == "www.youtube.com" && parts.path == "/oembed")
|
|
precondition(request.value(forHTTPHeaderField: "Cookie") == nil)
|
|
let page = parts.queryItems!.first { $0.name == "url" }!.value!
|
|
precondition(parts.queryItems!.first { $0.name == "format" }!.value == "json")
|
|
let id = URLComponents(string: page)!.queryItems!.first!.value!
|
|
if id == "CancelledAA" {
|
|
Self.stalled.continuation.yield()
|
|
return
|
|
}
|
|
let status = id == "Unavailable" ? 404 : 200
|
|
let mime = id == "WrongMimeAA" ? "text/html" : "application/json"
|
|
let oversized = ["OversizedAA", "ChunkedAAAA"].contains(id)
|
|
let body = oversized ? Data(repeating: 32, count: 65537) :
|
|
Data((id == "MalformedAA" ? "not JSON" : #"{"title":" Early YouTube title "}"#).utf8)
|
|
var headers = ["Content-Type": mime]
|
|
if id != "ChunkedAAAA" { headers["Content-Length"] = "\(body.count)" }
|
|
client!.urlProtocol(self, didReceive: HTTPURLResponse(url: request.url!, statusCode: status,
|
|
httpVersion: nil, headerFields: headers)!,
|
|
cacheStoragePolicy: .notAllowed)
|
|
client!.urlProtocol(self, didLoad: body)
|
|
client!.urlProtocolDidFinishLoading(self)
|
|
}
|
|
override func stopLoading() {}
|
|
}
|
|
|
|
static func titles(root: URL) async throws {
|
|
let configuration = URLSessionConfiguration.ephemeral
|
|
configuration.protocolClasses = [TitleProtocol.self]
|
|
let youtube = try await YouTubeResolver.resolveTitle(URL(string: "https://youtu.be/jNQXAC9IVRw")!,
|
|
executableURL: URL(fileURLWithPath: "/missing-helper"), configuration: configuration)
|
|
try require(youtube == "Early YouTube title", "native YouTube oEmbed needs no extractor, runtime or playable formats")
|
|
for id in ["Unavailable", "WrongMimeAA", "OversizedAA", "ChunkedAAAA", "MalformedAA"] {
|
|
do {
|
|
_ = try await YouTubeResolver.resolveTitle(URL(string: "https://youtu.be/\(id)")!, configuration: configuration)
|
|
throw Failure(description: "FAIL: invalid oEmbed response was accepted: \(id)")
|
|
} catch let failure as Failure { throw failure }
|
|
catch {}
|
|
}
|
|
let native = Task {
|
|
try await YouTubeResolver.resolveTitle(URL(string: "https://youtu.be/CancelledAA")!, configuration: configuration)
|
|
}
|
|
var started = TitleProtocol.stalled.stream.makeAsyncIterator()
|
|
_ = await started.next()
|
|
native.cancel()
|
|
do {
|
|
_ = try await native.value
|
|
throw Failure(description: "FAIL: cancelled oEmbed request returned a title")
|
|
} catch let failure as Failure { throw failure }
|
|
catch {
|
|
try require(error is CancellationError || (error as? URLError)?.code == .cancelled,
|
|
"native oEmbed cancellation ends its request without a title")
|
|
}
|
|
try require(try YouTubeResolver.decodeTitle(Data(#"{"title":" Metadata only ","formats":"ignored"}"#.utf8)) == "Metadata only",
|
|
"title decoding does not require or interpret playback formats")
|
|
try require(try YouTubeResolver.decodeTitle(Data(#"{"title":" "}"#.utf8)) == nil, "empty title retains fallback")
|
|
for json in [#"{"_type":"playlist","title":"Collection"}"#, #"{"title":"bad\u0000title"}"#, #"{"title":42}"#] {
|
|
try reject("unsafe, collection or malformed title") { _ = try YouTubeResolver.decodeTitle(Data(json.utf8)) }
|
|
}
|
|
let generic = URL(string: "https://video.example/watch?id=123")!
|
|
let extractor = try Extractor(root: root, name: "title only", json: #"{"title":"Generic title"}"#, body: #"""
|
|
printf '%s\n' "$@" > "$base/arguments"
|
|
/bin/cat "$base/response.json"
|
|
"""#)
|
|
let title = try await YouTubeResolver.resolveTitle(generic, executableURL: extractor.executable)
|
|
let arguments = try String(contentsOf: extractor.marker("arguments"), encoding: .utf8)
|
|
.split(separator: "\n").map(String.init)
|
|
try require(title == "Generic title" && arguments.suffix(2) == ["--", generic.absoluteString] &&
|
|
arguments.contains("--ignore-config") && arguments.contains("--no-plugin-dirs") &&
|
|
arguments.contains("--no-remote-components") && arguments.contains("--flat-playlist") &&
|
|
arguments.contains("--skip-download") && arguments.contains("--ignore-no-formats-error") &&
|
|
arguments.contains("--no-js-runtimes") && !arguments.contains("--js-runtimes") &&
|
|
!arguments.contains("--format"), "generic title extraction is isolated from full playback format/runtime policy")
|
|
do {
|
|
_ = try await YouTubeResolver.resolveTitle(URL(string: "https://user:password@video.example/")!,
|
|
executableURL: extractor.executable)
|
|
throw Failure(description: "FAIL: title lookup accepted embedded credentials")
|
|
} catch let failure as Failure { throw failure }
|
|
catch {}
|
|
let waiting = try Extractor(root: root, name: "cancel title", body: #"""
|
|
trap '' TERM INT
|
|
: > "$base/ready"
|
|
while [ ! -f "$base/release" ]; do /bin/sleep 0.01; done
|
|
/bin/cat "$base/response.json"
|
|
"""#)
|
|
let task = Task { try await YouTubeResolver.resolveTitle(generic, executableURL: waiting.executable) }
|
|
try await awaitMarker(waiting, "ready")
|
|
task.cancel()
|
|
try await cancelled(task, "title-only subprocess cancellation")
|
|
}
|
|
|
|
static func subprocesses(root: URL, runtime: URL) async throws {
|
|
let successful = try Extractor(root: root, name: "successful split")
|
|
let result = try await YouTubeResolver.resolve(page, executableURL: successful.executable, runtimeURL: runtime)
|
|
try require(result.video.url == "https://media.example/video?token=video"
|
|
&& result.audio?.url == "https://media.example/audio?token=audio"
|
|
&& result.title == "Separate streams" && result.duration == 19.25,
|
|
"actual subprocess output becomes usable separate media inputs")
|
|
|
|
let denied = try Extractor(root: root, name: "denied", body: #"""
|
|
/bin/cat "$base/response.json"
|
|
printf '%s\n' 'ERROR: Sign in to confirm your age: https://media.example/private?secret=not-for-errors' >&2
|
|
exit 17
|
|
"""#)
|
|
do {
|
|
_ = try await YouTubeResolver.resolve(page, executableURL: denied.executable, runtimeURL: runtime)
|
|
throw Failure(description: "FAIL: failed extractor returned media despite its exit status")
|
|
} catch let failure as Failure { throw failure }
|
|
catch {
|
|
let text = String(describing: error) + error.localizedDescription
|
|
try require(!text.contains("not-for-errors") && !text.contains("media.example"),
|
|
"extractor error does not leak signed media URLs or credentials")
|
|
}
|
|
|
|
let noisy = try Extractor(root: root, name: "pipe boundaries", body: #"""
|
|
/bin/dd if=/dev/zero bs=1024 count=512 2>/dev/null | /usr/bin/tr '\000' x >&2
|
|
/bin/dd if=/dev/zero bs=1024 count=512 2>/dev/null | /usr/bin/tr '\000' ' '
|
|
/bin/cat "$base/response.json"
|
|
"""#)
|
|
let drained = try await YouTubeResolver.resolve(page, executableURL: noisy.executable, runtimeURL: runtime)
|
|
try require(drained.audio?.url == result.audio?.url && drained.video.url == result.video.url,
|
|
"stderr beyond its capture boundary and stdout beyond pipe capacity drain without deadlock")
|
|
|
|
let oversized = try Extractor(root: root, name: "oversized stdout", body: #"""
|
|
/bin/dd if=/dev/zero bs=1024 count=17408 2>/dev/null | /usr/bin/tr '\000' ' '
|
|
/bin/cat "$base/response.json"
|
|
"""#)
|
|
do {
|
|
_ = try await YouTubeResolver.resolve(page, executableURL: oversized.executable, runtimeURL: runtime)
|
|
throw Failure(description: "FAIL: oversized extractor output was accepted")
|
|
} catch let failure as Failure { throw failure }
|
|
catch {}
|
|
|
|
let beforeLaunch = try Extractor(root: root, name: "already cancelled")
|
|
let preCancelled = Task {
|
|
withUnsafeCurrentTask { $0?.cancel() }
|
|
return try await YouTubeResolver.resolve(page, executableURL: beforeLaunch.executable, runtimeURL: runtime)
|
|
}
|
|
try await cancelled(preCancelled, "cancellation before launch")
|
|
try require(!FileManager.default.fileExists(atPath: beforeLaunch.marker("started").path),
|
|
"a pre-cancelled request never starts the dependency")
|
|
|
|
let waitingScript = #"""
|
|
trap '' TERM INT
|
|
printf '%s\n' "$$" > "$base/pid"
|
|
: > "$base/ready"
|
|
while [ ! -f "$base/release" ]; do /bin/sleep 0.01; done
|
|
/bin/cat "$base/response.json"
|
|
"""#
|
|
let old = try Extractor(root: root, name: "old request", body: waitingScript)
|
|
let newer = try Extractor(root: root, name: "new request", json: combinedJSON, body: waitingScript)
|
|
let oldTask = Task { try await YouTubeResolver.resolve(page, executableURL: old.executable, runtimeURL: runtime) }
|
|
defer { oldTask.cancel() }
|
|
try await awaitMarker(old, "ready")
|
|
let newTask = Task { try await YouTubeResolver.resolve(page, executableURL: newer.executable, runtimeURL: runtime) }
|
|
defer { newTask.cancel() }
|
|
try await awaitMarker(newer, "ready")
|
|
oldTask.cancel()
|
|
try await cancelled(oldTask, "in-flight cancellation")
|
|
let oldPID = try String(contentsOf: old.marker("pid"), encoding: .utf8)
|
|
.trimmingCharacters(in: .whitespacesAndNewlines)
|
|
try require(pid_t(oldPID).map { Darwin.kill($0, 0) == -1 && errno == ESRCH } == true,
|
|
"cancelled extractor is reaped even when it ignores termination")
|
|
oldTask.cancel()
|
|
try Data().write(to: newer.marker("release"))
|
|
let newResult = try await newTask.value
|
|
try require(newResult.title == "Combined stream" && newResult.audio == nil
|
|
&& newResult.video.url == "https://media.example/combined",
|
|
"cancelling an old request cannot terminate or replace a newer resolution")
|
|
}
|
|
|
|
static func actualQualitySelection(root: URL, runtime: URL) async throws {
|
|
let executable: URL
|
|
if CommandLine.arguments.count > 1 {
|
|
let app = URL(fileURLWithPath: CommandLine.arguments[1], isDirectory: true)
|
|
guard let bundled = try YouTubeResolver.bundledExecutable("yt-dlp", bundleURL: app) else {
|
|
throw Failure(description: "FAIL: supplied quality-selection app is not an app bundle")
|
|
}
|
|
executable = bundled
|
|
} else {
|
|
let directories = (ProcessInfo.processInfo.environment["PATH"] ?? "").split(separator: ":").map(String.init)
|
|
guard let found = directories.map({ URL(fileURLWithPath: $0).appendingPathComponent("yt-dlp") })
|
|
.first(where: { FileManager.default.isExecutableFile(atPath: $0.path) }) else {
|
|
print("Skipping actual yt-dlp quality selection: no bundled app supplied and yt-dlp is not on PATH")
|
|
return
|
|
}
|
|
executable = found
|
|
}
|
|
func video(_ height: Int, combined: Bool = false) -> [String: Any] {
|
|
["format_id": "\(height)", "height": height, "width": height * 16 / 9, "ext": "mp4",
|
|
"vcodec": "h264", "acodec": combined ? "aac" : "none", "dynamic_range": "SDR",
|
|
"protocol": "https", "url": "https://media.example/\(height).mp4", "preference": height == 720 ? 100 : 0]
|
|
}
|
|
var object: [String: Any] = [
|
|
"id": "jNQXAC9IVRw", "title": "Quality ordering", "duration": 12,
|
|
"webpage_url": page.absoluteString, "extractor": "youtube", "extractor_key": "Youtube",
|
|
"formats": [video(2160), video(1080, combined: true), video(720),
|
|
["format_id": "audio", "ext": "m4a", "vcodec": "none", "acodec": "aac",
|
|
"protocol": "https", "url": "https://media.example/audio.m4a"]]
|
|
]
|
|
let extractor = try Extractor(root: root, name: "actual quality selection")
|
|
let quotedExecutable = "'" + executable.path.replacingOccurrences(of: "'", with: "'\\''") + "'"
|
|
let script = """
|
|
#!/bin/bash
|
|
set -eu
|
|
base=${0%/*}
|
|
args=("$@")
|
|
exec \(quotedExecutable) --load-info-json "$base/response.json" "${args[@]:0:${#args[@]}-2}"
|
|
"""
|
|
try Data(script.utf8).write(to: extractor.executable)
|
|
try JSONSerialization.data(withJSONObject: object).write(to: extractor.marker("response.json"))
|
|
let normal = try await YouTubeResolver.resolve(page, executableURL: extractor.executable, runtimeURL: runtime)
|
|
try require(normal.height == 1080 && normal.audio == nil, "1080p combined video wins over preferred 720p and capped-out 4K")
|
|
let high = try await YouTubeResolver.resolve(page, height: 2160, executableURL: extractor.executable, runtimeURL: runtime)
|
|
try require(high.height == 2160 && high.audio != nil, "explicit 4K selects the higher video with separate audio")
|
|
var formats = object["formats"] as! [[String: Any]]
|
|
formats.removeAll { ($0["height"] as? Int) == 1080 }
|
|
object["formats"] = formats
|
|
try JSONSerialization.data(withJSONObject: object).write(to: extractor.marker("response.json"))
|
|
let lower = try await YouTubeResolver.resolve(page, executableURL: extractor.executable, runtimeURL: runtime)
|
|
try require(lower.height == 720, "default falls downward when 1080p is absent, never upward to available 4K")
|
|
let genericPage = URL(string: "https://video.example/watch?id=123")!
|
|
let generic: [String: Any] = [
|
|
"id": "123", "title": "Generic HTML5 video", "webpage_url": genericPage.absoluteString,
|
|
"extractor": "generic", "extractor_key": "Generic",
|
|
"formats": [["format_id": "0", "url": "https://media.example/unknown.mp4",
|
|
"ext": "mp4", "protocol": "https"]]
|
|
]
|
|
try JSONSerialization.data(withJSONObject: generic).write(to: extractor.marker("response.json"))
|
|
let unknown = try await YouTubeResolver.resolve(genericPage, executableURL: extractor.executable, runtimeURL: runtime)
|
|
try require(unknown.video.url == "https://media.example/unknown.mp4" &&
|
|
unknown.audio == nil && unknown.height == nil && unknown.webpageURL == genericPage,
|
|
"generic HTML5 video without codec or height hints remains playable and retains page identity")
|
|
let metadataOnly: [String: Any] = ["id": "123", "title": "Title without playable formats",
|
|
"webpage_url": genericPage.absoluteString, "extractor": "generic", "extractor_key": "Generic",
|
|
"formats": []]
|
|
try JSONSerialization.data(withJSONObject: metadataOnly).write(to: extractor.marker("response.json"))
|
|
let title = try await YouTubeResolver.resolveTitle(genericPage, executableURL: extractor.executable)
|
|
try require(title == "Title without playable formats", "bundled yt-dlp returns title metadata without playable formats or runtime")
|
|
}
|
|
|
|
static func bundledHelpers(root: URL) throws {
|
|
let app = root.appendingPathComponent("relocated player.app", isDirectory: true)
|
|
let macOS = app.appendingPathComponent("Contents/MacOS", isDirectory: true)
|
|
try FileManager.default.createDirectory(at: macOS, withIntermediateDirectories: true)
|
|
let external = root.appendingPathComponent("yt-dlp")
|
|
try Data("#!/bin/sh\nexit 99\n".utf8).write(to: external)
|
|
try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: external.path)
|
|
let oldPath = ProcessInfo.processInfo.environment["PATH"]
|
|
defer {
|
|
if let oldPath { setenv("PATH", oldPath, 1) } else { unsetenv("PATH") }
|
|
}
|
|
for name in ["yt-dlp", "deno"] {
|
|
for path in [root.path, ""] {
|
|
setenv("PATH", path, 1)
|
|
try reject("missing bundled helper cannot fall back to hostile or missing PATH") {
|
|
_ = try YouTubeResolver.bundledExecutable(name, bundleURL: app)
|
|
}
|
|
}
|
|
let helper = macOS.appendingPathComponent(name)
|
|
try Data("#!/bin/sh\nexit 0\n".utf8).write(to: helper)
|
|
try FileManager.default.setAttributes([.posixPermissions: 0o600], ofItemAtPath: helper.path)
|
|
try reject("nonexecutable bundled helper") {
|
|
_ = try YouTubeResolver.bundledExecutable(name, bundleURL: app)
|
|
}
|
|
try FileManager.default.setAttributes([.posixPermissions: 0o700], ofItemAtPath: helper.path)
|
|
try require(try YouTubeResolver.bundledExecutable(name, bundleURL: app) == helper,
|
|
"bundled helper location survives relocation independently of PATH")
|
|
}
|
|
try require(try YouTubeResolver.bundledExecutable("yt-dlp", bundleURL: root) == nil,
|
|
"standalone development executables can use PATH discovery")
|
|
}
|
|
|
|
static func main() async throws {
|
|
// Includes five cold starts of the bundled extractor against offline metadata.
|
|
let watchdog = Task {
|
|
try await Task.sleep(for: .seconds(45))
|
|
fatalError("FAIL: resolver checks exceeded their bounded subprocess deadline")
|
|
}
|
|
defer { watchdog.cancel() }
|
|
let root = FileManager.default.temporaryDirectory
|
|
.appendingPathComponent("terebi resolver checks '\(UUID().uuidString)", isDirectory: true)
|
|
try FileManager.default.createDirectory(at: root, withIntermediateDirectories: false)
|
|
defer { try? FileManager.default.removeItem(at: root) }
|
|
let runtime = root.appendingPathComponent("deno")
|
|
try FileManager.default.createSymbolicLink(at: runtime, withDestinationURL: URL(fileURLWithPath: "/usr/bin/true"))
|
|
try videoURLs()
|
|
try metadata()
|
|
try chapters()
|
|
try subtitles()
|
|
try malformedMetadata()
|
|
try qualities()
|
|
try bundledHelpers(root: root)
|
|
try await subprocesses(root: root, runtime: runtime)
|
|
print("PASS: extractor subprocess isolation and cancellation")
|
|
fflush(stdout)
|
|
try await titles(root: root)
|
|
print("PASS: independent native YouTube and generic title resolution")
|
|
fflush(stdout)
|
|
try await actualQualitySelection(root: root, runtime: runtime)
|
|
print("YouTube resolver checks passed")
|
|
}
|
|
}
|