From e1284936f9e7ee8450af41a21b795be4037cbe3e Mon Sep 17 00:00:00 2001 From: hl-valdemar Date: Sat, 17 Oct 2026 01:44:57 +0200 Subject: [PATCH] perf: benchmark native and Cast startup, seeks, and sustained playback --- Checks/cast-media.cpp | 148 +++++++++++++++++++++ Checks/playback-benchmark.mm | 245 +++++++++++++++++++++++++++++++++++ Checks/playback-benchmark.py | 151 +++++++++++++++++++++ Makefile | 7 + README.md | 2 + TODO.md | 21 ++- 6 files changed, 572 insertions(+), 2 deletions(-) create mode 100644 Checks/playback-benchmark.mm create mode 100755 Checks/playback-benchmark.py diff --git a/Checks/cast-media.cpp b/Checks/cast-media.cpp index d9a3a0f..1ec3de8 100644 --- a/Checks/cast-media.cpp +++ b/Checks/cast-media.cpp @@ -31,6 +31,7 @@ extern "C" { #include #include #include +#include #include extern char **environ; @@ -830,7 +831,154 @@ static void decodeWork(const std::filesystem::path &directory, const std::string decodeUsage(beforeDecode); std::cout << "\ncast-media: codec decode, independent IDRs, AAC continuity, sequential and random seeks passed\n"; } + +// External fixtures keep upstream serving CPU/RSS out of the relay process. +// Cache seek means retained generated TS, not a packet-cache seek. All output +// decoding happens after the final snapshot, so validation cannot inflate any +// phase's CPU or process high-water RSS. Only endpoint responses are retained. +static void playbackBenchmark(std::string origin, const std::string &kind) { + if (origin.ends_with('/')) origin.pop_back(); + const std::string prefix = "http://127.0.0.1:"; + assert(origin.starts_with(prefix)); + assert(origin.size() > prefix.size() && origin.find_first_not_of("0123456789", prefix.size()) == std::string::npos); + const int port = std::stoi(origin.substr(prefix.size())); assert(port > 0 && port <= 65535); + assert(kind == "split" || kind == "hls"); + const bool split = kind == "split"; + const std::string video = origin + (split ? "/media/bench-video.mp4" : "/media/bench.m3u8"); + const std::string audio = origin + "/media/bench-audio.m4a"; + const CastMediaSource source{video.c_str(), split ? "X-Terebi-Fixture: video\r\n" : "X-Terebi-Fixture: combined\r\n", + split ? audio.c_str() : nullptr, split ? "X-Terebi-Fixture: audio\r\n" : nullptr, false, !split, false, 120}; + auto acknowledge = [] { + std::string line; assert(bool(std::getline(std::cin, line))); + }; + auto snapshot = [] { + rusage usage{}; assert(getrusage(RUSAGE_SELF, &usage) == 0); + return std::pair{std::chrono::steady_clock::now(), usage}; + }; + auto begin = [&](const char *phase) { + std::cout << "{\"event\":\"begin\",\"phase\":\"" << phase << "\"}" << std::endl; + acknowledge(); + return snapshot(); + }; + auto cpuMS = [](const rusage &usage) { + return 1000.0 * (usage.ru_utime.tv_sec + usage.ru_stime.tv_sec) + + (usage.ru_utime.tv_usec + usage.ru_stime.tv_usec) / 1000.0; + }; + auto wallMS = [](auto from, auto to) { + return std::chrono::duration(to - from).count(); + }; + auto finish = [&](const char *phase, const auto &before, const auto &after, double seconds, size_t bytes, const std::string &extra = "") { +#ifdef __APPLE__ + const double rssMiB = double(after.second.ru_maxrss) / (1024 * 1024); +#else + const double rssMiB = double(after.second.ru_maxrss) / 1024; +#endif + std::cout << "{\"phase\":\"" << phase << "\",\"wall_ms\":" << wallMS(before.first, after.first) + << ",\"cpu_ms\":" << cpuMS(after.second) - cpuMS(before.second) + << ",\"peak_rss_mib\":" << rssMiB << ",\"media_seconds\":" << seconds + << ",\"output_bytes\":" << bytes << extra << "}" << std::endl; + acknowledge(); + }; + std::map outputs; + std::string url; double duration = 0; + const auto cold = begin("cold_start"); + auto *media = start(source, url, duration); + const auto prepared = std::chrono::steady_clock::now(); + const auto manifest = request(url); + const auto base = url.substr(0, url.rfind('/') + 1); + const auto firstBegan = std::chrono::steady_clock::now(); + outputs.emplace(0, request(base + "segment/0.ts")); + const auto coldEnd = snapshot(); + finish("cold_start", cold, coldEnd, 0, outputs.at(0).bytes.size(), + ",\"prepare_ms\":" + std::to_string(wallMS(cold.first, prepared)) + + ",\"manifest_ms\":" + std::to_string(wallMS(prepared, firstBegan)) + + ",\"first_segment_ms\":" + std::to_string(wallMS(firstBegan, coldEnd.first))); + assert(duration >= 120 && duration < 121 && manifest.status == 200); + const std::string playlist(manifest.bytes.begin(), manifest.bytes.end()); + assert(playlist.find("#EXT-X-ENDLIST") != std::string::npos); + std::vector boundaries{0}; + for (size_t at = 0; (at = playlist.find("#EXTINF:", at)) != std::string::npos; ++at) + boundaries.push_back(boundaries.back() + std::stod(playlist.substr(at + 8))); + assert(boundaries.size() >= 18); + for (int id = 0; id <= 17; ++id) assert(std::abs(boundaries[id] - id * 6) < .00002); + + // No intervening generation: segment zero is in the eight-entry TS cache. + { + const auto cached = begin("cached_seek"); + const auto repeated = request(base + "segment/0.ts"); + const auto cachedEnd = snapshot(); + finish("cached_seek", cached, cachedEnd, 0, repeated.bytes.size(), ",\"segment_id\":0"); + assert(repeated.status == 200 && repeated.bytes == outputs.at(0).bytes); + } + outputs.clear(); + + // Sequential generation stays on the warm pipeline. The initialization and + // first six seconds above are deliberately excluded; there is no pacing. + size_t producedBytes = 0; + const auto sustained = begin("sustained"); + for (int id = 1; id <= 10; ++id) { + auto response = request(base + "segment/" + std::to_string(id) + ".ts"); + producedBytes += response.bytes.size(); + assert(response.status == 200); + if (id == 10) outputs.emplace(id, std::move(response)); + } + const auto sustainedEnd = snapshot(); + finish("sustained", sustained, sustainedEnd, boundaries[11] - boundaries[1], producedBytes, + ",\"unpaced\":true,\"first_segment_excluded\":true,\"first_segment_id\":1,\"segment_count\":10"); + + // Generating through 66s evicts segment zero from the eight-entry TS cache + // and its packets from the thirty-second rewind history: a miss in both. + const auto uncached = begin("uncached_seek"); + outputs.emplace(0, request(base + "segment/0.ts")); + const auto uncachedEnd = snapshot(); + finish("uncached_seek", uncached, uncachedEnd, 0, outputs.at(0).bytes.size(), ",\"segment_id\":0"); + cast_media_destroy(media); + + char temp[] = "/tmp/terebi-cast-benchmark.XXXXXX"; assert(mkdtemp(temp)); + const auto output = (std::filesystem::path(temp) / "segment.ts").string(); + for (const auto &[id, response] : outputs) { + assert(response.status == 200); + writeFile(output, response.bytes); + // samples requires an IDR entry, strict independent decode, both A/V, + // monotonic DTS and one decoded frame for every compressed sample. + const auto rows = samples(output, 1); + int videos = 0, audios = 0; + int64_t firstAudio = 0, firstVideo = 0; + for (const auto &[key, value] : rows) { + assert(key.second >= id * 6 * 90000 && key.second < (id + 1) * 6 * 90000); + if (key.first) { + if (!videos) { + firstVideo = key.second; + // The parent's HLS TS starts AAC 21.334ms before video; + // split MP4 starts both at zero. Allow TS rescale rounding. + const int64_t expected = id * 6 * 90000 + (split ? 0 : 1920); + assert(std::abs(firstVideo - expected) <= 2); + } + assert(key.second == firstVideo + videos * 3000); + ++videos; + } else { + if (!audios) { + firstAudio = key.second; + assert(firstAudio < id * 6 * 90000 + 1922); + } + assert(key.second == firstAudio + audios * 1920); + ++audios; + } + } + assert(videos == 180 && audios >= 281 && audios <= 282); + } + std::filesystem::remove_all(temp); + std::cerr << "cast playback benchmark: independently decodable IDRs, 30fps timestamps/frame counts and AAC verified\n"; +} int main(int argc, char **argv) { + if (argc > 1 && std::string(argv[1]) == "--playback-benchmark") { + if (argc != 4 || (std::string(argv[3]) != "split" && std::string(argv[3]) != "hls")) { + std::cerr << "usage: " << argv[0] << " --playback-benchmark \n"; + return 2; + } + playbackBenchmark(argv[2], argv[3]); + return 0; + } bool liveCheck = false, seekCheck = false, hlsCheck = false, remuxCheck = false; for (int argument = 1; argument < argc; ++argument) { if (std::string(argv[argument]) == "--live") liveCheck = true; diff --git a/Checks/playback-benchmark.mm b/Checks/playback-benchmark.mm new file mode 100644 index 0000000..c1f1cce --- /dev/null +++ b/Checks/playback-benchmark.mm @@ -0,0 +1,245 @@ +#include "../Sources/Core/player_macos.h" +#import +#import +extern "C" { +#include +} +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using Clock = std::chrono::steady_clock; + +static void require(bool condition, const char *message) { + if (!condition) { + std::fprintf(stderr, "FAIL native benchmark: %s\n", message); + std::exit(1); + } +} + +static void emit(NSDictionary *row) { + NSError *error = nil; + NSData *json = [NSJSONSerialization dataWithJSONObject:row options:0 error:&error]; + require(json != nil, "encode JSON result"); + require(std::fwrite(json.bytes, 1, json.length, stdout) == json.length && + std::fputc('\n', stdout) != EOF && std::fflush(stdout) == 0, "write JSON result"); + // The runner snapshots upstream counters before acknowledging each boundary. + // Preparation is excluded; read-ahead concurrent with a measured phase is included. + std::string acknowledgement; + require(bool(std::getline(std::cin, acknowledgement)) && acknowledgement.empty(), + "runner must acknowledge each JSON boundary with an empty line"); +} + +struct Stamp { + Clock::time_point wall; + rusage usage; +}; + +static Stamp stamp() { + Stamp result{Clock::now(), {}}; + require(getrusage(RUSAGE_SELF, &result.usage) == 0, "read process resource usage"); + return result; +} + +static double cpuMilliseconds(const rusage &usage) { + return (usage.ru_utime.tv_sec + usage.ru_stime.tv_sec) * 1000.0 + + (usage.ru_utime.tv_usec + usage.ru_stime.tv_usec) / 1000.0; +} + +static void begin(NSString *phase) { + emit(@{@"event": @"begin", @"phase": phase}); +} + +static void finish(NSString *phase, const Stamp &start, const Stamp &end, + PlayerState state, NSDictionary *details) { + NSMutableDictionary *row = [@{ + @"phase": phase, + @"wall_ms": @(std::chrono::duration(end.wall - start.wall).count()), + @"cpu_ms": @(cpuMilliseconds(end.usage) - cpuMilliseconds(start.usage)), + // Darwin reports ru_maxrss in bytes. This is process-lifetime high-water RSS, + // not a per-phase delta; CPU includes every native decoder/output thread. + @"peak_rss_mib": @(end.usage.ru_maxrss / (1024.0 * 1024.0)), + @"media_seconds": @0, + @"output_bytes": @0, + @"position_seconds": @(state.position), + @"buffered_start": @(state.buffered_start), + @"buffered_end": @(state.buffered_end) + } mutableCopy]; + [row addEntriesFromDictionary:details]; + emit(row); +} + +static PlayerState until(Player *player, const std::function &ready, + const char *phase, MTKView *view = nil) { + const auto deadline = Clock::now() + std::chrono::seconds(60); + auto nextDraw = Clock::now(); + while (Clock::now() < deadline) { + const auto state = player_get_state(player); + if (state.status == PLAYER_ERROR) { + std::fprintf(stderr, "FAIL native benchmark %s: %s\n", phase, state.error); + std::exit(1); + } + // Exercise the real Metal renderer at the fixture's 30 fps, not at the + // 1 kHz state-poll rate. Manual draws also work for this offscreen window. + if (view && Clock::now() >= nextDraw) { + [view draw]; + nextDraw = Clock::now() + std::chrono::duration_cast( + std::chrono::duration(1.0 / 30)); + } + if (ready(state)) return state; + CFRunLoopRunInMode(kCFRunLoopDefaultMode, .001, false); + } + const auto state = player_get_state(player); + std::fprintf(stderr, + "FAIL native benchmark %s: deadline status=%d position=%.6f duration=%.6f " + "pending=%d buffering=%d cache=%.6f..%.6f\n", phase, state.status, state.position, + state.duration, state.seek_pending, state.buffering, state.buffered_start, state.buffered_end); + std::exit(1); +} + +static bool pausedReady(PlayerState state) { + return state.status == PLAYER_PAUSED && !state.seek_pending && !state.buffering && + state.has_audio && state.has_video && state.can_seek; +} + +static PlayerState seek(Player *player, double target, const char *phase) { + const auto revision = player_get_state(player).revision; + require(player_seek(player, target), "paused seek accepted"); + return until(player, [=](auto state) { + // !buffering is important: seek_pending alone acknowledges reader/command + // work, not decoded preview readiness. The paused tick clears buffering + // only after presenting a decoded frame and supplying initial audio. + return pausedReady(state) && state.revision != revision && + std::abs(state.position - target) < .05; + }, phase); +} + +static bool cached(PlayerState state, double target) { + // Leave room for the preceding two-second GOP and following audio/video. + return pausedReady(state) && state.buffered_end > state.buffered_start && + state.buffered_start <= target - 2 && state.buffered_end >= target + 1; +} + +static bool uncached(PlayerState state, double target) { + return pausedReady(state) && state.buffered_end > state.buffered_start && + target > state.buffered_end + 2; +} + +int main(int argc, char **argv) { + @autoreleasepool { + require(argc == 3, "usage: playback-benchmark "); + const std::string base = argv[1], mode = argv[2]; + require(base.rfind("http://127.0.0.1:", 0) == 0 && base.size() > 17 && base.size() <= 22 && + base.find_first_not_of("0123456789", 17) == std::string::npos && + std::strtoul(base.c_str() + 17, nullptr, 10) > 0 && + std::strtoul(base.c_str() + 17, nullptr, 10) <= 65535, + "fixture base URL must be an http://127.0.0.1: origin"); + require(mode == "split" || mode == "hls", "source must be split or hls"); + const bool split = mode == "split"; + const std::string video = base + (split ? "/media/bench-video.mp4" : "/media/bench.m3u8"); + const std::string audio = split ? base + "/media/bench-audio.m4a" : ""; + PlayerNetworkSource source{video.c_str(), split ? audio.c_str() : nullptr, + split ? "X-Terebi-Fixture: video\r\n" : "X-Terebi-Fixture: combined\r\n", + split ? "X-Terebi-Fixture: audio\r\n" : nullptr, + 120, false, split ? nullptr : "miruro:YmVuY2g:1", 0}; + av_log_set_level(AV_LOG_ERROR); + [NSApplication sharedApplication]; + + // Cold start follows queued autoplay through the first real audio-clock + // advancement and pause. Buffering gates audio and decoded video readiness. + // It includes player/audio/Metal setup, but excludes process launch and + // fixture creation; it is NOT just the initial PLAYER_PAUSED publication. + begin(@"cold_start"); + const auto coldStart = stamp(); + std::unique_ptr player(player_create(), player_destroy); + require(bool(player), "create player"); + require(player_set_volume(player.get(), 0), "mute real native audio output"); + void *native = player_macos_make_video_view(player.get()); + require(native != nullptr, "initialize native Metal view"); + MTKView *view = (__bridge_transfer MTKView *)native; + view.paused = YES; + view.autoResizeDrawable = NO; + view.drawableSize = CGSizeMake(1280, 720); + NSWindow *window = [[NSWindow alloc] initWithContentRect:NSMakeRect(-4000, -4000, 1280, 720) + styleMask:NSWindowStyleMaskBorderless backing:NSBackingStoreBuffered defer:NO]; + window.contentView = view; + [window orderBack:nil]; + require(player_open_network(player.get(), &source), "accept network source"); + require(player_play(player.get()), "queue autoplay during network open"); + until(player.get(), [](auto state) { + return state.status == PLAYER_PLAYING && !state.buffering && state.position >= .05; + }, "initial real playback readiness", view); + require(view.currentDrawable != nil, "initial Metal drawable"); + [view draw]; + require(player_pause(player.get()), "pause initial playback probe"); + auto state = until(player.get(), pausedReady, "initial playback probe paused", view); + const auto coldEnd = stamp(); + finish(@"cold_start", coldStart, coldEnd, state, + @{@"readiness": @"queued_autoplay_then_audio_clock_probe_and_pause", + @"probe_media_seconds": @(state.position), @"render_width": @1280, @"render_height": @720}); + + constexpr double hitTarget = 20.5, missTarget = 100.5; + // Waiting for retained packet coverage is preparation, not seek latency. + until(player.get(), [](auto s) { return cached(s, hitTarget); }, "cached target coverage"); + begin(@"cached_seek"); + auto before = player_get_state(player.get()); + require(cached(before, hitTarget), "cached seek target and GOP must remain inside retained coverage"); + const auto hitStart = stamp(); + state = seek(player.get(), hitTarget, "cached paused decoded seek"); + [view draw]; + const auto hitEnd = stamp(); + finish(@"cached_seek", hitStart, hitEnd, state, + @{@"target_seconds": @(hitTarget), @"cache_before_start": @(before.buffered_start), + @"cache_before_end": @(before.buffered_end), @"cache_precondition": @"retained_packet_hit"}); + + // Sustained is real paced playback from the cached seek, including + // moving-frontier read-ahead, start and final pause acknowledgement. + // Position is the actual native audio-master clock. Only watchdogs impose + // elapsed-time limits; the completion condition is twelve media seconds. + begin(@"sustained"); + before = player_get_state(player.get()); + require(pausedReady(before) && std::abs(before.position - hitTarget) < .05 && + before.duration > before.position + 12, "sustained starts at acknowledged cached target"); + const auto sustainedStart = stamp(); + auto previousPoll = sustainedStart.wall; + bool wasBuffering = true; + double bufferingMilliseconds = 0; + require(player_play(player.get()), "start paced sustained playback"); + state = until(player.get(), [&](auto s) { + const auto now = Clock::now(); + if (wasBuffering) + bufferingMilliseconds += std::chrono::duration(now - previousPoll).count(); + previousPoll = now; + wasBuffering = s.buffering; + return s.status == PLAYER_PLAYING && !s.seek_pending && !s.buffering && + s.position >= before.position + 12; + }, "twelve media seconds of native playback", view); + require(player_pause(player.get()), "pause sustained playback"); + state = until(player.get(), pausedReady, "sustained pause acknowledgement", view); + const auto sustainedEnd = stamp(); + finish(@"sustained", sustainedStart, sustainedEnd, state, + @{@"media_seconds": @(state.position - before.position), @"paced": @YES, + @"start_position_seconds": @(before.position), @"buffering_ms": @(bufferingMilliseconds), + @"buffering_measurement": @"sampled_at_approximately_1ms"}); + + begin(@"uncached_seek"); + before = player_get_state(player.get()); + require(uncached(before, missTarget), "uncached seek must lie beyond retained packet coverage"); + const auto missStart = stamp(); + state = seek(player.get(), missTarget, "uncached paused decoded seek"); + [view draw]; + const auto missEnd = stamp(); + finish(@"uncached_seek", missStart, missEnd, state, + @{@"target_seconds": @(missTarget), @"cache_before_start": @(before.buffered_start), + @"cache_before_end": @(before.buffered_end), @"cache_precondition": @"outside_retained_packets"}); + [window orderOut:nil]; + window.contentView = nil; + player.reset(); + } +} diff --git a/Checks/playback-benchmark.py b/Checks/playback-benchmark.py new file mode 100755 index 0000000..5e5bb62 --- /dev/null +++ b/Checks/playback-benchmark.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Measure real native/Cast source paths; no resolver, WAN, TLS or physical receiver.""" +from __future__ import annotations + +import argparse +import json +import platform +import statistics +import subprocess +import threading +from pathlib import Path + +from streaming import StreamingServer + + +def fixtures(directory: Path): + directory.mkdir(parents=True, exist_ok=True) + if (directory / "bench.m3u8").is_file(): + return + ffmpeg = ["ffmpeg", "-hide_banner", "-loglevel", "error", "-y"] + combined = directory / "bench.mp4" + subprocess.run(ffmpeg + [ + "-f", "lavfi", "-i", "testsrc2=size=1280x720:rate=30:duration=120", + "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=48000:duration=120", + "-c:v", "libx264", "-preset", "ultrafast", "-threads", "2", "-bf", "2", + "-g", "60", "-keyint_min", "60", "-sc_threshold", "0", "-pix_fmt", "yuv420p", + "-c:a", "aac", "-movflags", "+faststart", str(combined), + ], check=True, timeout=180) + for track, name in (("0:v:0", "bench-video.mp4"), ("0:a:0", "bench-audio.m4a")): + subprocess.run(ffmpeg + ["-i", str(combined), "-map", track, "-c", "copy", + "-movflags", "+faststart", str(directory / name)], check=True, timeout=60) + subprocess.run(ffmpeg + ["-i", str(combined), "-map", "0", "-c", "copy", + "-hls_time", "6", "-hls_playlist_type", "vod", + "-hls_segment_filename", str(directory / "bench-%03d.ts"), + str(directory / "bench.m3u8")], check=True, timeout=60) + + +class BenchmarkServer(StreamingServer): + def __init__(self, directory: Path, delay_ms: float): + super().__init__(directory) + self.headers = {path.name: "combined" for path in directory.iterdir() if path.is_file()} + self.headers.update({"bench-video.mp4": "video", "bench-audio.m4a": "audio"}) + self.delay_seconds = delay_ms / 1000 + self.connections: set[tuple] = set() + + def _get(self, request): + if request.path.startswith("/media/"): + with self.lock: + self.connections.add(request.client_address) + # Controlled per-request response latency, not a simulated bandwidth or RTT. + if self.delay_seconds: + self.stopping.wait(self.delay_seconds) + super()._get(request) + + def snapshot(self): + with self.lock: + return {**{key: sum(row[key] for row in self.stats.values()) + for key in ("requests", "ranges", "bytes", "denied")}, + "connections": len(self.connections)} + + +def sample(binary: Path, cast: bool, source: str, server: BenchmarkServer): + command = [str(binary.resolve())] + (["--playback-benchmark"] if cast else []) + command += [server.base_url, source] + rows = [] + # A process deadline is infrastructure protection, never a measured playback clock. + with subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True) as process: + timer = threading.Timer(120, process.kill) + timer.start() + errors = [] + reader = threading.Thread(target=lambda: errors.append(process.stderr.read()), daemon=True) + reader.start() + before = None + phase = None + try: + for line in process.stdout: + row = json.loads(line) + after = server.snapshot() + if row.get("event") == "begin": + if before is not None: + raise RuntimeError("nested benchmark phase") + before, phase = after, row["phase"] + else: + if before is None or phase != row["phase"]: + raise RuntimeError("missing benchmark phase boundary") + row.update({"upstream_" + key: after[key] - before[key] for key in after}) + if row["upstream_denied"]: + raise RuntimeError("benchmark source headers rejected") + rows.append(row) + before = None + process.stdin.write("\n") + process.stdin.flush() + result = process.wait() + reader.join() + if result: + # Diagnostics can contain fixture addresses; do not publish addresses/headers. + detail = "".join(errors).replace(server.base_url, "") + raise RuntimeError(f"{binary.name}/{source} exited {result}: {detail}") + finally: + timer.cancel() + if process.poll() is None: + process.kill() + process.wait() + if sorted(row["phase"] for row in rows) != ["cached_seek", "cold_start", "sustained", "uncached_seek"]: + raise RuntimeError("benchmark must report all four phases exactly once") + return rows + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("native", type=Path) + parser.add_argument("cast", type=Path) + parser.add_argument("fixtures", type=Path) + parser.add_argument("--runs", type=int, default=3) + parser.add_argument("--request-delay-ms", type=float, nargs="+", default=[0, 25]) + args = parser.parse_args() + if args.runs < 1 or any(value < 0 for value in args.request_delay_ms): + parser.error("runs must be positive and delays nonnegative") + fixtures(args.fixtures) + print(json.dumps({"environment": platform.platform(), "machine": platform.machine(), + "runs": args.runs, "request_delay_ms": args.request_delay_ms}), flush=True) + results = [] + for delay in args.request_delay_ms: + for run in range(1, args.runs + 1): + for source in ("split", "hls"): + for engine, binary in (("native", args.native), ("cast", args.cast)): + with BenchmarkServer(args.fixtures, delay) as server: + for row in sample(binary, engine == "cast", source, server): + row.update(engine=engine, source=source, request_delay_ms=delay, run=run) + results.append(row) + print(json.dumps(row, sort_keys=True), flush=True) + print("\nMedians; upstream counts include concurrent read-ahead; peak RSS is process-lifetime high water.") + print("| Delay ms | Engine | Source | Phase | Wall ms | CPU ms | Peak MiB | Requests | Connections | Upstream MiB | Media s |") + print("| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: |") + for delay in args.request_delay_ms: + for engine in ("native", "cast"): + for source in ("split", "hls"): + for phase in ("cold_start", "cached_seek", "uncached_seek", "sustained"): + group = [row for row in results if (row["request_delay_ms"], row["engine"], row["source"], row["phase"]) == + (delay, engine, source, phase)] + def median(key): + return statistics.median(row[key] for row in group) + print(f"| {delay:g} | {engine} | {source} | {phase} | {median('wall_ms'):.2f} | " + f"{median('cpu_ms'):.2f} | {median('peak_rss_mib'):.1f} | " + f"{median('upstream_requests'):g} | {median('upstream_connections'):g} | " + f"{median('upstream_bytes') / 1048576:.2f} | {median('media_seconds'):.2f} |") + + +if __name__ == "__main__": + main() diff --git a/Makefile b/Makefile index 6f2a628..0f51323 100644 --- a/Makefile +++ b/Makefile @@ -212,6 +212,13 @@ check-adaptive: $(BUILD_DIR)/automatic-quality-playback-check $(BUNDLE_STAMP) $(BUILD_DIR)/network-playback-check: Checks/network-playback.mm $(OBJECTS) Makefile | check-dependencies $(CXX) $(CXXFLAGS) -fobjc-arc $< $(OBJECTS) $(FF_LIBS) $(FRAMEWORKS) -o $@ +$(BUILD_DIR)/playback-benchmark: Checks/playback-benchmark.mm $(OBJECTS) Makefile | check-dependencies + $(CXX) $(CXXFLAGS) -fobjc-arc $< $(OBJECTS) $(FF_LIBS) $(FRAMEWORKS) -o $@ + +.PHONY: benchmark-playback +benchmark-playback: $(BUILD_DIR)/playback-benchmark $(BUILD_DIR)/cast-media-check + python3 Checks/playback-benchmark.py $(BUILD_DIR)/playback-benchmark $(BUILD_DIR)/cast-media-check $(BUILD_DIR)/fixtures/playback-benchmark + check-network: $(BUILD_DIR)/network-playback-check check-fixtures ffmpeg -hide_banner -loglevel error -y -i $(BUILD_DIR)/fixtures/frame-step.mp4 -an -c:v copy $(BUILD_DIR)/fixtures/frame-step-video.mp4 ffmpeg -hide_banner -loglevel error -y -i $(BUILD_DIR)/fixtures/frame-step.mp4 -vn -c:a copy $(BUILD_DIR)/fixtures/frame-step-audio.m4a diff --git a/README.md b/README.md index 1f796d9..eaa105f 100644 --- a/README.md +++ b/README.md @@ -137,4 +137,6 @@ make check The full suite requires a graphical macOS session and a working audio device; it launches isolated app instances and plays short test audio. Targeted `check-*` targets are listed in the [Makefile](Makefile). Visual appearance, physical media-key routing, and receiver behavior still need manual checks. +`make benchmark-playback` measures cold source startup, cached/uncached seeks, and sustained native/Cast playback with repeatable loopback fixtures. It requires the same macOS session/audio device and fixture-generation FFmpeg; results exclude live extraction, WAN/TLS, and physical-receiver latency. + Playback terminology: [CONTEXT.md](CONTEXT.md). diff --git a/TODO.md b/TODO.md index d0a99fc..849a8a1 100644 --- a/TODO.md +++ b/TODO.md @@ -3,7 +3,7 @@ - we don't need every minute detail (already described perfectly by the code btw) described in painstakingly verbose paragraphs in the readme. - implementation details are not appropriate in a readme file (without a very good reason to be there). - the readme should be compressed to actually useful information - nothing more - and verbose, irrelevant, or redundant information deleted. -- [ ] evaluate playback optimization opportunities for YouTube and Miruro — measure benefits before changing behavior +- [x] evaluate playback optimization opportunities for YouTube and Miruro — measure benefits before changing behavior - [x] coalesce superseded, not-yet-started seeks in the player and Cast controls; preserve accumulated relative skips and play/pause ordering - [x] expand Cast H.264/AAC remux coverage with keyframe-aligned segments; preserve seek accuracy, timestamps and receiver compatibility - [x] reuse decoder/encoder state during sequential Cast playback; reset on random seeks and preserve independent keyframes and AAC continuity @@ -12,7 +12,24 @@ - M4, three paired 12s/720p/30 MP4 runs: H.264/HEVC/VP9 native decode process CPU fell 77/88/92%; Cast relay process CPU fell 48/58/68%, with generation 2–7% slower. Prefer these measured codecs; AV1 keeps dav1d. Unpaced native decoding is slower; H.264 seek preview rose from 1.8ms to 4.1ms. These are process CPU measurements, not energy measurements. - `make check-video-decoding` covers native Metal upload, mapped-frame lifetime, exact seek/step/EOF, dependent-packet seek fallback, Cast independent IDRs/AAC, and unsupported codecs; `video-decoding-software-check` and `cast-media-software-check` provide matched baselines. Unsafe seek starts use software decoding (until reopen for native playback). - Existing Cast millisecond-timestamp limiting drops VP9 MKV frames in the software baseline; a direct relay comparison confirmed VideoToolbox preserves the same output PTS/counts. This evaluation does not change that limiter. - - [ ] benchmark cold start, cached/uncached seeks and sustained playback separately to prioritize the actual bottlenecks + - [x] benchmark cold start, cached/uncached seeks and sustained playback separately to prioritize the actual bottlenecks + - `make benchmark-playback`: M4/16 GiB, macOS 27.0.1; three fresh-process samples per source/latency profile. Shared 120s/720p/30 H.264 B-frame + 48kHz AAC fixture, two-second GOPs; YouTube-shaped split MP4 and Miruro-shaped restricted six-second HLS. Added response delay is per request, not simulated RTT/bandwidth. No playback behavior changed. + + | Added delay ms | Path | Cold ms | Cached seek ms | Uncached seek ms | Sustained wall s | Sustained process CPU s | + | ---: | --- | ---: | ---: | ---: | ---: | ---: | + | 0 | Native split | 374.93 | 21.15 | 26.58 | 12.20 | 0.81 | + | 0 | Native HLS | 473.42 | 28.88 | 225.08 | 12.22 | 0.80 | + | 0 | Cast split | 627.88 | 3.36 | 482.14 | 4.27 | 1.60 | + | 0 | Cast HLS | 715.66 | 3.79 | 492.37 | 4.16 | 0.77 | + | 25 | Native split | 406.77 | 20.96 | 127.40 | 12.30 | 1.05 | + | 25 | Native HLS | 920.65 | 40.56 | 245.96 | 12.23 | 0.87 | + | 25 | Cast split | 1251.29 | 3.55 | 1451.26 | 7.61 | 1.65 | + | 25 | Cast HLS | 662.73 | 1.97 | 521.57 | 4.50 | 0.72 | + + - Medians above: native sustained is 12 media seconds with real muted audio and offscreen 720p Metal draws; Cast is unpaced generation of 60 media seconds after the first segment. Native cold includes queued autoplay to 53ms of acknowledged audio-clock progress and pause; Cast cold includes preparation, manifest and first six-second segment. Native hits seek to retained 20.5s; misses seek to 100.5s beyond retained coverage. Cast hits repeat segment zero; misses regenerate it after both segment-cache and packet-history eviction. + - Prioritize investigating native HLS random-seek preroll and Cast first-segment/random-seek generation, not more sequential codec tuning: native HLS misses remained 220–415ms without injected delay versus 21–74ms for split MP4; Cast's first segment took 552–915ms while preparation took 5–39ms, and sustained generation remained faster than playback. These are bottleneck candidates, not measured benefits of an unimplemented optimization. + - Memory/request baseline: median pre-validation lifetime peak RSS across profiles was 230–251 MiB native / 238–253 MiB Cast, including renderer/loopback-client overhead. Zero-delay cold windows transferred roughly 51–55 MiB, with 6/12 native and 7/12 Cast upstream requests for split/HLS; counts and bytes include concurrent read-ahead, so even a cache hit can show network activity. Keep existing cache/segment limits and origin/header checks. + - All 24 source sessions passed (96 phase rows): accurate native seek acknowledgements, cache preconditions, real audio-clock advancement, repeated Cast byte equality, and independently decodable Cast endpoint segments with expected frame counts/timestamps/AAC. Large timing spread (25ms-delay Cast split cold 703–1973ms) prevents causal latency/CPU comparisons; no persistence/segment-size benefit is claimed. Fresh process/source is not cold OS caches; process CPU is not total hardware energy. Resolver/CLI launch, live CDN/TLS, bandwidth limits, other codecs/remux, visual appearance and physical receivers are not measured. - do not enable Miruro HTTP persistence without preserving origin/header checks, or increase caches/shorten Cast segments without measuring the memory/request/codec tradeoffs - [ ] add playlist support - maybe by allowing multiple file selection? 🤔