From fcad96de62546cbc596e8f75437c6a3b5cd9e2fb Mon Sep 17 00:00:00 2001 From: hl-valdemar Date: Sat, 17 Oct 2026 01:11:58 +0200 Subject: [PATCH] perf: prefer VideoToolbox decoding with software fallback --- Checks/cast-media.cpp | 86 ++++++++++- Checks/video-decoding.mm | 275 +++++++++++++++++++++++++++++++++ Makefile | 41 ++++- README.md | 2 +- Sources/Core/cast_media.cpp | 48 +++--- Sources/Core/player.cpp | 18 ++- Sources/Core/video_decoder.hpp | 74 +++++++++ TODO.md | 5 +- 8 files changed, 513 insertions(+), 36 deletions(-) create mode 100644 Checks/video-decoding.mm create mode 100644 Sources/Core/video_decoder.hpp diff --git a/Checks/cast-media.cpp b/Checks/cast-media.cpp index 8caae92..d9a3a0f 100644 --- a/Checks/cast-media.cpp +++ b/Checks/cast-media.cpp @@ -35,6 +35,33 @@ extern "C" { extern char **environ; +// Encoded H.264 output cannot identify its input decoder; observe received +// source frames only in the check build, before CPU-readable mapping. +#ifdef CAST_MEDIA_CHECK +extern "C" void cast_media_check_decode_counts(uint64_t *, uint64_t *); +#endif +static std::pair decodeCounts() { + uint64_t hardware = 0, software = 0; +#ifdef CAST_MEDIA_CHECK + cast_media_check_decode_counts(&hardware, &software); +#endif + return {hardware, software}; +} +static std::pair decodeUsage(std::pair before) { + const auto after = decodeCounts(); + const std::pair counts{after.first - before.first, after.second - before.second}; +#ifdef CAST_MEDIA_CHECK + assert(counts.first + counts.second > 0); +#ifdef TEREBI_SOFTWARE_DECODE + assert(counts.first == 0); +#endif + std::cout << " decoder_videotoolbox_frames=" << counts.first << " decoder_software_frames=" << counts.second; +#else + std::cout << " decoder_frames=unavailable (compile with CAST_MEDIA_CHECK)"; +#endif + return counts; +} + static void ok(int result) { assert(result >= 0); } static std::vector readFile(const std::string &path) { std::ifstream file(path, std::ios::binary); assert(file.good()); @@ -690,6 +717,7 @@ static void codecWork(const std::filesystem::path &directory) { Upstream upstream(readFile(source), readFile(audio)); upstream.start(); const auto videoURL = upstream.url("/source.ts"), audioURL = upstream.url("/audio.wav"); std::string url; double duration = 0; + const auto beforeDecode = decodeCounts(); const auto began = std::chrono::steady_clock::now(); auto *media = start({videoURL.c_str(), "X-Video: video-secret\r\n", audioURL.c_str(), "X-Audio: audio-secret\r\n", false, false, false, 0}, url, duration); @@ -723,7 +751,8 @@ static void codecWork(const std::filesystem::path &directory) { cast_media_destroy(media); std::cout << "codec-work prepare_ms=" << std::chrono::duration(prepared - began).count() << " sequential_60s_ms=" << sequentialMS << " sequential_cpu_ms=" << sequentialCPU - << " seek_sequence_ms=" << randomMS << std::endl; + << " seek_sequence_ms=" << randomMS; + decodeUsage(beforeDecode); std::cout << std::endl; // A long video gap must not turn bounded encoder lookahead into HTTP 503, // and resuming video after audio-only segments still needs an independent IDR. const auto sparsePath = (directory / "sparse.mp4").string(); @@ -752,6 +781,55 @@ static void codecWork(const std::filesystem::path &directory) { } cast_media_destroy(media); } +// Twelve-second, 30 fps video-only fixtures exercise each codec without +// regenerating sources. Separate PCM audio also keeps the AAC lane covered. +static void decodeWork(const std::filesystem::path &directory, const std::string &fixturePath) { +#ifndef CAST_MEDIA_CHECK + assert(false && "Compile --decode-work with CAST_MEDIA_CHECK to observe actual decoding"); +#endif + AVFormatContext *format = nullptr; + ok(avformat_open_input(&format, fixturePath.c_str(), nullptr, nullptr)); ok(avformat_find_stream_info(format, nullptr)); + const int videoIndex = av_find_best_stream(format, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0); assert(videoIndex >= 0); + const auto *video = format->streams[videoIndex]->codecpar; + const std::string codec = avcodec_get_name(video->codec_id); + avformat_close_input(&format); + Upstream upstream(readFile(fixturePath), wav()); upstream.start(); + const auto videoURL = upstream.url("/source.ts"), audioURL = upstream.url("/audio.wav"); + const auto output = (directory / "decode-output.ts").string(); + const auto beforeDecode = decodeCounts(); + double wallMS = 0, cpuMS = 0; + for (bool random : {false, true}) { + std::string url; double duration = 0; + auto *media = start({videoURL.c_str(), "X-Video: video-secret\r\n", audioURL.c_str(), + "X-Audio: audio-secret\r\n", false, false, false, 0}, url, duration); + assert(std::abs(duration - 12) < .1); + const auto base = url.substr(0, url.rfind('/') + 1); + std::map> spans; + for (int at = 0; at < 2; ++at) { + const int id = random ? 1 - at : at; + const auto began = std::chrono::steady_clock::now(); const auto cpu = std::clock(); + const auto response = request(base + "segment/" + std::to_string(id) + ".ts"); + wallMS += std::chrono::duration(std::chrono::steady_clock::now() - began).count(); + cpuMS += 1000.0 * (std::clock() - cpu) / CLOCKS_PER_SEC; + assert(response.status == 200); writeFile(output, response.bytes); + const auto rows = samples(output, 1); + int videos = 0; + for (const auto &[key, value] : rows) { + assert(key.second >= id * 6 * 90000 && key.second < (id + 1) * 6 * 90000); + if (key.first) ++videos; + } + assert(videos == 180); + spans[id] = audioSpan(response, output); + assert(request(base + "segment/" + std::to_string(id) + ".ts").bytes == response.bytes); + } + assert(std::abs(spans.at(1).first - spans.at(0).second) < 1.0 / 48000); + cast_media_destroy(media); + } + assert(upstream.foreignRequests == 0); + std::cout << "decode-work codec=" << codec << " relay_24s_ms=" << wallMS << " relay_cpu_ms=" << cpuMS; + decodeUsage(beforeDecode); + std::cout << "\ncast-media: codec decode, independent IDRs, AAC continuity, sequential and random seeks passed\n"; +} int main(int argc, char **argv) { bool liveCheck = false, seekCheck = false, hlsCheck = false, remuxCheck = false; for (int argument = 1; argument < argc; ++argument) { @@ -759,6 +837,12 @@ int main(int argc, char **argv) { if (std::string(argv[argument]) == "--seek-work") seekCheck = true; if (std::string(argv[argument]) == "--hls-work") hlsCheck = true; if (std::string(argv[argument]) == "--remux-work") remuxCheck = true; + if (std::string(argv[argument]) == "--decode-work") { + assert(argument + 1 < argc && "--decode-work requires a 12s/30fps fixture path"); + char temp[] = "/tmp/terebi-decode-check.XXXXXX"; assert(mkdtemp(temp)); + decodeWork(temp, argv[++argument]); std::filesystem::remove_all(temp); + return 0; + } if (std::string(argv[argument]) == "--codec-work") { char temp[] = "/tmp/terebi-codec-check.XXXXXX"; assert(mkdtemp(temp)); codecWork(temp); std::filesystem::remove_all(temp); diff --git a/Checks/video-decoding.mm b/Checks/video-decoding.mm new file mode 100644 index 0000000..11eca06 --- /dev/null +++ b/Checks/video-decoding.mm @@ -0,0 +1,275 @@ +// Build like seek-queue-check, with -fobjc-arc and the bundled FFmpeg flags, +// linking the normal player objects EXCEPT player.o (the implementation is below). +// Build the same source again with -DTEREBI_SOFTWARE_DECODE for the baseline. +// Run: video-decoding-check [--metal | --preroll] [--frames 360] [--fps 30] fixture ... +// Fixtures: video-only CFR testsrc2, zero-origin PTS, BT.709 SDR, square SAR. +// Benchmark excludes opening, pixel checks, seeking and Metal; there is no pacing, +// audio-device setup or playback clock. Wall/process CPU are measurements only. +#include "../Sources/Core/player.cpp" +#import +#import +extern "C" { +#include +} +#include +#include + +static void require(bool condition, const char *message) { + if (!condition) { std::fprintf(stderr, "FAIL: %s\n", message); std::exit(1); } +} + +static Frame retain(const AVFrame *source) { + Frame frame(av_frame_clone(source)); + if (!frame) throw std::bad_alloc(); + return frame; +} + +static void metadata(Player &player, const AVFrame *frame) { + const auto *parameters = player.tracks[1].stream()->codecpar; + require(frame->width == parameters->width && frame->height == parameters->height, + "readable frame preserves coded dimensions"); + require(frame->color_primaries == AVCOL_PRI_BT709 && frame->color_trc == AVCOL_TRC_BT709 && + frame->colorspace == AVCOL_SPC_BT709, "readable frame preserves BT.709 SDR metadata"); + require(parameters->color_range == AVCOL_RANGE_UNSPECIFIED || + frame->color_range == parameters->color_range, "readable frame preserves color range"); + const AVRational aspect = av_guess_sample_aspect_ratio(player.format, + player.tracks[1].stream(), const_cast(frame)); + require(aspect.num > 0 && aspect.num == aspect.den, "consumer sees square sample aspect ratio"); + const auto *description = av_pix_fmt_desc_get(static_cast(frame->format)); + require(description && !(description->flags & AV_PIX_FMT_FLAG_HWACCEL), + "queued frames have a CPU-readable pixel format"); +} + +// Read every visible byte, not padding: this detects a mapped frame whose pixel +// buffer was unlocked/recycled when a decoder frame, timeline or codec was freed. +static uint64_t pixels(const AVFrame *frame) { + const auto format = static_cast(frame->format); + int widths[4]{}; + require(av_image_fill_linesizes(widths, format, frame->width) >= 0, "readable plane widths"); + ptrdiff_t strides[4]{}; + for (int plane = 0; plane < 4; ++plane) strides[plane] = widths[plane]; + size_t sizes[4]{}; + require(av_image_fill_plane_sizes(sizes, format, frame->height, strides) >= 0, + "readable plane heights"); + uint64_t hash = 14695981039346656037ULL; + unsigned low = 255, high = 0; + for (int plane = 0; plane < 4; ++plane) { + if (!sizes[plane]) continue; + require(widths[plane] > 0 && frame->data[plane] && + std::abs(frame->linesize[plane]) >= widths[plane], "real readable pixel planes"); + const size_t rows = sizes[plane] / widths[plane]; + for (size_t y = 0; y < rows; ++y) { + const uint8_t *row = frame->data[plane] + ptrdiff_t(y) * frame->linesize[plane]; + for (int x = 0; x < widths[plane]; ++x) { + low = std::min(low, unsigned(row[x])); high = std::max(high, unsigned(row[x])); + hash = (hash ^ row[x]) * 1099511628211ULL; + } + } + } + require(low < high, "testsrc2 contains real nonconstant pixel data"); + return hash; +} + +static void exactFrame(Player &player, const VideoFrame &frame, int index, int fps) { + require(bool(frame.frame), "consumer receives a video frame"); + const auto timebase = player.tracks[1].stream()->time_base; + const int64_t expected = av_rescale_q(index, AVRational{1, fps}, timebase); + require(presentationTimestamp(frame.frame.get()) == expected, "exact consumer presentation timestamp"); + require(std::abs(frame.pts - expected * av_q2d(timebase)) < 0.000001, + "queue PTS matches the fixture media timestamp"); + require(frame.end > frame.pts, "queued frame retains a positive presentation duration"); + metadata(player, frame.frame.get()); +} + +struct DecodedRun { + Frame first, last; + int frames = 0; + bool hardware = false; + size_t peakFrames = 0, peakBytes = 0; +}; + +static DecodedRun drain(Player &player, int first, int expectedFrames, int fps) { + DecodedRun result; + double previous = -std::numeric_limits::infinity(); + while (!player.tracks[1].drained || !player.video.empty()) { + auto *codec = player.tracks[1].codec; + result.hardware |= codec->pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX || codec->hw_frames_ctx != nullptr; + result.peakFrames = std::max(result.peakFrames, player.video.size()); + result.peakBytes = std::max(result.peakBytes, player.videoBytes); + require(player.video.size() <= videoFrameLimit && player.videoBytes <= videoByteLimit, + "real decoded queue stays inside frame and byte budgets"); + if (!player.video.empty() && (player.videoFull() || player.tracks[1].drained)) { + while (!player.video.empty()) { + VideoFrame frame = std::move(player.video.front()); + player.video.pop_front(); + player.videoBytes -= frame.bytes; + require(frame.pts > previous, "consumer observes strictly monotonic PTS including B-frame EOF"); + require(result.frames < expectedFrames, "no extra decoded frames after decoder drain"); + exactFrame(player, frame, first + result.frames, fps); + previous = frame.pts; + if (!result.first) result.first = retain(frame.frame.get()); + result.last = std::move(frame.frame); + ++result.frames; + } + } else { + require(player.decodeOne(), "local synchronous decode makes progress without a clock"); + } + } + require(result.frames == expectedFrames, "all expected frames survive queueing and delayed EOF drain"); + require(player.demuxEnded && player.tracks[1].sourceEnded && player.tracks[1].drainSent && + player.tracks[1].drained && player.videoBytes == 0, "demux and codec EOF fully drain the queue"); + return result; +} + +static void draw(playback::Output &output, const AVFrame *frame) { + void *native = output.make_video_view(); + require(native != nullptr, "real Metal view and shaders initialize"); + MTKView *view = (__bridge_transfer MTKView *)native; + view.paused = YES; + NSWindow *window = [[NSWindow alloc] initWithContentRect:NSMakeRect(100, 100, 320, 180) + styleMask:NSWindowStyleMaskBorderless backing:NSBackingStoreBuffered defer:NO]; + window.contentView = view; + [window orderFront:nil]; + output.present(frame); + // AppKit must attach the layer before a native drawable can be acquired. + // This is an I/O watchdog, not a media clock. + const auto deadline = Clock::now() + std::chrono::seconds(5); + while (!view.currentDrawable && Clock::now() < deadline) + CFRunLoopRunInMode(kCFRunLoopDefaultMode, .005, false); + require(view.currentDrawable != nil, "real Metal surface supplies a drawable"); + [view draw]; + [window orderOut:nil]; +} + +static void fixture(const char *path, int frames, int fps, bool metal, bool preroll) { + Player player; + player.shutdown(); // Join the worker: use actual open/decode/seek synchronously. + player.stopping.store(false); + player.activeEpoch = player.requestedEpoch.load(); + Command command{Action::Open, path}; + player.open(command); + require(player.tracks[1].codec && !player.tracks[0].codec, "fixture is video-only (no audio device)"); + require(player.canSeek && std::abs(player.duration - double(frames) / fps) < .002, + "fixture has the fixed expected media duration and is seekable"); + if (preroll) { + while (player.video.empty() && !player.tracks[1].drained) + require(player.decodeOne(), "opening transport stream makes progress"); + require(!player.video.empty(), "transport stream supplies an opening picture"); + auto retained = retain(player.video.front().frame.get()); + const auto hash = pixels(retained.get()); + const double target = (frames * 3 / 5 + .4) / fps; + player.seek(target, false); + while (player.video.empty() && !player.tracks[1].drained) + require(player.decodeOne(), "dependent-packet seek makes progress"); + require(!player.video.empty() && player.video.front().pts >= target && + player.video.front().pts < player.duration, "transport seek supplies a usable preview after preroll"); + metadata(player, player.video.front().frame.get()); + pixels(player.video.front().frame.get()); + const bool software = player.tracks[1].codec->pix_fmt != AV_PIX_FMT_VIDEOTOOLBOX; + player.release(); + require(pixels(retained.get()) == hash, "mapped opening picture survives preroll decoder replacement"); + std::printf("PASS native dependent-packet preroll/preview/lifetime: %s software=%s\n", + path, software ? "yes" : "no"); + return; + } + const auto wallStart = std::chrono::steady_clock::now(); + const auto cpuStart = std::clock(); + auto decoded = drain(player, 0, frames, fps); + const double cpu = double(std::clock() - cpuStart) / CLOCKS_PER_SEC; + const double wall = std::chrono::duration(std::chrono::steady_clock::now() - wallStart).count(); +#ifdef TEREBI_SOFTWARE_DECODE + constexpr const char *mode = "software"; + require(!decoded.hardware, "matched software baseline never negotiates hardware"); +#else + constexpr const char *mode = "preferred"; +#endif + std::printf("native-decode mode=%s fixture=%s codec=%s videotoolbox=%s format=%s frames=%d " + "wall=%.6f cpu=%.6f peak_frames=%zu peak_bytes=%zu\n", mode, path, + player.tracks[1].codec->codec->name, decoded.hardware ? "yes" : "no", + av_get_pix_fmt_name(static_cast(decoded.first->format)), decoded.frames, + wall, cpu, decoded.peakFrames, decoded.peakBytes); + const auto firstHash = pixels(decoded.first.get()); + const auto lastHash = pixels(decoded.last.get()); + require(firstHash != lastHash, "retained first and last frames contain distinct fixture pictures"); + if (metal) draw(player.output, decoded.first.get()); + + // A non-keyframe seek must discard preroll and expose the exact next PTS. + const int middle = frames / 2; + const auto seekStart = Clock::now(); + player.seek((middle + .4) / fps, false); + require(player.video.empty() && player.videoBytes == 0, "seek clears the old readable queue"); + while (player.video.empty() && !player.tracks[1].drained) + require(player.decodeOne(), "seek preroll makes progress"); + std::printf("native-seek mode=%s fixture=%s preview_ms=%.3f\n", mode, path, + std::chrono::duration(Clock::now() - seekStart).count()); + require(pixels(decoded.first.get()) == firstHash && pixels(decoded.last.get()) == lastHash, + "mapped frame references survive decoder flush"); + auto sought = drain(player, middle + 1, frames - middle - 1, fps); + pixels(sought.first.get()); + + // Seek backward after EOF, then step in both directions without a clock. + player.seek((middle + .4) / fps, false); + player.step(1); + exactFrame(player, player.presented, middle + 2, fps); + player.step(-1); + exactFrame(player, player.presented, middle + 1, fps); + player.step(1); + exactFrame(player, player.presented, middle + 2, fps); + player.seek(1.4 / fps, false); + auto restarted = drain(player, 2, frames - 2, fps); + const double lastPts = presentationTimestamp(restarted.last.get()) * + av_q2d(player.tracks[1].stream()->time_base) - player.origin; + const int lastBytes = av_image_get_buffer_size(static_cast(restarted.last->format), + restarted.last->width, restarted.last->height, 1); + require(lastBytes > 0, "last readable frame has a valid byte size"); + player.presentVideo(VideoFrame{retain(restarted.last.get()), lastPts, + lastPts + player.videoFrameTime, static_cast(lastBytes)}); + player.step(1); + exactFrame(player, player.presented, frames - 1, fps); + player.step(-1); + exactFrame(player, player.presented, frames - 2, fps); + player.step(1); + exactFrame(player, player.presented, frames - 1, fps); + + player.release(); // Retained mapped planes must also outlive the codec/device. + require(pixels(decoded.first.get()) == firstHash && pixels(decoded.last.get()) == lastHash, + "mapped frame references survive full decoder release and pool reuse"); + if (metal) draw(player.output, decoded.last.get()); + std::printf("PASS native readable frames/SDR/SAR/seek/exact steps/EOF/lifetimes: %s\n", path); +} + +int main(int argc, char **argv) { + @autoreleasepool { + int frames = 360, fps = 30; + bool metal = false, preroll = false; + std::vector paths; + for (int i = 1; i < argc; ++i) { + if (std::strcmp(argv[i], "--metal") == 0) { metal = true; continue; } + if (std::strcmp(argv[i], "--preroll") == 0) { preroll = true; continue; } + if (std::strcmp(argv[i], "--frames") == 0 || std::strcmp(argv[i], "--fps") == 0) { + const bool count = std::strcmp(argv[i], "--frames") == 0; + require(++i < argc, "numeric option needs a value"); + char *end = nullptr; + const long value = std::strtol(argv[i], &end, 10); + require(end != argv[i] && !*end && value > 0 && value <= 1000000, + "frame count and integer frame rate must be positive and bounded"); + (count ? frames : fps) = static_cast(value); + } else { + require(argv[i][0] != '-', "unknown option"); + paths.push_back(argv[i]); + } + } + require(!paths.empty() && frames >= 8, "provide fixtures with at least eight frames"); + if (metal) { + [NSApplication sharedApplication]; + [NSApp setActivationPolicy:NSApplicationActivationPolicyAccessory]; + [NSApp finishLaunching]; + } + try { + for (const char *path : paths) fixture(path, frames, fps, metal, preroll); + } catch (const std::exception &error) { + std::fprintf(stderr, "FAIL: %s\n", error.what()); + return 1; + } + } +} diff --git a/Makefile b/Makefile index 93ca69a..6f2a628 100644 --- a/Makefile +++ b/Makefile @@ -26,8 +26,9 @@ BOOTSTRAP := $(APP)/Contents/MacOS/terebi CLI := $(APP)/Contents/MacOS/terebi-cli BUNDLE_STAMP := $(BUILD_DIR)/bundle.stamp SIGNING_IDENTITY ?= - +DECODE_FIXTURES := $(addprefix $(BUILD_DIR)/fixtures/decode-,h264.mp4 hevc.mp4 vp9.mp4 av1.mp4 h264-10.mp4 mpeg4.mp4) -.PHONY: all release install check-dependencies run check check-seek-queue check-config check-fixtures check-audio check-window check-youtube check-quality check-adaptive check-network check-progress check-reporting check-cli check-runtime check-media check-startup check-bundle check-subtitles check-resolved check-cast +.PHONY: all release install check-dependencies run check check-seek-queue check-video-decoding check-config check-fixtures check-audio check-window check-youtube check-quality check-adaptive check-network check-progress check-reporting check-cli check-runtime check-media check-startup check-bundle check-subtitles check-resolved check-cast all: $(BUILD_DIR)/terebi $(NATIVE_STAMP): Scripts/dependencies.py Dependencies.lock.json @@ -62,13 +63,13 @@ $(APP)/Contents/Resources/TOMLDecoder-LICENSE.txt: $(CONFIG_LIBRARY) mkdir -p "$(APP)/Contents/Resources" cp -f "$(CONFIG_BUILD)/checkouts/TOMLDecoder/LICENSE.md" "$@" -$(BUILD_DIR)/player.o: Sources/Core/player.cpp Sources/Core/player.h Sources/Core/player_macos.h Sources/Core/output.hpp Sources/Core/network_input.hpp Sources/Core/network_policy.hpp Sources/Core/progress_reporter.hpp Makefile $(NATIVE_STAMP) | $(BUILD_DIR) check-dependencies +$(BUILD_DIR)/player.o: Sources/Core/player.cpp Sources/Core/player.h Sources/Core/player_macos.h Sources/Core/output.hpp Sources/Core/video_decoder.hpp Sources/Core/network_input.hpp Sources/Core/network_policy.hpp Sources/Core/progress_reporter.hpp Makefile $(NATIVE_STAMP) | $(BUILD_DIR) check-dependencies $(CXX) $(CXXFLAGS) -c $< -o $@ $(BUILD_DIR)/progress_reporter.o: Sources/Core/progress_reporter.cpp Sources/Core/progress_reporter.hpp Sources/Core/player.h Makefile $(NATIVE_STAMP) | $(BUILD_DIR) $(CXX) $(CXXFLAGS) -c $< -o $@ -$(BUILD_DIR)/cast_media.o: Sources/Core/cast_media.cpp Sources/Core/cast_media.h Sources/Core/network_policy.hpp Sources/Core/network_input.hpp Makefile $(NATIVE_STAMP) | $(BUILD_DIR) check-dependencies +$(BUILD_DIR)/cast_media.o: Sources/Core/cast_media.cpp Sources/Core/cast_media.h Sources/Core/video_decoder.hpp Sources/Core/network_policy.hpp Sources/Core/network_input.hpp Makefile $(NATIVE_STAMP) | $(BUILD_DIR) check-dependencies $(CXX) $(CXXFLAGS) -c $< -o $@ $(BUILD_DIR)/Output.o: Sources/macOS/Output.mm Sources/Core/output.hpp Makefile $(NATIVE_STAMP) | $(BUILD_DIR) check-dependencies @@ -105,13 +106,41 @@ run: all $(BUILD_DIR)/playback-check: Checks/playback.mm $(OBJECTS) Makefile | check-dependencies $(CXX) $(CXXFLAGS) -fobjc-arc $< $(OBJECTS) $(FF_LIBS) $(FRAMEWORKS) -o $@ -$(BUILD_DIR)/seek-queue-check: Checks/seek-queue.cpp Sources/Core/player.cpp Sources/Core/player.h Sources/Core/player_macos.h Sources/Core/output.hpp Sources/Core/network_input.hpp Sources/Core/network_policy.hpp Sources/Core/progress_reporter.hpp $(filter-out $(BUILD_DIR)/player.o,$(OBJECTS)) Makefile | check-dependencies +$(BUILD_DIR)/video-decoding-check: Checks/video-decoding.mm Sources/Core/player.cpp Sources/Core/player.h Sources/Core/player_macos.h Sources/Core/output.hpp Sources/Core/video_decoder.hpp Sources/Core/network_input.hpp Sources/Core/network_policy.hpp Sources/Core/progress_reporter.hpp $(filter-out $(BUILD_DIR)/player.o,$(OBJECTS)) Makefile | check-dependencies + $(CXX) $(CXXFLAGS) -fobjc-arc $< $(filter-out $(BUILD_DIR)/player.o,$(OBJECTS)) $(FF_LIBS) $(FRAMEWORKS) -o $@ + +$(BUILD_DIR)/video-decoding-software-check: Checks/video-decoding.mm Sources/Core/player.cpp Sources/Core/player.h Sources/Core/player_macos.h Sources/Core/output.hpp Sources/Core/video_decoder.hpp Sources/Core/network_input.hpp Sources/Core/network_policy.hpp Sources/Core/progress_reporter.hpp $(filter-out $(BUILD_DIR)/player.o,$(OBJECTS)) Makefile | check-dependencies + $(CXX) $(CXXFLAGS) -DTEREBI_SOFTWARE_DECODE -fobjc-arc $< $(filter-out $(BUILD_DIR)/player.o,$(OBJECTS)) $(FF_LIBS) $(FRAMEWORKS) -o $@ + +$(BUILD_DIR)/cast-media-software-check: Checks/cast-media.cpp Sources/Core/cast_media.cpp Sources/Core/cast_media.h Sources/Core/video_decoder.hpp Sources/Core/network_policy.hpp Sources/Core/network_input.hpp Makefile | check-dependencies + $(CXX) $(CXXFLAGS) -ISources/Core -DCAST_MEDIA_CHECK -DTEREBI_SOFTWARE_DECODE Checks/cast-media.cpp Sources/Core/cast_media.cpp $(FF_LIBS) -o $@ + +check-video-decoding: $(BUILD_DIR)/video-decoding-check $(BUILD_DIR)/cast-media-check $(DECODE_FIXTURES) $(BUILD_DIR)/fixtures/decode-preroll.ts + $(BUILD_DIR)/video-decoding-check --metal $(DECODE_FIXTURES) + $(BUILD_DIR)/video-decoding-check --preroll $(BUILD_DIR)/fixtures/decode-preroll.ts + for fixture in $(DECODE_FIXTURES); do $(BUILD_DIR)/cast-media-check --decode-work "$$fixture" || exit; done + +$(DECODE_FIXTURES): Makefile | check-dependencies + mkdir -p $(BUILD_DIR)/fixtures + ffmpeg -hide_banner -loglevel error -y -f lavfi -i testsrc2=size=1280x720:rate=30:duration=12 -vf setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709 -an -threads 2 $(DECODE_ENCODER) $@ + +$(BUILD_DIR)/fixtures/decode-h264.mp4: DECODE_ENCODER = -c:v libx264 -preset ultrafast -bf 2 -g 60 +$(BUILD_DIR)/fixtures/decode-hevc.mp4: DECODE_ENCODER = -c:v libx265 -preset ultrafast -x265-params log-level=error:pools=2 -g 60 +$(BUILD_DIR)/fixtures/decode-vp9.mp4: DECODE_ENCODER = -c:v libvpx-vp9 -deadline realtime -cpu-used 8 -row-mt 1 -g 60 +$(BUILD_DIR)/fixtures/decode-av1.mp4: DECODE_ENCODER = -c:v libsvtav1 -preset 12 -svtav1-params lp=2 -g 60 +$(BUILD_DIR)/fixtures/decode-h264-10.mp4: DECODE_ENCODER = -c:v libx264 -preset ultrafast -pix_fmt yuv420p10le -bf 2 -g 60 +$(BUILD_DIR)/fixtures/decode-mpeg4.mp4: DECODE_ENCODER = -c:v mpeg4 -bf 2 -g 60 + +$(BUILD_DIR)/fixtures/decode-preroll.ts: $(BUILD_DIR)/fixtures/decode-h264.mp4 Makefile + ffmpeg -hide_banner -loglevel error -y -i $< -map 0:v:0 -c copy -mpegts_copyts 1 -muxdelay 0 -avoid_negative_ts disabled $@ + +$(BUILD_DIR)/seek-queue-check: Checks/seek-queue.cpp Sources/Core/player.cpp Sources/Core/player.h Sources/Core/player_macos.h Sources/Core/output.hpp Sources/Core/video_decoder.hpp Sources/Core/network_input.hpp Sources/Core/network_policy.hpp Sources/Core/progress_reporter.hpp $(filter-out $(BUILD_DIR)/player.o,$(OBJECTS)) Makefile | check-dependencies $(CXX) $(CXXFLAGS) $< $(filter-out $(BUILD_DIR)/player.o,$(OBJECTS)) $(FF_LIBS) $(FRAMEWORKS) -o $@ check-seek-queue: $(BUILD_DIR)/seek-queue-check $(BUILD_DIR)/seek-queue-check -check: check-dependencies $(BUILD_DIR)/playback-check check-seek-queue check-fixtures check-audio check-window check-config check-youtube check-quality check-adaptive check-network check-progress check-reporting check-cli check-runtime check-media check-startup check-bundle check-subtitles check-resolved check-cast +check: check-dependencies $(BUILD_DIR)/playback-check check-seek-queue check-video-decoding check-fixtures check-audio check-window check-config check-youtube check-quality check-adaptive check-network check-progress check-reporting check-cli check-runtime check-media check-startup check-bundle check-subtitles check-resolved check-cast $(BUILD_DIR)/playback-check $(BUILD_DIR)/fixtures check-fixtures: | check-dependencies @@ -236,7 +265,7 @@ check-media: $(BUILD_DIR)/system-playback-check check-fixtures $(BUILD_DIR)/cast-session-check: Checks/cast-session.swift Sources/App/CastSession.swift Makefile | $(BUILD_DIR) $(SWIFT) -target $(ARCH)-apple-macosx15.0 -swift-version 6 -parse-as-library -DCAST_SESSION_CHECKS -O $(DEBUG_FLAGS) Checks/cast-session.swift Sources/App/CastSession.swift -framework Foundation -framework Network -framework Security -o $@ -$(BUILD_DIR)/cast_media-check.o: Sources/Core/cast_media.cpp Sources/Core/cast_media.h Sources/Core/network_policy.hpp Sources/Core/network_input.hpp Makefile $(NATIVE_STAMP) | $(BUILD_DIR) check-dependencies +$(BUILD_DIR)/cast_media-check.o: Sources/Core/cast_media.cpp Sources/Core/cast_media.h Sources/Core/video_decoder.hpp Sources/Core/network_policy.hpp Sources/Core/network_input.hpp Makefile $(NATIVE_STAMP) | $(BUILD_DIR) check-dependencies $(CXX) $(CXXFLAGS) -DCAST_MEDIA_CHECK -c $< -o $@ $(BUILD_DIR)/cast-media-check: Checks/cast-media.cpp $(BUILD_DIR)/cast_media-check.o Makefile | check-dependencies diff --git a/README.md b/README.md index ec74e38..1f796d9 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ Settings load once at startup. Menu changes are session-only and never write the ## Limits - SDR output only; PQ/HLG HDR is rejected. No playlists, embedded/YouTube captions, manual audio-track selection, rotation, or deinterlacing. External resolved-source WebVTT subtitles are supported. -- Video decoding is software-only. Backward frame stepping can be slow on long-GOP or unindexed media. +- H.264, HEVC, and VP9 prefer VideoToolbox decoding with software fallback; other codecs, including AV1, use software. Backward frame stepping can be slow on long-GOP or unindexed media. - Network buffering is bounded and memory-only; live/non-seekable sources disable seeking and frame stepping. Expired stream URLs are not automatically refreshed. - Audio-device changes during playback are not handled. diff --git a/Sources/Core/cast_media.cpp b/Sources/Core/cast_media.cpp index c548578..d661936 100644 --- a/Sources/Core/cast_media.cpp +++ b/Sources/Core/cast_media.cpp @@ -1,6 +1,7 @@ #include "cast_media.h" #include "network_policy.hpp" #include "network_input.hpp" +#include "video_decoder.hpp" extern "C" { #include #include @@ -40,6 +41,9 @@ using Clock = std::chrono::steady_clock; constexpr double segmentSeconds = 6; constexpr size_t segmentLimit = 32 * 1024 * 1024; constexpr size_t windowSegments = 8; +#ifdef CAST_MEDIA_CHECK +std::atomic hardwareVideoFrames{0}, softwareVideoFrames{0}; +#endif void check(int value) { if (value < 0) throw std::runtime_error("Cast media operation failed (private details redacted)."); } void require(bool value) { if (!value) throw std::runtime_error("Cast media is unavailable or unsupported (private details redacted)."); } void privateLog(void *, int level, const char *, va_list) { @@ -55,26 +59,6 @@ Frame frame() { auto p = Frame(av_frame_alloc()); require(bool(p)); return p; } Packet packet() { auto p = Packet(av_packet_alloc()); require(bool(p)); return p; } int64_t aacBoundary(double seconds) { return int64_t(std::ceil(seconds * 48000 / 1024)) * 1024; } double aacPreroll(double seconds) { return double(std::max(0, aacBoundary(seconds) - 12 * 1024)) / 48000; } -bool hasIDR(const AVPacket &packet, const AVCodecParameters &codec) { - const auto *bytes = packet.data; - const size_t size = size_t(std::max(0, packet.size)); - if (codec.extradata_size >= 5 && codec.extradata[0] == 1) { - const size_t lengthSize = (codec.extradata[4] & 3) + 1; - for (size_t at = 0; at + lengthSize <= size;) { - uint32_t length = 0; - for (size_t i = 0; i < lengthSize; ++i) length = (length << 8) | bytes[at++]; - if (!length || length > size - at) return false; - if ((bytes[at] & 31) == 5) return true; - at += length; - } - return false; - } - for (size_t at = 0; at + 3 < size; ++at) { - if (bytes[at] == 0 && bytes[at + 1] == 0 && bytes[at + 2] == 1 && - (bytes[at + 3] & 31) == 5) return true; - } - return false; -} struct Source { std::string video, videoHeaders, audio, audioHeaders; bool local, restricted, live; @@ -234,7 +218,7 @@ struct Track { const auto rate = av_guess_frame_rate(input->format, stream, nullptr); const double openingGap = rate.num > 0 ? av_q2d(av_inv_q(rate)) : 0; remux = remux && !live && pending && (pending->flags & AV_PKT_FLAG_KEY) && - hasIDR(*pending, *stream->codecpar) && + playback::hasIDR(*pending, *stream->codecpar) && (std::abs(packetTime() - start) < .001 || (start == 0 && packetTime() >= 0 && packetTime() <= openingGap)); if (aligned && !remux) { @@ -243,7 +227,7 @@ struct Track { } } if (!live && type == AVMEDIA_TYPE_VIDEO && pending && (pending->flags & AV_PKT_FLAG_KEY) && - !hasIDR(*pending, *stream->codecpar) && stream->codecpar->codec_id == AV_CODEC_ID_H264) { + !playback::hasIDR(*pending, *stream->codecpar) && stream->codecpar->codec_id == AV_CODEC_ID_H264) { // ponytail: rare intra-refresh seeks decode from the beginning; index real IDRs if this becomes slow. input->seek(0); nextPacket(); @@ -258,6 +242,9 @@ struct Track { decoder->pkt_timebase = stream->time_base; decoder->thread_count = 2; decoder->max_pixels = 4096LL * 2160; + if (type == AVMEDIA_TYPE_VIDEO && pending && + playback::independentVideoPacket(*pending, *stream->codecpar)) + playback::preferVideoToolbox(decoder); check(avcodec_open2(decoder, codec, nullptr)); } catch (...) { avcodec_free_context(&decoder); throw; } } @@ -294,6 +281,10 @@ struct Track { require(!input->interrupted()); int result = avcodec_receive_frame(decoder, f.get()); if (result == 0) { +#ifdef CAST_MEDIA_CHECK + if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) + (f->format == AV_PIX_FMT_VIDEOTOOLBOX ? hardwareVideoFrames : softwareVideoFrames).fetch_add(1, std::memory_order_relaxed); +#endif decoded = std::move(f); const double step = stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO ? double(decoded->nb_samples) / decoded->sample_rate : 1.0 / 30; nextTime = frameTime() + step; @@ -337,6 +328,7 @@ struct OutputTrack { AVStream *stream = nullptr; AVCodecContext *encoder = nullptr; SwsContext *scale = nullptr; + Frame readable; SwrContext *resample = nullptr; AVAudioFifo *fifo = nullptr; int64_t audioPTS = AV_NOPTS_VALUE; @@ -463,8 +455,14 @@ struct OutputTrack { auto out = frame(); out->format = encoder->pix_fmt; out->width = encoder->width; out->height = encoder->height; out->sample_aspect_ratio = encoder->sample_aspect_ratio; check(av_frame_get_buffer(out.get(), 32)); + if (f->format == AV_PIX_FMT_VIDEOTOOLBOX) { + if (!readable) readable = frame(); + check(playback::refVideoFrame(readable.get(), f)); + f = readable.get(); + } scale = sws_getCachedContext(scale, f->width, f->height, AVPixelFormat(f->format), out->width, out->height, encoder->pix_fmt, SWS_BILINEAR, nullptr, nullptr, nullptr); require(scale != nullptr); check(sws_scale(scale, f->data, f->linesize, 0, f->height, out->data, out->linesize)); + if (readable) av_frame_unref(readable.get()); out->pts = int64_t(std::llround(std::max(start, time) * 90000)); out->pict_type = firstVideo ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_NONE; firstVideo = false; encoded(out.get()); lastVideoTime = time; @@ -816,7 +814,7 @@ struct CastMedia { const bool independent = !video || !video->remux || (video->pending && std::abs(video->packetTime() - start) < .001 && (video->pending->flags & AV_PKT_FLAG_KEY) && - hasIDR(*video->pending, *video->stream->codecpar)); + playback::hasIDR(*video->pending, *video->stream->codecpar)); const bool drained = (vodPipeline->videoOutput && vodPipeline->videoOutput->drained) || (vodPipeline->audioOutput && vodPipeline->audioOutput->drained); if (vodPipeline->nextStart != start || !independent || drained) @@ -971,6 +969,10 @@ extern "C" bool cast_media_subtitle(CastMedia *media, int id, const void *bytes, extern "C" void cast_media_cancel(CastMedia *media) { if (media) media->cancel(); } extern "C" void cast_media_destroy(CastMedia *media) { delete media; } #ifdef CAST_MEDIA_CHECK +extern "C" void cast_media_check_decode_counts(uint64_t *hardware, uint64_t *software) { + *hardware = hardwareVideoFrames.load(std::memory_order_relaxed); + *software = softwareVideoFrames.load(std::memory_order_relaxed); +} extern "C" bool cast_media_check_advance(CastMedia *media, double seconds) { if (!media || !media->source.live || !std::isfinite(seconds) || seconds < 6 || seconds > 3600) return false; std::unique_lock guard(media->lock); diff --git a/Sources/Core/player.cpp b/Sources/Core/player.cpp index c1ef6ca..c340f5a 100644 --- a/Sources/Core/player.cpp +++ b/Sources/Core/player.cpp @@ -2,6 +2,7 @@ #include "player_macos.h" #include "output.hpp" #include "progress_reporter.hpp" +#include "video_decoder.hpp" extern "C" { #include @@ -64,6 +65,7 @@ struct Track { AVPacket *packet = nullptr; bool packetPending = false; bool receiving = false; + bool firstPacket = true; double packetBufferedEnd = 0; double decoderBufferedEnd = 0; bool sourceEnded = false; @@ -456,6 +458,7 @@ struct Player { track.codec->pkt_timebase = source->streams[index]->time_base; // A bounded software decoder pool; FFmpeg owns its internal frame reorder buffer. track.codec->thread_count = 2; + if (type == AVMEDIA_TYPE_VIDEO) playback::preferVideoToolbox(track.codec); check(avcodec_open2(track.codec, decoder, nullptr), "Opening decoder"); } @@ -466,6 +469,7 @@ struct Player { if (!track.codec) continue; avcodec_flush_buffers(track.codec); track.drainSent = track.drained = false; + track.firstPacket = true; av_packet_unref(track.packet); track.packetPending = track.receiving = track.sourceEnded = false; track.packetBufferedEnd = track.decoderBufferedEnd = 0; @@ -1086,8 +1090,11 @@ struct Player { return; skipPresentedTimestamp = AV_NOPTS_VALUE; } - const int size = av_image_get_buffer_size(static_cast(decoded->format), - decoded->width, decoded->height, 1); + Frame frame(av_frame_alloc()); + if (!frame) throw std::bad_alloc(); + check(playback::refVideoFrame(frame.get(), decoded), "Mapping decoded video"); + const int size = av_image_get_buffer_size(static_cast(frame->format), + frame->width, frame->height, 1); check(size, "Sizing decoded video"); const auto bytes = static_cast(size); if (bytes > videoByteLimit) @@ -1100,8 +1107,6 @@ struct Player { videoBytes -= video.back().bytes; video.pop_back(); } - Frame frame(av_frame_clone(decoded)); - if (!frame) throw std::bad_alloc(); VideoFrame item{std::move(frame), pts, pts + frameDuration, bytes, track.timestampOccurrence}; auto insertion = std::upper_bound(video.begin(), video.end(), pts, [](double value, const VideoFrame &entry) { return value < entry.pts; }); @@ -1137,6 +1142,11 @@ struct Player { return true; } if (track.packetPending) { + if (slot == 1 && track.firstPacket) { + check(playback::prepareVideoPacket(track.codec, *track.stream()->codecpar, *track.packet), + "Preparing video decoder preroll"); + track.firstPacket = false; + } const int result = avcodec_send_packet(track.codec, track.packet); if (result != AVERROR(EAGAIN)) { check(result, "Sending media packet to decoder"); diff --git a/Sources/Core/video_decoder.hpp b/Sources/Core/video_decoder.hpp new file mode 100644 index 0000000..3163265 --- /dev/null +++ b/Sources/Core/video_decoder.hpp @@ -0,0 +1,74 @@ +#pragma once +#include +extern "C" { +#include +#include +} + +namespace playback { +inline bool hasIDR(const AVPacket &packet, const AVCodecParameters &codec) { + const auto *bytes = packet.data; + const size_t size = size_t(std::max(0, packet.size)); + if (codec.extradata_size >= 5 && codec.extradata[0] == 1) { + const size_t lengthSize = (codec.extradata[4] & 3) + 1; + for (size_t at = 0; at + lengthSize <= size;) { + uint32_t length = 0; + for (size_t i = 0; i < lengthSize; ++i) length = (length << 8) | bytes[at++]; + if (!length || length > size - at) return false; + if ((bytes[at] & 31) == 5) return true; + at += length; + } + return false; + } + for (size_t at = 0; at + 3 < size; ++at) { + if (bytes[at] == 0 && bytes[at + 1] == 0 && bytes[at + 2] == 1 && + (bytes[at + 3] & 31) == 5) return true; + } + return false; +} +// FFmpeg retries software if VideoToolbox rejects the stream; HEVC may also use +// Apple's platform software fallback. Keep the normal decoder for other codecs. +static inline void preferVideoToolbox(AVCodecContext *context) { +#ifndef TEREBI_SOFTWARE_DECODE + // Prefer only the codecs measured in native playback and the Cast relay. + if (context->codec_id != AV_CODEC_ID_H264 && context->codec_id != AV_CODEC_ID_HEVC && + context->codec_id != AV_CODEC_ID_VP9) return; + for (int index = 0; const auto *config = avcodec_get_hw_config(context->codec, index); ++index) { + if (config->device_type != AV_HWDEVICE_TYPE_VIDEOTOOLBOX || + !(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) continue; + // The default get_format prefers this device and retries software if + // VideoToolbox rejects the stream; device creation failure is also safe. + av_hwdevice_ctx_create(&context->hw_device_ctx, AV_HWDEVICE_TYPE_VIDEOTOOLBOX, + nullptr, nullptr, 0); + break; + } +#endif +} +// Starting at a dependent packet can make VideoToolbox fail during send_packet, +// after format negotiation has already succeeded. Preserve software preroll. +inline bool independentVideoPacket(const AVPacket &packet, const AVCodecParameters ¶meters) { + return (packet.flags & AV_PKT_FLAG_KEY) && + (parameters.codec_id != AV_CODEC_ID_H264 || hasIDR(packet, parameters)); +} +inline int prepareVideoPacket(AVCodecContext *&context, const AVCodecParameters ¶meters, + const AVPacket &packet) { + if (!context->hw_device_ctx || independentVideoPacket(packet, parameters)) return 0; + auto *software = avcodec_alloc_context3(context->codec); + if (!software) return AVERROR(ENOMEM); + int result = avcodec_parameters_to_context(software, ¶meters); + software->pkt_timebase = context->pkt_timebase; + software->thread_count = context->thread_count; + software->max_pixels = context->max_pixels; + if (result >= 0) result = avcodec_open2(software, context->codec, nullptr); + if (result < 0) { avcodec_free_context(&software); return result; } + avcodec_free_context(&context); context = software; + return 0; +} +// Mapping retains and locks the CVPixelBuffer until the last frame reference +// is released. Consumers see ordinary CPU-readable planes without a pixel copy. +inline int refVideoFrame(AVFrame *destination, const AVFrame *source) { + if (source->format == AV_PIX_FMT_VIDEOTOOLBOX) + return av_hwframe_map(destination, source, AV_HWFRAME_MAP_READ); + return av_frame_ref(destination, source); +} +} diff --git a/TODO.md b/TODO.md index bd279ca..d0a99fc 100644 --- a/TODO.md +++ b/TODO.md @@ -8,7 +8,10 @@ - [x] expand Cast H.264/AAC remux coverage with keyframe-aligned segments; preserve seek accuracy, timestamps and receiver compatibility - [x] reuse decoder/encoder state during sequential Cast playback; reset on random seeks and preserve independent keyframes and AAC continuity - `.build/cast-media-check --codec-work`: 60s of 720p/30 B-frame video plus 44.1kHz PCM→AAC over loopback HTTP; three paired runs reduced median relay CPU from 3.56s to 2.58s (27%) and generation time from 4.97s to 4.43s (11%); physical-receiver playback still needs manual verification - - [ ] evaluate VideoToolbox hardware video decoding for native and Cast playback; retain software fallback for unsupported formats + - [x] evaluate VideoToolbox hardware video decoding for native and Cast playback; retain software fallback for unsupported formats + - M4, three paired 12s/720p/30 MP4 runs: H.264/HEVC/VP9 native decode process CPU fell 77/88/92%; Cast relay process CPU fell 48/58/68%, with generation 2–7% slower. Prefer these measured codecs; AV1 keeps dav1d. Unpaced native decoding is slower; H.264 seek preview rose from 1.8ms to 4.1ms. These are process CPU measurements, not energy measurements. + - `make check-video-decoding` covers native Metal upload, mapped-frame lifetime, exact seek/step/EOF, dependent-packet seek fallback, Cast independent IDRs/AAC, and unsupported codecs; `video-decoding-software-check` and `cast-media-software-check` provide matched baselines. Unsafe seek starts use software decoding (until reopen for native playback). + - Existing Cast millisecond-timestamp limiting drops VP9 MKV frames in the software baseline; a direct relay comparison confirmed VideoToolbox preserves the same output PTS/counts. This evaluation does not change that limiter. - [ ] benchmark cold start, cached/uncached seeks and sustained playback separately to prioritize the actual bottlenecks - do not enable Miruro HTTP persistence without preserving origin/header checks, or increase caches/shorten Cast segments without measuring the memory/request/codec tradeoffs - [ ] add playlist support