perf: prefer VideoToolbox decoding with software fallback

Checks/cast-media.cpp

@@ -14,6 +14,7 @@ extern "C" {
 #include <condition_variable>
 #include <cmath>
 #include <cstring>
+#include <ctime>
 #include <filesystem>
 #include <fstream>
 #include <future>
@@ -34,6 +35,33 @@ extern "C" {
 
 extern char **environ;
 
+// Encoded H.264 output cannot identify its input decoder; observe received
+// source frames only in the check build, before CPU-readable mapping.
+#ifdef CAST_MEDIA_CHECK
+extern "C" void cast_media_check_decode_counts(uint64_t *, uint64_t *);
+#endif
+static std::pair<uint64_t, uint64_t> decodeCounts() {
+    uint64_t hardware = 0, software = 0;
+#ifdef CAST_MEDIA_CHECK
+    cast_media_check_decode_counts(&hardware, &software);
+#endif
+    return {hardware, software};
+}
+static std::pair<uint64_t, uint64_t> decodeUsage(std::pair<uint64_t, uint64_t> before) {
+    const auto after = decodeCounts();
+    const std::pair counts{after.first - before.first, after.second - before.second};
+#ifdef CAST_MEDIA_CHECK
+    assert(counts.first + counts.second > 0);
+#ifdef TEREBI_SOFTWARE_DECODE
+    assert(counts.first == 0);
+#endif
+    std::cout << " decoder_videotoolbox_frames=" << counts.first << " decoder_software_frames=" << counts.second;
+#else
+    std::cout << " decoder_frames=unavailable (compile with CAST_MEDIA_CHECK)";
+#endif
+    return counts;
+}
+
 static void ok(int result) { assert(result >= 0); }
 static std::vector<uint8_t> readFile(const std::string &path) {
     std::ifstream file(path, std::ios::binary); assert(file.good());
@@ -673,6 +701,135 @@ static void remuxWork(const std::filesystem::path &directory) {
         cast_media_destroy(media);
     }
 }
+// Exercise the HTTP relay, not an isolated codec: delayed video, resampled AAC,
+// independent segment decoding, cache eviction, and random-to-sequential seeks.
+static void codecWork(const std::filesystem::path &directory) {
+    const auto source = (directory / "codec.mp4").string(), audio = (directory / "codec.wav").string();
+    const char *args[] = {"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
+        "-f", "lavfi", "-i", "testsrc2=size=1280x720:rate=30:duration=61",
+        "-f", "lavfi", "-i", "sine=frequency=440:sample_rate=44100:duration=61",
+        "-c:v", "libx264", "-preset", "ultrafast", "-bf", "2", "-g", "210",
+        "-an", source.c_str(), "-vn", "-c:a", "pcm_s16le", audio.c_str(), nullptr};
+    pid_t child = 0;
+    assert(posix_spawnp(&child, args[0], nullptr, nullptr, const_cast<char *const *>(args), environ) == 0);
+    int status = 0; assert(waitpid(child, &status, 0) == child);
+    assert(WIFEXITED(status) && WEXITSTATUS(status) == 0);
+    Upstream upstream(readFile(source), readFile(audio)); upstream.start();
+    const auto videoURL = upstream.url("/source.ts"), audioURL = upstream.url("/audio.wav");
+    std::string url; double duration = 0;
+    const auto beforeDecode = decodeCounts();
+    const auto began = std::chrono::steady_clock::now();
+    auto *media = start({videoURL.c_str(), "X-Video: video-secret\r\n", audioURL.c_str(),
+        "X-Audio: audio-secret\r\n", false, false, false, 0}, url, duration);
+    const auto prepared = std::chrono::steady_clock::now();
+    const auto base = url.substr(0, url.rfind('/') + 1), output = (directory / "codec-output.ts").string();
+    std::map<int, std::pair<double, double>> spans;
+    double sequentialMS = 0, randomMS = 0, sequentialCPU = 0;
+    for (int id : {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 2, 3, 0, 1, 10}) {
+        const auto before = std::chrono::steady_clock::now();
+        const auto cpuBefore = std::clock();
+        const auto response = request(base + "segment/" + std::to_string(id) + ".ts");
+        const auto elapsed = std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - before).count();
+        const double cpuMS = 1000.0 * (std::clock() - cpuBefore) / CLOCKS_PER_SEC;
+        if (spans.size() < 10) { sequentialMS += elapsed; sequentialCPU += cpuMS; }
+        else randomMS += elapsed;
+        assert(response.status == 200); writeFile(output, response.bytes);
+        const auto rows = samples(output, 1);
+        int videos = 0;
+        for (const auto &[key, value] : rows) {
+            assert(key.second >= id * 6 * 90000 && key.second < std::min(61, (id + 1) * 6) * 90000);
+            if (key.first) ++videos;
+        }
+        assert(videos == (id == 10 ? 30 : 180));
+        auto span = audioSpan(response, output);
+        if (spans.contains(id - 1)) assert(std::abs(span.first - spans.at(id - 1).second) < 1.0 / 48000);
+        if (spans.contains(id + 1)) assert(std::abs(spans.at(id + 1).first - span.second) < 1.0 / 48000);
+        spans[id] = span;
+        assert(request(base + "segment/" + std::to_string(id) + ".ts").bytes == response.bytes);
+    }
+    assert(upstream.foreignRequests == 0);
+    cast_media_destroy(media);
+    std::cout << "codec-work prepare_ms=" << std::chrono::duration<double, std::milli>(prepared - began).count()
+              << " sequential_60s_ms=" << sequentialMS << " sequential_cpu_ms=" << sequentialCPU
+              << " seek_sequence_ms=" << randomMS;
+    decodeUsage(beforeDecode); std::cout << std::endl;
+    // A long video gap must not turn bounded encoder lookahead into HTTP 503,
+    // and resuming video after audio-only segments still needs an independent IDR.
+    const auto sparsePath = (directory / "sparse.mp4").string();
+    const char *sparseArgs[] = {"ffmpeg", "-hide_banner", "-loglevel", "error", "-y",
+        "-i", source.c_str(), "-t", "25", "-vf", "select='lt(t,3)+gte(t,18)'",
+        "-fps_mode", "vfr", "-c:v", "libx264", "-preset", "ultrafast", "-bf", "2",
+        "-an", sparsePath.c_str(), nullptr};
+    assert(posix_spawnp(&child, sparseArgs[0], nullptr, nullptr, const_cast<char *const *>(sparseArgs), environ) == 0);
+    assert(waitpid(child, &status, 0) == child && WIFEXITED(status) && WEXITSTATUS(status) == 0);
+    Upstream sparse(readFile(sparsePath), readFile(audio)); sparse.start();
+    const auto sparseVideo = sparse.url("/source.ts"), sparseAudio = sparse.url("/audio.wav");
+    media = start({sparseVideo.c_str(), "X-Video: video-secret\r\n", sparseAudio.c_str(),
+        "X-Audio: audio-secret\r\n", false, false, false, 0}, url, duration);
+    const auto sparseBase = url.substr(0, url.rfind('/') + 1);
+    double audioEnd = NAN;
+    for (int id = 0; id < 5; ++id) {
+        const auto response = request(sparseBase + "segment/" + std::to_string(id) + ".ts");
+        const auto span = audioSpan(response, output);
+        if (id) assert(std::abs(span.first - audioEnd) < 1.0 / 48000);
+        audioEnd = span.second;
+        if (id == 0 || id >= 3) {
+            const auto rows = samples(output, 1);
+            const auto videos = std::count_if(rows.begin(), rows.end(), [](const auto &row) { return row.first.first == 1; });
+            assert(videos == (id == 0 ? 90 : id == 3 ? 180 : 30));
+        }
+    }
+    cast_media_destroy(media);
+}
+// Twelve-second, 30 fps video-only fixtures exercise each codec without
+// regenerating sources. Separate PCM audio also keeps the AAC lane covered.
+static void decodeWork(const std::filesystem::path &directory, const std::string &fixturePath) {
+#ifndef CAST_MEDIA_CHECK
+    assert(false && "Compile --decode-work with CAST_MEDIA_CHECK to observe actual decoding");
+#endif
+    AVFormatContext *format = nullptr;
+    ok(avformat_open_input(&format, fixturePath.c_str(), nullptr, nullptr)); ok(avformat_find_stream_info(format, nullptr));
+    const int videoIndex = av_find_best_stream(format, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0); assert(videoIndex >= 0);
+    const auto *video = format->streams[videoIndex]->codecpar;
+    const std::string codec = avcodec_get_name(video->codec_id);
+    avformat_close_input(&format);
+    Upstream upstream(readFile(fixturePath), wav()); upstream.start();
+    const auto videoURL = upstream.url("/source.ts"), audioURL = upstream.url("/audio.wav");
+    const auto output = (directory / "decode-output.ts").string();
+    const auto beforeDecode = decodeCounts();
+    double wallMS = 0, cpuMS = 0;
+    for (bool random : {false, true}) {
+        std::string url; double duration = 0;
+        auto *media = start({videoURL.c_str(), "X-Video: video-secret\r\n", audioURL.c_str(),
+            "X-Audio: audio-secret\r\n", false, false, false, 0}, url, duration);
+        assert(std::abs(duration - 12) < .1);
+        const auto base = url.substr(0, url.rfind('/') + 1);
+        std::map<int, std::pair<double, double>> spans;
+        for (int at = 0; at < 2; ++at) {
+            const int id = random ? 1 - at : at;
+            const auto began = std::chrono::steady_clock::now(); const auto cpu = std::clock();
+            const auto response = request(base + "segment/" + std::to_string(id) + ".ts");
+            wallMS += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - began).count();
+            cpuMS += 1000.0 * (std::clock() - cpu) / CLOCKS_PER_SEC;
+            assert(response.status == 200); writeFile(output, response.bytes);
+            const auto rows = samples(output, 1);
+            int videos = 0;
+            for (const auto &[key, value] : rows) {
+                assert(key.second >= id * 6 * 90000 && key.second < (id + 1) * 6 * 90000);
+                if (key.first) ++videos;
+            }
+            assert(videos == 180);
+            spans[id] = audioSpan(response, output);
+            assert(request(base + "segment/" + std::to_string(id) + ".ts").bytes == response.bytes);
+        }
+        assert(std::abs(spans.at(1).first - spans.at(0).second) < 1.0 / 48000);
+        cast_media_destroy(media);
+    }
+    assert(upstream.foreignRequests == 0);
+    std::cout << "decode-work codec=" << codec << " relay_24s_ms=" << wallMS << " relay_cpu_ms=" << cpuMS;
+    decodeUsage(beforeDecode);
+    std::cout << "\ncast-media: codec decode, independent IDRs, AAC continuity, sequential and random seeks passed\n";
+}
 int main(int argc, char **argv) {
     bool liveCheck = false, seekCheck = false, hlsCheck = false, remuxCheck = false;
     for (int argument = 1; argument < argc; ++argument) {
@@ -680,6 +837,18 @@ int main(int argc, char **argv) {
         if (std::string(argv[argument]) == "--seek-work") seekCheck = true;
         if (std::string(argv[argument]) == "--hls-work") hlsCheck = true;
         if (std::string(argv[argument]) == "--remux-work") remuxCheck = true;
+        if (std::string(argv[argument]) == "--decode-work") {
+            assert(argument + 1 < argc && "--decode-work requires a 12s/30fps fixture path");
+            char temp[] = "/tmp/terebi-decode-check.XXXXXX"; assert(mkdtemp(temp));
+            decodeWork(temp, argv[++argument]); std::filesystem::remove_all(temp);
+            return 0;
+        }
+        if (std::string(argv[argument]) == "--codec-work") {
+            char temp[] = "/tmp/terebi-codec-check.XXXXXX"; assert(mkdtemp(temp));
+            codecWork(temp); std::filesystem::remove_all(temp);
+            std::cout << "cast-media: sequential codec reuse, independent IDRs, AAC continuity and random seeks passed\n";
+            return 0;
+        }
     }
     char temp[] = "/tmp/terebi-cast-check.XXXXXX"; assert(mkdtemp(temp)); std::filesystem::path directory(temp);
     if (remuxCheck) {

Checks/video-decoding.mm

@@ -0,0 +1,275 @@
+// Build like seek-queue-check, with -fobjc-arc and the bundled FFmpeg flags,
+// linking the normal player objects EXCEPT player.o (the implementation is below).
+// Build the same source again with -DTEREBI_SOFTWARE_DECODE for the baseline.
+// Run: video-decoding-check [--metal | --preroll] [--frames 360] [--fps 30] fixture ...
+// Fixtures: video-only CFR testsrc2, zero-origin PTS, BT.709 SDR, square SAR.
+// Benchmark excludes opening, pixel checks, seeking and Metal; there is no pacing,
+// audio-device setup or playback clock. Wall/process CPU are measurements only.
+#include "../Sources/Core/player.cpp"
+#import <AppKit/AppKit.h>
+#import <MetalKit/MetalKit.h>
+extern "C" {
+#include <libavutil/pixdesc.h>
+}
+#include <cstdlib>
+#include <ctime>
+
+static void require(bool condition, const char *message) {
+    if (!condition) { std::fprintf(stderr, "FAIL: %s\n", message); std::exit(1); }
+}
+
+static Frame retain(const AVFrame *source) {
+    Frame frame(av_frame_clone(source));
+    if (!frame) throw std::bad_alloc();
+    return frame;
+}
+
+static void metadata(Player &player, const AVFrame *frame) {
+    const auto *parameters = player.tracks[1].stream()->codecpar;
+    require(frame->width == parameters->width && frame->height == parameters->height,
+            "readable frame preserves coded dimensions");
+    require(frame->color_primaries == AVCOL_PRI_BT709 && frame->color_trc == AVCOL_TRC_BT709 &&
+            frame->colorspace == AVCOL_SPC_BT709, "readable frame preserves BT.709 SDR metadata");
+    require(parameters->color_range == AVCOL_RANGE_UNSPECIFIED ||
+            frame->color_range == parameters->color_range, "readable frame preserves color range");
+    const AVRational aspect = av_guess_sample_aspect_ratio(player.format,
+        player.tracks[1].stream(), const_cast<AVFrame *>(frame));
+    require(aspect.num > 0 && aspect.num == aspect.den, "consumer sees square sample aspect ratio");
+    const auto *description = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(frame->format));
+    require(description && !(description->flags & AV_PIX_FMT_FLAG_HWACCEL),
+            "queued frames have a CPU-readable pixel format");
+}
+
+// Read every visible byte, not padding: this detects a mapped frame whose pixel
+// buffer was unlocked/recycled when a decoder frame, timeline or codec was freed.
+static uint64_t pixels(const AVFrame *frame) {
+    const auto format = static_cast<AVPixelFormat>(frame->format);
+    int widths[4]{};
+    require(av_image_fill_linesizes(widths, format, frame->width) >= 0, "readable plane widths");
+    ptrdiff_t strides[4]{};
+    for (int plane = 0; plane < 4; ++plane) strides[plane] = widths[plane];
+    size_t sizes[4]{};
+    require(av_image_fill_plane_sizes(sizes, format, frame->height, strides) >= 0,
+            "readable plane heights");
+    uint64_t hash = 14695981039346656037ULL;
+    unsigned low = 255, high = 0;
+    for (int plane = 0; plane < 4; ++plane) {
+        if (!sizes[plane]) continue;
+        require(widths[plane] > 0 && frame->data[plane] &&
+                std::abs(frame->linesize[plane]) >= widths[plane], "real readable pixel planes");
+        const size_t rows = sizes[plane] / widths[plane];
+        for (size_t y = 0; y < rows; ++y) {
+            const uint8_t *row = frame->data[plane] + ptrdiff_t(y) * frame->linesize[plane];
+            for (int x = 0; x < widths[plane]; ++x) {
+                low = std::min(low, unsigned(row[x])); high = std::max(high, unsigned(row[x]));
+                hash = (hash ^ row[x]) * 1099511628211ULL;
+            }
+        }
+    }
+    require(low < high, "testsrc2 contains real nonconstant pixel data");
+    return hash;
+}
+
+static void exactFrame(Player &player, const VideoFrame &frame, int index, int fps) {
+    require(bool(frame.frame), "consumer receives a video frame");
+    const auto timebase = player.tracks[1].stream()->time_base;
+    const int64_t expected = av_rescale_q(index, AVRational{1, fps}, timebase);
+    require(presentationTimestamp(frame.frame.get()) == expected, "exact consumer presentation timestamp");
+    require(std::abs(frame.pts - expected * av_q2d(timebase)) < 0.000001,
+            "queue PTS matches the fixture media timestamp");
+    require(frame.end > frame.pts, "queued frame retains a positive presentation duration");
+    metadata(player, frame.frame.get());
+}
+
+struct DecodedRun {
+    Frame first, last;
+    int frames = 0;
+    bool hardware = false;
+    size_t peakFrames = 0, peakBytes = 0;
+};
+
+static DecodedRun drain(Player &player, int first, int expectedFrames, int fps) {
+    DecodedRun result;
+    double previous = -std::numeric_limits<double>::infinity();
+    while (!player.tracks[1].drained || !player.video.empty()) {
+        auto *codec = player.tracks[1].codec;
+        result.hardware |= codec->pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX || codec->hw_frames_ctx != nullptr;
+        result.peakFrames = std::max(result.peakFrames, player.video.size());
+        result.peakBytes = std::max(result.peakBytes, player.videoBytes);
+        require(player.video.size() <= videoFrameLimit && player.videoBytes <= videoByteLimit,
+                "real decoded queue stays inside frame and byte budgets");
+        if (!player.video.empty() && (player.videoFull() || player.tracks[1].drained)) {
+            while (!player.video.empty()) {
+                VideoFrame frame = std::move(player.video.front());
+                player.video.pop_front();
+                player.videoBytes -= frame.bytes;
+                require(frame.pts > previous, "consumer observes strictly monotonic PTS including B-frame EOF");
+                require(result.frames < expectedFrames, "no extra decoded frames after decoder drain");
+                exactFrame(player, frame, first + result.frames, fps);
+                previous = frame.pts;
+                if (!result.first) result.first = retain(frame.frame.get());
+                result.last = std::move(frame.frame);
+                ++result.frames;
+            }
+        } else {
+            require(player.decodeOne(), "local synchronous decode makes progress without a clock");
+        }
+    }
+    require(result.frames == expectedFrames, "all expected frames survive queueing and delayed EOF drain");
+    require(player.demuxEnded && player.tracks[1].sourceEnded && player.tracks[1].drainSent &&
+            player.tracks[1].drained && player.videoBytes == 0, "demux and codec EOF fully drain the queue");
+    return result;
+}
+
+static void draw(playback::Output &output, const AVFrame *frame) {
+    void *native = output.make_video_view();
+    require(native != nullptr, "real Metal view and shaders initialize");
+    MTKView *view = (__bridge_transfer MTKView *)native;
+    view.paused = YES;
+    NSWindow *window = [[NSWindow alloc] initWithContentRect:NSMakeRect(100, 100, 320, 180)
+        styleMask:NSWindowStyleMaskBorderless backing:NSBackingStoreBuffered defer:NO];
+    window.contentView = view;
+    [window orderFront:nil];
+    output.present(frame);
+    // AppKit must attach the layer before a native drawable can be acquired.
+    // This is an I/O watchdog, not a media clock.
+    const auto deadline = Clock::now() + std::chrono::seconds(5);
+    while (!view.currentDrawable && Clock::now() < deadline)
+        CFRunLoopRunInMode(kCFRunLoopDefaultMode, .005, false);
+    require(view.currentDrawable != nil, "real Metal surface supplies a drawable");
+    [view draw];
+    [window orderOut:nil];
+}
+
+static void fixture(const char *path, int frames, int fps, bool metal, bool preroll) {
+    Player player;
+    player.shutdown(); // Join the worker: use actual open/decode/seek synchronously.
+    player.stopping.store(false);
+    player.activeEpoch = player.requestedEpoch.load();
+    Command command{Action::Open, path};
+    player.open(command);
+    require(player.tracks[1].codec && !player.tracks[0].codec, "fixture is video-only (no audio device)");
+    require(player.canSeek && std::abs(player.duration - double(frames) / fps) < .002,
+            "fixture has the fixed expected media duration and is seekable");
+    if (preroll) {
+        while (player.video.empty() && !player.tracks[1].drained)
+            require(player.decodeOne(), "opening transport stream makes progress");
+        require(!player.video.empty(), "transport stream supplies an opening picture");
+        auto retained = retain(player.video.front().frame.get());
+        const auto hash = pixels(retained.get());
+        const double target = (frames * 3 / 5 + .4) / fps;
+        player.seek(target, false);
+        while (player.video.empty() && !player.tracks[1].drained)
+            require(player.decodeOne(), "dependent-packet seek makes progress");
+        require(!player.video.empty() && player.video.front().pts >= target &&
+                player.video.front().pts < player.duration, "transport seek supplies a usable preview after preroll");
+        metadata(player, player.video.front().frame.get());
+        pixels(player.video.front().frame.get());
+        const bool software = player.tracks[1].codec->pix_fmt != AV_PIX_FMT_VIDEOTOOLBOX;
+        player.release();
+        require(pixels(retained.get()) == hash, "mapped opening picture survives preroll decoder replacement");
+        std::printf("PASS native dependent-packet preroll/preview/lifetime: %s software=%s\n",
+                    path, software ? "yes" : "no");
+        return;
+    }
+    const auto wallStart = std::chrono::steady_clock::now();
+    const auto cpuStart = std::clock();
+    auto decoded = drain(player, 0, frames, fps);
+    const double cpu = double(std::clock() - cpuStart) / CLOCKS_PER_SEC;
+    const double wall = std::chrono::duration<double>(std::chrono::steady_clock::now() - wallStart).count();
+#ifdef TEREBI_SOFTWARE_DECODE
+    constexpr const char *mode = "software";
+    require(!decoded.hardware, "matched software baseline never negotiates hardware");
+#else
+    constexpr const char *mode = "preferred";
+#endif
+    std::printf("native-decode mode=%s fixture=%s codec=%s videotoolbox=%s format=%s frames=%d "
+                "wall=%.6f cpu=%.6f peak_frames=%zu peak_bytes=%zu\n", mode, path,
+                player.tracks[1].codec->codec->name, decoded.hardware ? "yes" : "no",
+                av_get_pix_fmt_name(static_cast<AVPixelFormat>(decoded.first->format)), decoded.frames,
+                wall, cpu, decoded.peakFrames, decoded.peakBytes);
+    const auto firstHash = pixels(decoded.first.get());
+    const auto lastHash = pixels(decoded.last.get());
+    require(firstHash != lastHash, "retained first and last frames contain distinct fixture pictures");
+    if (metal) draw(player.output, decoded.first.get());
+
+    // A non-keyframe seek must discard preroll and expose the exact next PTS.
+    const int middle = frames / 2;
+    const auto seekStart = Clock::now();
+    player.seek((middle + .4) / fps, false);
+    require(player.video.empty() && player.videoBytes == 0, "seek clears the old readable queue");
+    while (player.video.empty() && !player.tracks[1].drained)
+        require(player.decodeOne(), "seek preroll makes progress");
+    std::printf("native-seek mode=%s fixture=%s preview_ms=%.3f\n", mode, path,
+        std::chrono::duration<double, std::milli>(Clock::now() - seekStart).count());
+    require(pixels(decoded.first.get()) == firstHash && pixels(decoded.last.get()) == lastHash,
+            "mapped frame references survive decoder flush");
+    auto sought = drain(player, middle + 1, frames - middle - 1, fps);
+    pixels(sought.first.get());
+
+    // Seek backward after EOF, then step in both directions without a clock.
+    player.seek((middle + .4) / fps, false);
+    player.step(1);
+    exactFrame(player, player.presented, middle + 2, fps);
+    player.step(-1);
+    exactFrame(player, player.presented, middle + 1, fps);
+    player.step(1);
+    exactFrame(player, player.presented, middle + 2, fps);
+    player.seek(1.4 / fps, false);
+    auto restarted = drain(player, 2, frames - 2, fps);
+    const double lastPts = presentationTimestamp(restarted.last.get()) *
+        av_q2d(player.tracks[1].stream()->time_base) - player.origin;
+    const int lastBytes = av_image_get_buffer_size(static_cast<AVPixelFormat>(restarted.last->format),
+        restarted.last->width, restarted.last->height, 1);
+    require(lastBytes > 0, "last readable frame has a valid byte size");
+    player.presentVideo(VideoFrame{retain(restarted.last.get()), lastPts,
+        lastPts + player.videoFrameTime, static_cast<size_t>(lastBytes)});
+    player.step(1);
+    exactFrame(player, player.presented, frames - 1, fps);
+    player.step(-1);
+    exactFrame(player, player.presented, frames - 2, fps);
+    player.step(1);
+    exactFrame(player, player.presented, frames - 1, fps);
+
+    player.release(); // Retained mapped planes must also outlive the codec/device.
+    require(pixels(decoded.first.get()) == firstHash && pixels(decoded.last.get()) == lastHash,
+            "mapped frame references survive full decoder release and pool reuse");
+    if (metal) draw(player.output, decoded.last.get());
+    std::printf("PASS native readable frames/SDR/SAR/seek/exact steps/EOF/lifetimes: %s\n", path);
+}
+
+int main(int argc, char **argv) {
+    @autoreleasepool {
+        int frames = 360, fps = 30;
+        bool metal = false, preroll = false;
+        std::vector<const char *> paths;
+        for (int i = 1; i < argc; ++i) {
+            if (std::strcmp(argv[i], "--metal") == 0) { metal = true; continue; }
+            if (std::strcmp(argv[i], "--preroll") == 0) { preroll = true; continue; }
+            if (std::strcmp(argv[i], "--frames") == 0 || std::strcmp(argv[i], "--fps") == 0) {
+                const bool count = std::strcmp(argv[i], "--frames") == 0;
+                require(++i < argc, "numeric option needs a value");
+                char *end = nullptr;
+                const long value = std::strtol(argv[i], &end, 10);
+                require(end != argv[i] && !*end && value > 0 && value <= 1000000,
+                        "frame count and integer frame rate must be positive and bounded");
+                (count ? frames : fps) = static_cast<int>(value);
+            } else {
+                require(argv[i][0] != '-', "unknown option");
+                paths.push_back(argv[i]);
+            }
+        }
+        require(!paths.empty() && frames >= 8, "provide fixtures with at least eight frames");
+        if (metal) {
+            [NSApplication sharedApplication];
+            [NSApp setActivationPolicy:NSApplicationActivationPolicyAccessory];
+            [NSApp finishLaunching];
+        }
+        try {
+            for (const char *path : paths) fixture(path, frames, fps, metal, preroll);
+        } catch (const std::exception &error) {
+            std::fprintf(stderr, "FAIL: %s\n", error.what());
+            return 1;
+        }
+    }
+}

Makefile

@@ -26,8 +26,9 @@ BOOTSTRAP := $(APP)/Contents/MacOS/terebi
 CLI := $(APP)/Contents/MacOS/terebi-cli
 BUNDLE_STAMP := $(BUILD_DIR)/bundle.stamp
 SIGNING_IDENTITY ?= -
+DECODE_FIXTURES := $(addprefix $(BUILD_DIR)/fixtures/decode-,h264.mp4 hevc.mp4 vp9.mp4 av1.mp4 h264-10.mp4 mpeg4.mp4)
 
-.PHONY: all release install check-dependencies run check check-seek-queue check-config check-fixtures check-audio check-window check-youtube check-quality check-adaptive check-network check-progress check-reporting check-cli check-runtime check-media check-startup check-bundle check-subtitles check-resolved check-cast
+.PHONY: all release install check-dependencies run check check-seek-queue check-video-decoding check-config check-fixtures check-audio check-window check-youtube check-quality check-adaptive check-network check-progress check-reporting check-cli check-runtime check-media check-startup check-bundle check-subtitles check-resolved check-cast
 all: $(BUILD_DIR)/terebi
 
 $(NATIVE_STAMP): Scripts/dependencies.py Dependencies.lock.json
@@ -62,13 +63,13 @@ $(APP)/Contents/Resources/TOMLDecoder-LICENSE.txt: $(CONFIG_LIBRARY)
 	mkdir -p "$(APP)/Contents/Resources"
 	cp -f "$(CONFIG_BUILD)/checkouts/TOMLDecoder/LICENSE.md" "$@"
 
-$(BUILD_DIR)/player.o: Sources/Core/player.cpp Sources/Core/player.h Sources/Core/player_macos.h Sources/Core/output.hpp Sources/Core/network_input.hpp Sources/Core/network_policy.hpp Sources/Core/progress_reporter.hpp Makefile $(NATIVE_STAMP) | $(BUILD_DIR) check-dependencies
+$(BUILD_DIR)/player.o: Sources/Core/player.cpp Sources/Core/player.h Sources/Core/player_macos.h Sources/Core/output.hpp Sources/Core/video_decoder.hpp Sources/Core/network_input.hpp Sources/Core/network_policy.hpp Sources/Core/progress_reporter.hpp Makefile $(NATIVE_STAMP) | $(BUILD_DIR) check-dependencies
 	$(CXX) $(CXXFLAGS) -c $< -o $@
 
 $(BUILD_DIR)/progress_reporter.o: Sources/Core/progress_reporter.cpp Sources/Core/progress_reporter.hpp Sources/Core/player.h Makefile $(NATIVE_STAMP) | $(BUILD_DIR)
 	$(CXX) $(CXXFLAGS) -c $< -o $@
 
-$(BUILD_DIR)/cast_media.o: Sources/Core/cast_media.cpp Sources/Core/cast_media.h Sources/Core/network_policy.hpp Sources/Core/network_input.hpp Makefile $(NATIVE_STAMP) | $(BUILD_DIR) check-dependencies
+$(BUILD_DIR)/cast_media.o: Sources/Core/cast_media.cpp Sources/Core/cast_media.h Sources/Core/video_decoder.hpp Sources/Core/network_policy.hpp Sources/Core/network_input.hpp Makefile $(NATIVE_STAMP) | $(BUILD_DIR) check-dependencies
 	$(CXX) $(CXXFLAGS) -c $< -o $@
 
 $(BUILD_DIR)/Output.o: Sources/macOS/Output.mm Sources/Core/output.hpp Makefile $(NATIVE_STAMP) | $(BUILD_DIR) check-dependencies
@@ -105,13 +106,41 @@ run: all
 $(BUILD_DIR)/playback-check: Checks/playback.mm $(OBJECTS) Makefile | check-dependencies
 	$(CXX) $(CXXFLAGS) -fobjc-arc $< $(OBJECTS) $(FF_LIBS) $(FRAMEWORKS) -o $@
 
-$(BUILD_DIR)/seek-queue-check: Checks/seek-queue.cpp Sources/Core/player.cpp Sources/Core/player.h Sources/Core/player_macos.h Sources/Core/output.hpp Sources/Core/network_input.hpp Sources/Core/network_policy.hpp Sources/Core/progress_reporter.hpp $(filter-out $(BUILD_DIR)/player.o,$(OBJECTS)) Makefile | check-dependencies
+$(BUILD_DIR)/video-decoding-check: Checks/video-decoding.mm Sources/Core/player.cpp Sources/Core/player.h Sources/Core/player_macos.h Sources/Core/output.hpp Sources/Core/video_decoder.hpp Sources/Core/network_input.hpp Sources/Core/network_policy.hpp Sources/Core/progress_reporter.hpp $(filter-out $(BUILD_DIR)/player.o,$(OBJECTS)) Makefile | check-dependencies
+	$(CXX) $(CXXFLAGS) -fobjc-arc $< $(filter-out $(BUILD_DIR)/player.o,$(OBJECTS)) $(FF_LIBS) $(FRAMEWORKS) -o $@
+
+$(BUILD_DIR)/video-decoding-software-check: Checks/video-decoding.mm Sources/Core/player.cpp Sources/Core/player.h Sources/Core/player_macos.h Sources/Core/output.hpp Sources/Core/video_decoder.hpp Sources/Core/network_input.hpp Sources/Core/network_policy.hpp Sources/Core/progress_reporter.hpp $(filter-out $(BUILD_DIR)/player.o,$(OBJECTS)) Makefile | check-dependencies
+	$(CXX) $(CXXFLAGS) -DTEREBI_SOFTWARE_DECODE -fobjc-arc $< $(filter-out $(BUILD_DIR)/player.o,$(OBJECTS)) $(FF_LIBS) $(FRAMEWORKS) -o $@
+
+$(BUILD_DIR)/cast-media-software-check: Checks/cast-media.cpp Sources/Core/cast_media.cpp Sources/Core/cast_media.h Sources/Core/video_decoder.hpp Sources/Core/network_policy.hpp Sources/Core/network_input.hpp Makefile | check-dependencies
+	$(CXX) $(CXXFLAGS) -ISources/Core -DCAST_MEDIA_CHECK -DTEREBI_SOFTWARE_DECODE Checks/cast-media.cpp Sources/Core/cast_media.cpp $(FF_LIBS) -o $@
+
+check-video-decoding: $(BUILD_DIR)/video-decoding-check $(BUILD_DIR)/cast-media-check $(DECODE_FIXTURES) $(BUILD_DIR)/fixtures/decode-preroll.ts
+	$(BUILD_DIR)/video-decoding-check --metal $(DECODE_FIXTURES)
+	$(BUILD_DIR)/video-decoding-check --preroll $(BUILD_DIR)/fixtures/decode-preroll.ts
+	for fixture in $(DECODE_FIXTURES); do $(BUILD_DIR)/cast-media-check --decode-work "$$fixture" || exit; done
+
+$(DECODE_FIXTURES): Makefile | check-dependencies
+	mkdir -p $(BUILD_DIR)/fixtures
+	ffmpeg -hide_banner -loglevel error -y -f lavfi -i testsrc2=size=1280x720:rate=30:duration=12 -vf setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709 -an -threads 2 $(DECODE_ENCODER) $@
+
+$(BUILD_DIR)/fixtures/decode-h264.mp4: DECODE_ENCODER = -c:v libx264 -preset ultrafast -bf 2 -g 60
+$(BUILD_DIR)/fixtures/decode-hevc.mp4: DECODE_ENCODER = -c:v libx265 -preset ultrafast -x265-params log-level=error:pools=2 -g 60
+$(BUILD_DIR)/fixtures/decode-vp9.mp4: DECODE_ENCODER = -c:v libvpx-vp9 -deadline realtime -cpu-used 8 -row-mt 1 -g 60
+$(BUILD_DIR)/fixtures/decode-av1.mp4: DECODE_ENCODER = -c:v libsvtav1 -preset 12 -svtav1-params lp=2 -g 60
+$(BUILD_DIR)/fixtures/decode-h264-10.mp4: DECODE_ENCODER = -c:v libx264 -preset ultrafast -pix_fmt yuv420p10le -bf 2 -g 60
+$(BUILD_DIR)/fixtures/decode-mpeg4.mp4: DECODE_ENCODER = -c:v mpeg4 -bf 2 -g 60
+
+$(BUILD_DIR)/fixtures/decode-preroll.ts: $(BUILD_DIR)/fixtures/decode-h264.mp4 Makefile
+	ffmpeg -hide_banner -loglevel error -y -i $< -map 0:v:0 -c copy -mpegts_copyts 1 -muxdelay 0 -avoid_negative_ts disabled $@
+
+$(BUILD_DIR)/seek-queue-check: Checks/seek-queue.cpp Sources/Core/player.cpp Sources/Core/player.h Sources/Core/player_macos.h Sources/Core/output.hpp Sources/Core/video_decoder.hpp Sources/Core/network_input.hpp Sources/Core/network_policy.hpp Sources/Core/progress_reporter.hpp $(filter-out $(BUILD_DIR)/player.o,$(OBJECTS)) Makefile | check-dependencies
 	$(CXX) $(CXXFLAGS) $< $(filter-out $(BUILD_DIR)/player.o,$(OBJECTS)) $(FF_LIBS) $(FRAMEWORKS) -o $@
 
 check-seek-queue: $(BUILD_DIR)/seek-queue-check
 	$(BUILD_DIR)/seek-queue-check
 
-check: check-dependencies $(BUILD_DIR)/playback-check check-seek-queue check-fixtures check-audio check-window check-config check-youtube check-quality check-adaptive check-network check-progress check-reporting check-cli check-runtime check-media check-startup check-bundle check-subtitles check-resolved check-cast
+check: check-dependencies $(BUILD_DIR)/playback-check check-seek-queue check-video-decoding check-fixtures check-audio check-window check-config check-youtube check-quality check-adaptive check-network check-progress check-reporting check-cli check-runtime check-media check-startup check-bundle check-subtitles check-resolved check-cast
 	$(BUILD_DIR)/playback-check $(BUILD_DIR)/fixtures
 
 check-fixtures: | check-dependencies
@@ -236,7 +265,7 @@ check-media: $(BUILD_DIR)/system-playback-check check-fixtures
 $(BUILD_DIR)/cast-session-check: Checks/cast-session.swift Sources/App/CastSession.swift Makefile | $(BUILD_DIR)
 	$(SWIFT) -target $(ARCH)-apple-macosx15.0 -swift-version 6 -parse-as-library -DCAST_SESSION_CHECKS -O $(DEBUG_FLAGS) Checks/cast-session.swift Sources/App/CastSession.swift -framework Foundation -framework Network -framework Security -o $@
 
-$(BUILD_DIR)/cast_media-check.o: Sources/Core/cast_media.cpp Sources/Core/cast_media.h Sources/Core/network_policy.hpp Sources/Core/network_input.hpp Makefile $(NATIVE_STAMP) | $(BUILD_DIR) check-dependencies
+$(BUILD_DIR)/cast_media-check.o: Sources/Core/cast_media.cpp Sources/Core/cast_media.h Sources/Core/video_decoder.hpp Sources/Core/network_policy.hpp Sources/Core/network_input.hpp Makefile $(NATIVE_STAMP) | $(BUILD_DIR) check-dependencies
 	$(CXX) $(CXXFLAGS) -DCAST_MEDIA_CHECK -c $< -o $@
 
 $(BUILD_DIR)/cast-media-check: Checks/cast-media.cpp $(BUILD_DIR)/cast_media-check.o Makefile | check-dependencies
@@ -249,6 +278,7 @@ check-cast: $(BUILD_DIR)/cast-session-check $(BUILD_DIR)/cast-media-check $(BUIL
 	$(BUILD_DIR)/cast-session-check
 	$(BUILD_DIR)/cast-media-check --live
 	$(BUILD_DIR)/cast-media-check --remux-work
+	$(BUILD_DIR)/cast-media-check --codec-work
 	$(BUILD_DIR)/cast-session-check --playback $(BUILD_DIR)/cast-playback-check $(BUILD_DIR)/fixtures/cache.mp4
 
 $(BUILD_DIR)/app-startup-check.dylib: Checks/app-startup.swift $(SWIFT_SOURCES) $(OBJECTS) $(CONFIG_LIBRARY) Makefile | check-dependencies

README.md

@@ -124,7 +124,7 @@ Settings load once at startup. Menu changes are session-only and never write the
 ## Limits
 
 - SDR output only; PQ/HLG HDR is rejected. No playlists, embedded/YouTube captions, manual audio-track selection, rotation, or deinterlacing. External resolved-source WebVTT subtitles are supported.
-- Video decoding is software-only. Backward frame stepping can be slow on long-GOP or unindexed media.
+- H.264, HEVC, and VP9 prefer VideoToolbox decoding with software fallback; other codecs, including AV1, use software. Backward frame stepping can be slow on long-GOP or unindexed media.
 - Network buffering is bounded and memory-only; live/non-seekable sources disable seeking and frame stepping. Expired stream URLs are not automatically refreshed.
 - Audio-device changes during playback are not handled.
 

Sources/Core/cast_media.cpp

@@ -1,6 +1,7 @@
 #include "cast_media.h"
 #include "network_policy.hpp"
 #include "network_input.hpp"
+#include "video_decoder.hpp"
 extern "C" {
 #include <libavformat/avformat.h>
 #include <libavcodec/avcodec.h>
@@ -40,6 +41,9 @@ using Clock = std::chrono::steady_clock;
 constexpr double segmentSeconds = 6;
 constexpr size_t segmentLimit = 32 * 1024 * 1024;
 constexpr size_t windowSegments = 8;
+#ifdef CAST_MEDIA_CHECK
+std::atomic<uint64_t> hardwareVideoFrames{0}, softwareVideoFrames{0};
+#endif
 void check(int value) { if (value < 0) throw std::runtime_error("Cast media operation failed (private details redacted)."); }
 void require(bool value) { if (!value) throw std::runtime_error("Cast media is unavailable or unsupported (private details redacted)."); }
 void privateLog(void *, int level, const char *, va_list) {
@@ -55,26 +59,6 @@ Frame frame() { auto p = Frame(av_frame_alloc()); require(bool(p)); return p; }
 Packet packet() { auto p = Packet(av_packet_alloc()); require(bool(p)); return p; }
 int64_t aacBoundary(double seconds) { return int64_t(std::ceil(seconds * 48000 / 1024)) * 1024; }
 double aacPreroll(double seconds) { return double(std::max<int64_t>(0, aacBoundary(seconds) - 12 * 1024)) / 48000; }
-bool hasIDR(const AVPacket &packet, const AVCodecParameters &codec) {
-    const auto *bytes = packet.data;
-    const size_t size = size_t(std::max(0, packet.size));
-    if (codec.extradata_size >= 5 && codec.extradata[0] == 1) {
-        const size_t lengthSize = (codec.extradata[4] & 3) + 1;
-        for (size_t at = 0; at + lengthSize <= size;) {
-            uint32_t length = 0;
-            for (size_t i = 0; i < lengthSize; ++i) length = (length << 8) | bytes[at++];
-            if (!length || length > size - at) return false;
-            if ((bytes[at] & 31) == 5) return true;
-            at += length;
-        }
-        return false;
-    }
-    for (size_t at = 0; at + 3 < size; ++at) {
-        if (bytes[at] == 0 && bytes[at + 1] == 0 && bytes[at + 2] == 1 &&
-            (bytes[at + 3] & 31) == 5) return true;
-    }
-    return false;
-}
 struct Source {
     std::string video, videoHeaders, audio, audioHeaders;
     bool local, restricted, live;
@@ -234,7 +218,7 @@ struct Track {
             const auto rate = av_guess_frame_rate(input->format, stream, nullptr);
             const double openingGap = rate.num > 0 ? av_q2d(av_inv_q(rate)) : 0;
             remux = remux && !live && pending && (pending->flags & AV_PKT_FLAG_KEY) &&
-                hasIDR(*pending, *stream->codecpar) &&
+                playback::hasIDR(*pending, *stream->codecpar) &&
                 (std::abs(packetTime() - start) < .001 ||
                  (start == 0 && packetTime() >= 0 && packetTime() <= openingGap));
             if (aligned && !remux) {
@@ -243,7 +227,7 @@ struct Track {
             }
         }
         if (!live && type == AVMEDIA_TYPE_VIDEO && pending && (pending->flags & AV_PKT_FLAG_KEY) &&
-            !hasIDR(*pending, *stream->codecpar) && stream->codecpar->codec_id == AV_CODEC_ID_H264) {
+            !playback::hasIDR(*pending, *stream->codecpar) && stream->codecpar->codec_id == AV_CODEC_ID_H264) {
             // ponytail: rare intra-refresh seeks decode from the beginning; index real IDRs if this becomes slow.
             input->seek(0);
             nextPacket();
@@ -258,6 +242,9 @@ struct Track {
                 decoder->pkt_timebase = stream->time_base;
                 decoder->thread_count = 2;
                 decoder->max_pixels = 4096LL * 2160;
+                if (type == AVMEDIA_TYPE_VIDEO && pending &&
+                    playback::independentVideoPacket(*pending, *stream->codecpar))
+                    playback::preferVideoToolbox(decoder);
                 check(avcodec_open2(decoder, codec, nullptr));
             } catch (...) { avcodec_free_context(&decoder); throw; }
         }
@@ -294,6 +281,10 @@ struct Track {
             require(!input->interrupted());
             int result = avcodec_receive_frame(decoder, f.get());
             if (result == 0) {
+#ifdef CAST_MEDIA_CHECK
+                if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
+                    (f->format == AV_PIX_FMT_VIDEOTOOLBOX ? hardwareVideoFrames : softwareVideoFrames).fetch_add(1, std::memory_order_relaxed);
+#endif
                 decoded = std::move(f);
                 const double step = stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO ? double(decoded->nb_samples) / decoded->sample_rate : 1.0 / 30;
                 nextTime = frameTime() + step;
@@ -337,14 +328,18 @@ struct OutputTrack {
     AVStream *stream = nullptr;
     AVCodecContext *encoder = nullptr;
     SwsContext *scale = nullptr;
+    Frame readable;
     SwrContext *resample = nullptr;
     AVAudioFifo *fifo = nullptr;
     int64_t audioPTS = AV_NOPTS_VALUE;
     bool firstVideo = true;
     double lastVideoTime = -INFINITY;
-    bool continuous;
+    bool live, drained = false;
-    int64_t audioKeepFrom = AV_NOPTS_VALUE, audioKeepUntil = INT64_MAX;
+    std::deque<Packet> packets;
-    OutputTrack(Track &track, Mux &output, bool live) : source(track), mux(&output), continuous(live) {
+    size_t packetBytes = 0;
+    int64_t lastPacketPTS = AV_NOPTS_VALUE;
+    int64_t keepFrom = INT64_MIN, keepUntil = INT64_MAX;
+    OutputTrack(Track &track, Mux &output, bool live) : source(track), mux(&output), live(live) {
         try {
         if (!source.stream) return;
         stream = avformat_new_stream(mux->format, nullptr); require(stream != nullptr);
@@ -397,7 +392,6 @@ struct OutputTrack {
             stream->time_base = encoder->time_base;
             check(avcodec_parameters_from_context(stream->codecpar, encoder));
         }
-        firstVideo = true;
     }
     void writePacket(AVPacket *packet) {
         // A shared, positive TS clock origin prevents per-segment AAC priming shifts.
@@ -413,11 +407,17 @@ struct OutputTrack {
             int result = avcodec_receive_packet(encoder, p.get());
             if (result == AVERROR(EAGAIN) || result == AVERROR_EOF) break;
             check(result);
-            if (fifo && !continuous && audioKeepFrom != AV_NOPTS_VALUE &&
+            lastPacketPTS = p->pts;
-                (p->pts < audioKeepFrom || p->pts >= audioKeepUntil)) {
+            if (!live) {
-                av_packet_unref(p.get());
+                require(p->pts != AV_NOPTS_VALUE && p->size >= 0);
+                if (p->pts >= keepUntil) {
+                    require(packetBytes + size_t(p->size) <= segmentLimit);
+                    packetBytes += size_t(p->size);
+                    packets.push_back(std::move(p)); p = packet();
                     continue;
                 }
+                if (p->pts < keepFrom) { av_packet_unref(p.get()); continue; }
+            }
             av_packet_rescale_ts(p.get(), encoder->time_base, stream->time_base);
             writePacket(p.get()); av_packet_unref(p.get());
         }
@@ -455,20 +455,22 @@ struct OutputTrack {
                 auto out = frame(); out->format = encoder->pix_fmt; out->width = encoder->width; out->height = encoder->height;
                 out->sample_aspect_ratio = encoder->sample_aspect_ratio;
                 check(av_frame_get_buffer(out.get(), 32));
+                if (f->format == AV_PIX_FMT_VIDEOTOOLBOX) {
+                    if (!readable) readable = frame();
+                    check(playback::refVideoFrame(readable.get(), f));
+                    f = readable.get();
+                }
                 scale = sws_getCachedContext(scale, f->width, f->height, AVPixelFormat(f->format), out->width, out->height, encoder->pix_fmt, SWS_BILINEAR, nullptr, nullptr, nullptr);
                 require(scale != nullptr); check(sws_scale(scale, f->data, f->linesize, 0, f->height, out->data, out->linesize));
+                if (readable) av_frame_unref(readable.get());
                 out->pts = int64_t(std::llround(std::max(start, time) * 90000));
                 out->pict_type = firstVideo ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_NONE; firstVideo = false; encoded(out.get());
                 lastVideoTime = time;
             }
         } else {
             require(f->sample_rate > 0 && f->nb_samples >= 0 && f->nb_samples <= 65536 && f->ch_layout.nb_channels <= 32);
-            const double begin = continuous ? start : aacPreroll(start);
+            const double begin = live ? start : aacPreroll(start);
-            const double limit = continuous ? end : double(aacBoundary(end)) / 48000;
+            const double limit = end;
-            if (!continuous) {
-                audioKeepFrom = aacBoundary(start);
-                audioKeepUntil = int64_t(std::llround(end * 48000));
-            }
             if (time + double(f->nb_samples) / f->sample_rate > begin && time < limit) {
                 if (!resample) {
                     check(swr_alloc_set_opts2(&resample, &encoder->ch_layout, encoder->sample_fmt, 48000, &f->ch_layout, AVSampleFormat(f->format), f->sample_rate, 0, nullptr));
@@ -479,10 +481,9 @@ struct OutputTrack {
                 auto out = frame(); out->format = encoder->sample_fmt; out->sample_rate = 48000;
                 check(av_channel_layout_copy(&out->ch_layout, &encoder->ch_layout)); out->nb_samples = capacity; check(av_frame_get_buffer(out.get(), 0));
                 int count = swr_convert(resample, out->data, capacity, const_cast<const uint8_t **>(f->extended_data), f->nb_samples); check(count);
-                int skip = continuous && audioPTS != AV_NOPTS_VALUE ? 0 :
+                int skip = audioPTS != AV_NOPTS_VALUE ? 0 :
                     std::clamp(int(std::llround((begin - time) * 48000)), 0, count);
-                int take = continuous ? count - skip :
+                int take = count - skip;
-                    std::clamp(int(std::llround((limit - time) * 48000)) - skip, 0, count - skip);
                 if (audioPTS == AV_NOPTS_VALUE) audioPTS = int64_t(std::llround(std::max(begin, time) * 48000));
                 uint8_t *data[2] = {out->data[0] + skip * 4, out->data[1] + skip * 4};
                 require(av_audio_fifo_size(fifo) + take <= 131072);
@@ -491,13 +492,52 @@ struct OutputTrack {
         }
         source.nextFrame();
     }
-    void finish(double end) {
+    void beginPackets(double start, double end) {
+        if (!encoder || live) return;
+        keepFrom = fifo ? aacBoundary(start) : int64_t(std::llround(start * 90000));
+        keepUntil = fifo ? int64_t(std::llround(end * 48000)) : int64_t(std::llround(end * 90000));
+        while (!packets.empty() && packets.front()->pts < keepUntil) {
+            auto p = std::move(packets.front()); packets.pop_front();
+            packetBytes -= size_t(p->size);
+            if (p->pts < keepFrom) continue;
+            av_packet_rescale_ts(p.get(), encoder->time_base, stream->time_base);
+            writePacket(p.get());
+        }
+    }
+    void finish(double start, double end) {
         if (!encoder) return;
-        if (continuous) {
+        if (live) {
-            if (fifo) audioFrames(false, end);
+            if (fifo) audioFrames(false, end); else encoded(nullptr);
-        } else {
+            return;
-            if (fifo) audioFrames(true);
+        }
-            encoded(nullptr);
+        // Neither encoder can resume after a drain. Feed just enough future
+        // input to receive this segment's delayed packets; retain the rest.
+        const int64_t target = fifo ? aacBoundary(end) - encoder->frame_size :
+            (std::isfinite(lastVideoTime) ? int64_t(std::llround(lastVideoTime * 90000)) : AV_NOPTS_VALUE);
+        if (!fifo) firstVideo = true;
+        while (!drained && target != AV_NOPTS_VALUE && (lastPacketPTS == AV_NOPTS_VALUE || lastPacketPTS < target)) {
+            if (fifo) audioFrames(false);
+            if (lastPacketPTS != AV_NOPTS_VALUE && lastPacketPTS >= target) break;
+            const double time = source.time();
+            // ponytail: cap lookahead at six seconds; drain and reset on the
+            // next segment rather than following a sparse stream arbitrarily far.
+            if (!std::isfinite(time) || time >= end + segmentSeconds) {
+                if (fifo) {
+                    // Drain resampler delay before padding the final AAC frame.
+                    if (resample) {
+                        auto tail = frame(); tail->format = encoder->sample_fmt;
+                        tail->nb_samples = 4096;
+                        check(av_channel_layout_copy(&tail->ch_layout, &encoder->ch_layout));
+                        check(av_frame_get_buffer(tail.get(), 0));
+                        int count = swr_convert(resample, tail->data, tail->nb_samples, nullptr, 0); check(count);
+                        require(av_audio_fifo_write(fifo, reinterpret_cast<void **>(tail->data), count) == count);
+                    }
+                    audioFrames(true);
+                }
+                encoded(nullptr); drained = true;
+                break;
+            }
+            consume(fifo ? start : end, INFINITY);
         }
     }
 };
@@ -513,6 +553,7 @@ struct Pipeline {
     std::atomic<bool> &cancel;
     double duration = 0;
     std::vector<double> boundaries;
+    double nextStart = 0;
     Pipeline(const Source &source, std::atomic<bool> &stop, double start) : live(source.live), cancel(stop) {
         videoInput = std::make_unique<Input>(stop, source.video, source.videoHeaders, source.local, source.restricted, source.live);
         duration = std::max(source.duration, videoInput->duration());
@@ -567,8 +608,8 @@ struct Pipeline {
     void armOperation() {
         deadline = Clock::now() + std::chrono::seconds(45);
         const auto limit = std::chrono::duration_cast<std::chrono::nanoseconds>(deadline.time_since_epoch()).count();
-        videoInput->operationDeadline = limit;
+        videoInput->operationDeadline = limit; videoInput->arm();
-        if (audioInput) audioInput->operationDeadline = limit;
+        if (audioInput) { audioInput->operationDeadline = limit; audioInput->arm(); }
     }
     void releaseOperation() {
         videoInput->operationDeadline = std::numeric_limits<int64_t>::max();
@@ -578,8 +619,8 @@ struct Pipeline {
         armOperation();
         videoOutput.reset(); audioOutput.reset();
         video.reset(); audio.reset();
-        // Fresh codecs retain independent IDRs/AAC priming. One recorded network
+        // Random access resets codec history; the network demuxer rewinds both
-        // demuxer atomically rewinds both packet lanes, seeking upstream only on misses.
+        // packet lanes atomically, seeking upstream only on cache misses.
         const double audioTarget = audioIndex >= 0 && !remuxAudio(*audioSource->format->streams[audioIndex]->codecpar) ?
             aacPreroll(start) : NAN;
         if (videoInput->buffered)
@@ -587,13 +628,15 @@ struct Pipeline {
         if (audioInput && audioInput->buffered) audioInput->seek(start, audioTarget);
         if (videoIndex >= 0) video = std::make_unique<Track>(*videoInput, videoIndex, AVMEDIA_TYPE_VIDEO, start, live, aligned);
         if (audioIndex >= 0) audio = std::make_unique<Track>(*audioSource, audioIndex, AVMEDIA_TYPE_AUDIO, start, live);
+        nextStart = start;
     }
     Segment segment(int64_t id, double start, double end) {
-        if (live) armOperation();
+        armOperation();
         Mux mux;
         if (video) {
-            videoOutput.reset();
+            if (live) videoOutput.reset();
-            videoOutput = std::make_unique<OutputTrack>(*video, mux, false);
+            if (videoOutput) videoOutput->beginSegment(mux);
+            else videoOutput = std::make_unique<OutputTrack>(*video, mux, live);
         }
         if (audio) {
             if (audioOutput) audioOutput->beginSegment(mux);
@@ -604,6 +647,8 @@ struct Pipeline {
         av_dict_set(&options, "mpegts_copyts", "1", 0);
         av_dict_set(&options, "muxdelay", "0", 0);
         int result = avformat_write_header(mux.format, &options); av_dict_free(&options); check(result);
+        if (videoOutput) videoOutput->beginPackets(start, end);
+        if (audioOutput) audioOutput->beginPackets(start, end);
         const double audioEnd = audio && !audio->remux && !live ? double(aacBoundary(end)) / 48000 : end;
         for (;;) {
             require(!cancel && Clock::now() < deadline);
@@ -613,12 +658,13 @@ struct Pipeline {
             if (!std::isfinite(vt) && !std::isfinite(at)) break;
             if (vt <= at) videoOutput->consume(start, end); else audioOutput->consume(start, end);
         }
-        if (videoOutput) videoOutput->finish(end);
+        if (videoOutput) videoOutput->finish(start, end);
-        if (audioOutput) audioOutput->finish(end);
+        if (audioOutput) audioOutput->finish(start, end);
         check(av_write_trailer(mux.format)); avio_flush(mux.io);
         require(mux.io->error >= 0 && !mux.bytes.empty());
         if (videoInput->buffered) videoInput->buffered->updatePosition(end);
         if (audioInput && audioInput->buffered) audioInput->buffered->updatePosition(end);
+        nextStart = end;
         releaseOperation();
         return {id, end - start, std::move(mux.bytes)};
     }
@@ -740,7 +786,7 @@ struct CastMedia {
         for (int64_t id = first; id < first + count; ++id) {
             char value[64]; double length = source.live ? segments[size_t(id - first)]->duration : boundary(id + 1) - boundary(id);
             std::snprintf(value, sizeof(value), "#EXTINF:%.6f,\n", length);
-            // Each independently generated segment resets codec/mux state.
+            // Each segment retains an independent mux and random-access entry.
             if (id > first) text += "#EXT-X-DISCONTINUITY\n";
             text += value; text += "segment/" + std::to_string(id) + ".ts\n";
         }
@@ -761,8 +807,19 @@ struct CastMedia {
         if (auto segment = cached()) return segment;
         try {
             const double start = boundary(id), end = boundary(id + 1);
-            if (vodPipeline) vodPipeline->reset(start, !boundaries.empty());
+            if (vodPipeline) {
-            else vodPipeline = std::make_unique<Pipeline>(source, cancelled, start);
+                const auto &video = vodPipeline->video;
+                // Fixed-grid remux is safe only when the next boundary is an
+                // actual IDR. Otherwise reset once into the transcode path.
+                const bool independent = !video || !video->remux ||
+                    (video->pending && std::abs(video->packetTime() - start) < .001 &&
+                     (video->pending->flags & AV_PKT_FLAG_KEY) &&
+                     playback::hasIDR(*video->pending, *video->stream->codecpar));
+                const bool drained = (vodPipeline->videoOutput && vodPipeline->videoOutput->drained) ||
+                    (vodPipeline->audioOutput && vodPipeline->audioOutput->drained);
+                if (vodPipeline->nextStart != start || !independent || drained)
+                    vodPipeline->reset(start, !boundaries.empty());
+            } else vodPipeline = std::make_unique<Pipeline>(source, cancelled, start);
             auto segment = std::make_shared<const Segment>(vodPipeline->segment(id, start, end));
             { std::lock_guard guard(lock); segments.push_back(segment); while (segments.size() > windowSegments) segments.pop_front(); }
             return segment;
@@ -912,6 +969,10 @@ extern "C" bool cast_media_subtitle(CastMedia *media, int id, const void *bytes,
 extern "C" void cast_media_cancel(CastMedia *media) { if (media) media->cancel(); }
 extern "C" void cast_media_destroy(CastMedia *media) { delete media; }
 #ifdef CAST_MEDIA_CHECK
+extern "C" void cast_media_check_decode_counts(uint64_t *hardware, uint64_t *software) {
+    *hardware = hardwareVideoFrames.load(std::memory_order_relaxed);
+    *software = softwareVideoFrames.load(std::memory_order_relaxed);
+}
 extern "C" bool cast_media_check_advance(CastMedia *media, double seconds) {
     if (!media || !media->source.live || !std::isfinite(seconds) || seconds < 6 || seconds > 3600) return false;
     std::unique_lock guard(media->lock);

Sources/Core/player.cpp

@@ -2,6 +2,7 @@
 #include "player_macos.h"
 #include "output.hpp"
 #include "progress_reporter.hpp"
+#include "video_decoder.hpp"
 
 extern "C" {
 #include <libavcodec/avcodec.h>
@@ -64,6 +65,7 @@ struct Track {
     AVPacket *packet = nullptr;
     bool packetPending = false;
     bool receiving = false;
+    bool firstPacket = true;
     double packetBufferedEnd = 0;
     double decoderBufferedEnd = 0;
     bool sourceEnded = false;
@@ -456,6 +458,7 @@ struct Player {
         track.codec->pkt_timebase = source->streams[index]->time_base;
         // A bounded software decoder pool; FFmpeg owns its internal frame reorder buffer.
         track.codec->thread_count = 2;
+        if (type == AVMEDIA_TYPE_VIDEO) playback::preferVideoToolbox(track.codec);
         check(avcodec_open2(track.codec, decoder, nullptr), "Opening decoder");
     }
 
@@ -466,6 +469,7 @@ struct Player {
             if (!track.codec) continue;
             avcodec_flush_buffers(track.codec);
             track.drainSent = track.drained = false;
+            track.firstPacket = true;
             av_packet_unref(track.packet);
             track.packetPending = track.receiving = track.sourceEnded = false;
             track.packetBufferedEnd = track.decoderBufferedEnd = 0;
@@ -1086,8 +1090,11 @@ struct Player {
                 return;
             skipPresentedTimestamp = AV_NOPTS_VALUE;
         }
-        const int size = av_image_get_buffer_size(static_cast<AVPixelFormat>(decoded->format),
+        Frame frame(av_frame_alloc());
-                                                   decoded->width, decoded->height, 1);
+        if (!frame) throw std::bad_alloc();
+        check(playback::refVideoFrame(frame.get(), decoded), "Mapping decoded video");
+        const int size = av_image_get_buffer_size(static_cast<AVPixelFormat>(frame->format),
+                                                   frame->width, frame->height, 1);
         check(size, "Sizing decoded video");
         const auto bytes = static_cast<std::size_t>(size);
         if (bytes > videoByteLimit)
@@ -1100,8 +1107,6 @@ struct Player {
             videoBytes -= video.back().bytes;
             video.pop_back();
         }
-        Frame frame(av_frame_clone(decoded));
-        if (!frame) throw std::bad_alloc();
         VideoFrame item{std::move(frame), pts, pts + frameDuration, bytes, track.timestampOccurrence};
         auto insertion = std::upper_bound(video.begin(), video.end(), pts,
             [](double value, const VideoFrame &entry) { return value < entry.pts; });
@@ -1137,6 +1142,11 @@ struct Player {
                 return true;
             }
             if (track.packetPending) {
+                if (slot == 1 && track.firstPacket) {
+                    check(playback::prepareVideoPacket(track.codec, *track.stream()->codecpar, *track.packet),
+                          "Preparing video decoder preroll");
+                    track.firstPacket = false;
+                }
                 const int result = avcodec_send_packet(track.codec, track.packet);
                 if (result != AVERROR(EAGAIN)) {
                     check(result, "Sending media packet to decoder");

Sources/Core/video_decoder.hpp

@@ -0,0 +1,74 @@
+#pragma once
+#include <algorithm>
+extern "C" {
+#include <libavcodec/avcodec.h>
+#include <libavutil/hwcontext.h>
+}
+
+namespace playback {
+inline bool hasIDR(const AVPacket &packet, const AVCodecParameters &codec) {
+    const auto *bytes = packet.data;
+    const size_t size = size_t(std::max(0, packet.size));
+    if (codec.extradata_size >= 5 && codec.extradata[0] == 1) {
+        const size_t lengthSize = (codec.extradata[4] & 3) + 1;
+        for (size_t at = 0; at + lengthSize <= size;) {
+            uint32_t length = 0;
+            for (size_t i = 0; i < lengthSize; ++i) length = (length << 8) | bytes[at++];
+            if (!length || length > size - at) return false;
+            if ((bytes[at] & 31) == 5) return true;
+            at += length;
+        }
+        return false;
+    }
+    for (size_t at = 0; at + 3 < size; ++at) {
+        if (bytes[at] == 0 && bytes[at + 1] == 0 && bytes[at + 2] == 1 &&
+            (bytes[at + 3] & 31) == 5) return true;
+    }
+    return false;
+}
+// FFmpeg retries software if VideoToolbox rejects the stream; HEVC may also use
+// Apple's platform software fallback. Keep the normal decoder for other codecs.
+static inline void preferVideoToolbox(AVCodecContext *context) {
+#ifndef TEREBI_SOFTWARE_DECODE
+    // Prefer only the codecs measured in native playback and the Cast relay.
+    if (context->codec_id != AV_CODEC_ID_H264 && context->codec_id != AV_CODEC_ID_HEVC &&
+        context->codec_id != AV_CODEC_ID_VP9) return;
+    for (int index = 0; const auto *config = avcodec_get_hw_config(context->codec, index); ++index) {
+        if (config->device_type != AV_HWDEVICE_TYPE_VIDEOTOOLBOX ||
+            !(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) continue;
+        // The default get_format prefers this device and retries software if
+        // VideoToolbox rejects the stream; device creation failure is also safe.
+        av_hwdevice_ctx_create(&context->hw_device_ctx, AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
+                               nullptr, nullptr, 0);
+        break;
+    }
+#endif
+}
+// Starting at a dependent packet can make VideoToolbox fail during send_packet,
+// after format negotiation has already succeeded. Preserve software preroll.
+inline bool independentVideoPacket(const AVPacket &packet, const AVCodecParameters &parameters) {
+    return (packet.flags & AV_PKT_FLAG_KEY) &&
+        (parameters.codec_id != AV_CODEC_ID_H264 || hasIDR(packet, parameters));
+}
+inline int prepareVideoPacket(AVCodecContext *&context, const AVCodecParameters &parameters,
+                              const AVPacket &packet) {
+    if (!context->hw_device_ctx || independentVideoPacket(packet, parameters)) return 0;
+    auto *software = avcodec_alloc_context3(context->codec);
+    if (!software) return AVERROR(ENOMEM);
+    int result = avcodec_parameters_to_context(software, &parameters);
+    software->pkt_timebase = context->pkt_timebase;
+    software->thread_count = context->thread_count;
+    software->max_pixels = context->max_pixels;
+    if (result >= 0) result = avcodec_open2(software, context->codec, nullptr);
+    if (result < 0) { avcodec_free_context(&software); return result; }
+    avcodec_free_context(&context); context = software;
+    return 0;
+}
+// Mapping retains and locks the CVPixelBuffer until the last frame reference
+// is released. Consumers see ordinary CPU-readable planes without a pixel copy.
+inline int refVideoFrame(AVFrame *destination, const AVFrame *source) {
+    if (source->format == AV_PIX_FMT_VIDEOTOOLBOX)
+        return av_hwframe_map(destination, source, AV_HWFRAME_MAP_READ);
+    return av_frame_ref(destination, source);
+}
+}

TODO.md

@@ -6,8 +6,12 @@
 - [ ] evaluate playback optimization opportunities for YouTube and Miruro — measure benefits before changing behavior
 	- [x] coalesce superseded, not-yet-started seeks in the player and Cast controls; preserve accumulated relative skips and play/pause ordering
 	- [x] expand Cast H.264/AAC remux coverage with keyframe-aligned segments; preserve seek accuracy, timestamps and receiver compatibility
-	- [ ] reuse decoder/encoder state during sequential Cast playback; reset on random seeks and preserve independent keyframes and AAC continuity
+	- [x] reuse decoder/encoder state during sequential Cast playback; reset on random seeks and preserve independent keyframes and AAC continuity
-	- [ ] evaluate VideoToolbox hardware video decoding for native and Cast playback; retain software fallback for unsupported formats
+		- `.build/cast-media-check --codec-work`: 60s of 720p/30 B-frame video plus 44.1kHz PCM→AAC over loopback HTTP; three paired runs reduced median relay CPU from 3.56s to 2.58s (27%) and generation time from 4.97s to 4.43s (11%); physical-receiver playback still needs manual verification
+	- [x] evaluate VideoToolbox hardware video decoding for native and Cast playback; retain software fallback for unsupported formats
+		- M4, three paired 12s/720p/30 MP4 runs: H.264/HEVC/VP9 native decode process CPU fell 77/88/92%; Cast relay process CPU fell 48/58/68%, with generation 2–7% slower. Prefer these measured codecs; AV1 keeps dav1d. Unpaced native decoding is slower; H.264 seek preview rose from 1.8ms to 4.1ms. These are process CPU measurements, not energy measurements.
+		- `make check-video-decoding` covers native Metal upload, mapped-frame lifetime, exact seek/step/EOF, dependent-packet seek fallback, Cast independent IDRs/AAC, and unsupported codecs; `video-decoding-software-check` and `cast-media-software-check` provide matched baselines. Unsafe seek starts use software decoding (until reopen for native playback).
+		- Existing Cast millisecond-timestamp limiting drops VP9 MKV frames in the software baseline; a direct relay comparison confirmed VideoToolbox preserves the same output PTS/counts. This evaluation does not change that limiter.
 	- [ ] benchmark cold start, cached/uncached seeks and sustained playback separately to prioritize the actual bottlenecks
 	- do not enable Miruro HTTP persistence without preserving origin/header checks, or increase caches/shorten Cast segments without measuring the memory/request/codec tradeoffs
 - [ ] add playlist support