hl-valdemar/terebi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

perf: prefer VideoToolbox decoding with software fallback

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-10-17 01:11:58 +02:00
parent 599c1492ee
commit fcad96de62
8 changed files with 513 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Checks/video-decoding.mm
+275
View File
@@ -0,0 +1,275 @@
// Build like seek-queue-check, with -fobjc-arc and the bundled FFmpeg flags,
// linking the normal player objects EXCEPT player.o (the implementation is below).
// Build the same source again with -DTEREBI_SOFTWARE_DECODE for the baseline.
// Run: video-decoding-check [--metal | --preroll] [--frames 360] [--fps 30] fixture ...
// Fixtures: video-only CFR testsrc2, zero-origin PTS, BT.709 SDR, square SAR.
// Benchmark excludes opening, pixel checks, seeking and Metal; there is no pacing,
// audio-device setup or playback clock. Wall/process CPU are measurements only.
#include "../Sources/Core/player.cpp"
#import <AppKit/AppKit.h>
#import <MetalKit/MetalKit.h>
extern "C" {
#include <libavutil/pixdesc.h>
}
#include <cstdlib>
#include <ctime>
static void require(bool condition, const char *message) {
if (!condition) { std::fprintf(stderr, "FAIL: %s\n", message); std::exit(1); }
}
static Frame retain(const AVFrame *source) {
Frame frame(av_frame_clone(source));
if (!frame) throw std::bad_alloc();
return frame;
}
static void metadata(Player &player, const AVFrame *frame) {
const auto *parameters = player.tracks[1].stream()->codecpar;
require(frame->width == parameters->width && frame->height == parameters->height,
"readable frame preserves coded dimensions");
require(frame->color_primaries == AVCOL_PRI_BT709 && frame->color_trc == AVCOL_TRC_BT709 &&
frame->colorspace == AVCOL_SPC_BT709, "readable frame preserves BT.709 SDR metadata");
require(parameters->color_range == AVCOL_RANGE_UNSPECIFIED ||
frame->color_range == parameters->color_range, "readable frame preserves color range");
const AVRational aspect = av_guess_sample_aspect_ratio(player.format,
player.tracks[1].stream(), const_cast<AVFrame *>(frame));
require(aspect.num > 0 && aspect.num == aspect.den, "consumer sees square sample aspect ratio");
const auto *description = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(frame->format));
require(description && !(description->flags & AV_PIX_FMT_FLAG_HWACCEL),
"queued frames have a CPU-readable pixel format");
}
// Read every visible byte, not padding: this detects a mapped frame whose pixel
// buffer was unlocked/recycled when a decoder frame, timeline or codec was freed.
static uint64_t pixels(const AVFrame *frame) {
const auto format = static_cast<AVPixelFormat>(frame->format);
int widths[4]{};
require(av_image_fill_linesizes(widths, format, frame->width) >= 0, "readable plane widths");
ptrdiff_t strides[4]{};
for (int plane = 0; plane < 4; ++plane) strides[plane] = widths[plane];
size_t sizes[4]{};
require(av_image_fill_plane_sizes(sizes, format, frame->height, strides) >= 0,
"readable plane heights");
uint64_t hash = 14695981039346656037ULL;
unsigned low = 255, high = 0;
for (int plane = 0; plane < 4; ++plane) {
if (!sizes[plane]) continue;
require(widths[plane] > 0 && frame->data[plane] &&
std::abs(frame->linesize[plane]) >= widths[plane], "real readable pixel planes");
const size_t rows = sizes[plane] / widths[plane];
for (size_t y = 0; y < rows; ++y) {
const uint8_t *row = frame->data[plane] + ptrdiff_t(y) * frame->linesize[plane];
for (int x = 0; x < widths[plane]; ++x) {
low = std::min(low, unsigned(row[x])); high = std::max(high, unsigned(row[x]));
hash = (hash ^ row[x]) * 1099511628211ULL;
}
}
}
require(low < high, "testsrc2 contains real nonconstant pixel data");
return hash;
}
static void exactFrame(Player &player, const VideoFrame &frame, int index, int fps) {
require(bool(frame.frame), "consumer receives a video frame");
const auto timebase = player.tracks[1].stream()->time_base;
const int64_t expected = av_rescale_q(index, AVRational{1, fps}, timebase);
require(presentationTimestamp(frame.frame.get()) == expected, "exact consumer presentation timestamp");
require(std::abs(frame.pts - expected * av_q2d(timebase)) < 0.000001,
"queue PTS matches the fixture media timestamp");
require(frame.end > frame.pts, "queued frame retains a positive presentation duration");
metadata(player, frame.frame.get());
}
struct DecodedRun {
Frame first, last;
int frames = 0;
bool hardware = false;
size_t peakFrames = 0, peakBytes = 0;
};
static DecodedRun drain(Player &player, int first, int expectedFrames, int fps) {
DecodedRun result;
double previous = -std::numeric_limits<double>::infinity();
while (!player.tracks[1].drained || !player.video.empty()) {
auto *codec = player.tracks[1].codec;
result.hardware |= codec->pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX || codec->hw_frames_ctx != nullptr;
result.peakFrames = std::max(result.peakFrames, player.video.size());
result.peakBytes = std::max(result.peakBytes, player.videoBytes);
require(player.video.size() <= videoFrameLimit && player.videoBytes <= videoByteLimit,
"real decoded queue stays inside frame and byte budgets");
if (!player.video.empty() && (player.videoFull() || player.tracks[1].drained)) {
while (!player.video.empty()) {
VideoFrame frame = std::move(player.video.front());
player.video.pop_front();
player.videoBytes -= frame.bytes;
require(frame.pts > previous, "consumer observes strictly monotonic PTS including B-frame EOF");
require(result.frames < expectedFrames, "no extra decoded frames after decoder drain");
exactFrame(player, frame, first + result.frames, fps);
previous = frame.pts;
if (!result.first) result.first = retain(frame.frame.get());
result.last = std::move(frame.frame);
++result.frames;
}
} else {
require(player.decodeOne(), "local synchronous decode makes progress without a clock");
}
}
require(result.frames == expectedFrames, "all expected frames survive queueing and delayed EOF drain");
require(player.demuxEnded && player.tracks[1].sourceEnded && player.tracks[1].drainSent &&
player.tracks[1].drained && player.videoBytes == 0, "demux and codec EOF fully drain the queue");
return result;
}
static void draw(playback::Output &output, const AVFrame *frame) {
void *native = output.make_video_view();
require(native != nullptr, "real Metal view and shaders initialize");
MTKView *view = (__bridge_transfer MTKView *)native;
view.paused = YES;
NSWindow *window = [[NSWindow alloc] initWithContentRect:NSMakeRect(100, 100, 320, 180)
styleMask:NSWindowStyleMaskBorderless backing:NSBackingStoreBuffered defer:NO];
window.contentView = view;
[window orderFront:nil];
output.present(frame);
// AppKit must attach the layer before a native drawable can be acquired.
// This is an I/O watchdog, not a media clock.
const auto deadline = Clock::now() + std::chrono::seconds(5);
while (!view.currentDrawable && Clock::now() < deadline)
CFRunLoopRunInMode(kCFRunLoopDefaultMode, .005, false);
require(view.currentDrawable != nil, "real Metal surface supplies a drawable");
[view draw];
[window orderOut:nil];
}
static void fixture(const char *path, int frames, int fps, bool metal, bool preroll) {
Player player;
player.shutdown(); // Join the worker: use actual open/decode/seek synchronously.
player.stopping.store(false);
player.activeEpoch = player.requestedEpoch.load();
Command command{Action::Open, path};
player.open(command);
require(player.tracks[1].codec && !player.tracks[0].codec, "fixture is video-only (no audio device)");
require(player.canSeek && std::abs(player.duration - double(frames) / fps) < .002,
"fixture has the fixed expected media duration and is seekable");
if (preroll) {
while (player.video.empty() && !player.tracks[1].drained)
require(player.decodeOne(), "opening transport stream makes progress");
require(!player.video.empty(), "transport stream supplies an opening picture");
auto retained = retain(player.video.front().frame.get());
const auto hash = pixels(retained.get());
const double target = (frames * 3 / 5 + .4) / fps;
player.seek(target, false);
while (player.video.empty() && !player.tracks[1].drained)
require(player.decodeOne(), "dependent-packet seek makes progress");
require(!player.video.empty() && player.video.front().pts >= target &&
player.video.front().pts < player.duration, "transport seek supplies a usable preview after preroll");
metadata(player, player.video.front().frame.get());
pixels(player.video.front().frame.get());
const bool software = player.tracks[1].codec->pix_fmt != AV_PIX_FMT_VIDEOTOOLBOX;
player.release();
require(pixels(retained.get()) == hash, "mapped opening picture survives preroll decoder replacement");
std::printf("PASS native dependent-packet preroll/preview/lifetime: %s software=%s\n",
path, software ? "yes" : "no");
return;
}
const auto wallStart = std::chrono::steady_clock::now();
const auto cpuStart = std::clock();
auto decoded = drain(player, 0, frames, fps);
const double cpu = double(std::clock() - cpuStart) / CLOCKS_PER_SEC;
const double wall = std::chrono::duration<double>(std::chrono::steady_clock::now() - wallStart).count();
#ifdef TEREBI_SOFTWARE_DECODE
constexpr const char *mode = "software";
require(!decoded.hardware, "matched software baseline never negotiates hardware");
#else
constexpr const char *mode = "preferred";
#endif
std::printf("native-decode mode=%s fixture=%s codec=%s videotoolbox=%s format=%s frames=%d "
"wall=%.6f cpu=%.6f peak_frames=%zu peak_bytes=%zu\n", mode, path,
player.tracks[1].codec->codec->name, decoded.hardware ? "yes" : "no",
av_get_pix_fmt_name(static_cast<AVPixelFormat>(decoded.first->format)), decoded.frames,
wall, cpu, decoded.peakFrames, decoded.peakBytes);
const auto firstHash = pixels(decoded.first.get());
const auto lastHash = pixels(decoded.last.get());
require(firstHash != lastHash, "retained first and last frames contain distinct fixture pictures");
if (metal) draw(player.output, decoded.first.get());
// A non-keyframe seek must discard preroll and expose the exact next PTS.
const int middle = frames / 2;
const auto seekStart = Clock::now();
player.seek((middle + .4) / fps, false);
require(player.video.empty() && player.videoBytes == 0, "seek clears the old readable queue");
while (player.video.empty() && !player.tracks[1].drained)
require(player.decodeOne(), "seek preroll makes progress");
std::printf("native-seek mode=%s fixture=%s preview_ms=%.3f\n", mode, path,
std::chrono::duration<double, std::milli>(Clock::now() - seekStart).count());
require(pixels(decoded.first.get()) == firstHash && pixels(decoded.last.get()) == lastHash,
"mapped frame references survive decoder flush");
auto sought = drain(player, middle + 1, frames - middle - 1, fps);
pixels(sought.first.get());
// Seek backward after EOF, then step in both directions without a clock.
player.seek((middle + .4) / fps, false);
player.step(1);
exactFrame(player, player.presented, middle + 2, fps);
player.step(-1);
exactFrame(player, player.presented, middle + 1, fps);
player.step(1);
exactFrame(player, player.presented, middle + 2, fps);
player.seek(1.4 / fps, false);
auto restarted = drain(player, 2, frames - 2, fps);
const double lastPts = presentationTimestamp(restarted.last.get()) *
av_q2d(player.tracks[1].stream()->time_base) - player.origin;
const int lastBytes = av_image_get_buffer_size(static_cast<AVPixelFormat>(restarted.last->format),
restarted.last->width, restarted.last->height, 1);
require(lastBytes > 0, "last readable frame has a valid byte size");
player.presentVideo(VideoFrame{retain(restarted.last.get()), lastPts,
lastPts + player.videoFrameTime, static_cast<size_t>(lastBytes)});
player.step(1);
exactFrame(player, player.presented, frames - 1, fps);
player.step(-1);
exactFrame(player, player.presented, frames - 2, fps);
player.step(1);
exactFrame(player, player.presented, frames - 1, fps);
player.release(); // Retained mapped planes must also outlive the codec/device.
require(pixels(decoded.first.get()) == firstHash && pixels(decoded.last.get()) == lastHash,
"mapped frame references survive full decoder release and pool reuse");
if (metal) draw(player.output, decoded.last.get());
std::printf("PASS native readable frames/SDR/SAR/seek/exact steps/EOF/lifetimes: %s\n", path);
}
int main(int argc, char **argv) {
@autoreleasepool {
int frames = 360, fps = 30;
bool metal = false, preroll = false;
std::vector<const char *> paths;
for (int i = 1; i < argc; ++i) {
if (std::strcmp(argv[i], "--metal") == 0) { metal = true; continue; }
if (std::strcmp(argv[i], "--preroll") == 0) { preroll = true; continue; }
if (std::strcmp(argv[i], "--frames") == 0 || std::strcmp(argv[i], "--fps") == 0) {
const bool count = std::strcmp(argv[i], "--frames") == 0;
require(++i < argc, "numeric option needs a value");
char *end = nullptr;
const long value = std::strtol(argv[i], &end, 10);
require(end != argv[i] && !*end && value > 0 && value <= 1000000,
"frame count and integer frame rate must be positive and bounded");
(count ? frames : fps) = static_cast<int>(value);
} else {
require(argv[i][0] != '-', "unknown option");
paths.push_back(argv[i]);
}
}
require(!paths.empty() && frames >= 8, "provide fixtures with at least eight frames");
if (metal) {
[NSApplication sharedApplication];
[NSApp setActivationPolicy:NSApplicationActivationPolicyAccessory];
[NSApp finishLaunching];
}
try {
for (const char *path : paths) fixture(path, frames, fps, metal, preroll);
} catch (const std::exception &error) {
std::fprintf(stderr, "FAIL: %s\n", error.what());
return 1;
}
}
}