perf: prefer VideoToolbox decoding with software fallback

This commit is contained in:
2026-10-17 01:11:58 +02:00
parent 599c1492ee
commit fcad96de62
8 changed files with 513 additions and 36 deletions
+85 -1
View File
@@ -35,6 +35,33 @@ extern "C" {
extern char **environ;
// Encoded H.264 output cannot identify its input decoder; observe received
// source frames only in the check build, before CPU-readable mapping.
#ifdef CAST_MEDIA_CHECK
extern "C" void cast_media_check_decode_counts(uint64_t *, uint64_t *);
#endif
static std::pair<uint64_t, uint64_t> decodeCounts() {
uint64_t hardware = 0, software = 0;
#ifdef CAST_MEDIA_CHECK
cast_media_check_decode_counts(&hardware, &software);
#endif
return {hardware, software};
}
static std::pair<uint64_t, uint64_t> decodeUsage(std::pair<uint64_t, uint64_t> before) {
const auto after = decodeCounts();
const std::pair counts{after.first - before.first, after.second - before.second};
#ifdef CAST_MEDIA_CHECK
assert(counts.first + counts.second > 0);
#ifdef TEREBI_SOFTWARE_DECODE
assert(counts.first == 0);
#endif
std::cout << " decoder_videotoolbox_frames=" << counts.first << " decoder_software_frames=" << counts.second;
#else
std::cout << " decoder_frames=unavailable (compile with CAST_MEDIA_CHECK)";
#endif
return counts;
}
static void ok(int result) { assert(result >= 0); }
static std::vector<uint8_t> readFile(const std::string &path) {
std::ifstream file(path, std::ios::binary); assert(file.good());
@@ -690,6 +717,7 @@ static void codecWork(const std::filesystem::path &directory) {
Upstream upstream(readFile(source), readFile(audio)); upstream.start();
const auto videoURL = upstream.url("/source.ts"), audioURL = upstream.url("/audio.wav");
std::string url; double duration = 0;
const auto beforeDecode = decodeCounts();
const auto began = std::chrono::steady_clock::now();
auto *media = start({videoURL.c_str(), "X-Video: video-secret\r\n", audioURL.c_str(),
"X-Audio: audio-secret\r\n", false, false, false, 0}, url, duration);
@@ -723,7 +751,8 @@ static void codecWork(const std::filesystem::path &directory) {
cast_media_destroy(media);
std::cout << "codec-work prepare_ms=" << std::chrono::duration<double, std::milli>(prepared - began).count()
<< " sequential_60s_ms=" << sequentialMS << " sequential_cpu_ms=" << sequentialCPU
<< " seek_sequence_ms=" << randomMS << std::endl;
<< " seek_sequence_ms=" << randomMS;
decodeUsage(beforeDecode); std::cout << std::endl;
// A long video gap must not turn bounded encoder lookahead into HTTP 503,
// and resuming video after audio-only segments still needs an independent IDR.
const auto sparsePath = (directory / "sparse.mp4").string();
@@ -752,6 +781,55 @@ static void codecWork(const std::filesystem::path &directory) {
}
cast_media_destroy(media);
}
// Twelve-second, 30 fps video-only fixtures exercise each codec without
// regenerating sources. Separate PCM audio also keeps the AAC lane covered.
static void decodeWork(const std::filesystem::path &directory, const std::string &fixturePath) {
#ifndef CAST_MEDIA_CHECK
assert(false && "Compile --decode-work with CAST_MEDIA_CHECK to observe actual decoding");
#endif
AVFormatContext *format = nullptr;
ok(avformat_open_input(&format, fixturePath.c_str(), nullptr, nullptr)); ok(avformat_find_stream_info(format, nullptr));
const int videoIndex = av_find_best_stream(format, AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0); assert(videoIndex >= 0);
const auto *video = format->streams[videoIndex]->codecpar;
const std::string codec = avcodec_get_name(video->codec_id);
avformat_close_input(&format);
Upstream upstream(readFile(fixturePath), wav()); upstream.start();
const auto videoURL = upstream.url("/source.ts"), audioURL = upstream.url("/audio.wav");
const auto output = (directory / "decode-output.ts").string();
const auto beforeDecode = decodeCounts();
double wallMS = 0, cpuMS = 0;
for (bool random : {false, true}) {
std::string url; double duration = 0;
auto *media = start({videoURL.c_str(), "X-Video: video-secret\r\n", audioURL.c_str(),
"X-Audio: audio-secret\r\n", false, false, false, 0}, url, duration);
assert(std::abs(duration - 12) < .1);
const auto base = url.substr(0, url.rfind('/') + 1);
std::map<int, std::pair<double, double>> spans;
for (int at = 0; at < 2; ++at) {
const int id = random ? 1 - at : at;
const auto began = std::chrono::steady_clock::now(); const auto cpu = std::clock();
const auto response = request(base + "segment/" + std::to_string(id) + ".ts");
wallMS += std::chrono::duration<double, std::milli>(std::chrono::steady_clock::now() - began).count();
cpuMS += 1000.0 * (std::clock() - cpu) / CLOCKS_PER_SEC;
assert(response.status == 200); writeFile(output, response.bytes);
const auto rows = samples(output, 1);
int videos = 0;
for (const auto &[key, value] : rows) {
assert(key.second >= id * 6 * 90000 && key.second < (id + 1) * 6 * 90000);
if (key.first) ++videos;
}
assert(videos == 180);
spans[id] = audioSpan(response, output);
assert(request(base + "segment/" + std::to_string(id) + ".ts").bytes == response.bytes);
}
assert(std::abs(spans.at(1).first - spans.at(0).second) < 1.0 / 48000);
cast_media_destroy(media);
}
assert(upstream.foreignRequests == 0);
std::cout << "decode-work codec=" << codec << " relay_24s_ms=" << wallMS << " relay_cpu_ms=" << cpuMS;
decodeUsage(beforeDecode);
std::cout << "\ncast-media: codec decode, independent IDRs, AAC continuity, sequential and random seeks passed\n";
}
int main(int argc, char **argv) {
bool liveCheck = false, seekCheck = false, hlsCheck = false, remuxCheck = false;
for (int argument = 1; argument < argc; ++argument) {
@@ -759,6 +837,12 @@ int main(int argc, char **argv) {
if (std::string(argv[argument]) == "--seek-work") seekCheck = true;
if (std::string(argv[argument]) == "--hls-work") hlsCheck = true;
if (std::string(argv[argument]) == "--remux-work") remuxCheck = true;
if (std::string(argv[argument]) == "--decode-work") {
assert(argument + 1 < argc && "--decode-work requires a 12s/30fps fixture path");
char temp[] = "/tmp/terebi-decode-check.XXXXXX"; assert(mkdtemp(temp));
decodeWork(temp, argv[++argument]); std::filesystem::remove_all(temp);
return 0;
}
if (std::string(argv[argument]) == "--codec-work") {
char temp[] = "/tmp/terebi-codec-check.XXXXXX"; assert(mkdtemp(temp));
codecWork(temp); std::filesystem::remove_all(temp);
+275
View File
@@ -0,0 +1,275 @@
// Build like seek-queue-check, with -fobjc-arc and the bundled FFmpeg flags,
// linking the normal player objects EXCEPT player.o (the implementation is below).
// Build the same source again with -DTEREBI_SOFTWARE_DECODE for the baseline.
// Run: video-decoding-check [--metal | --preroll] [--frames 360] [--fps 30] fixture ...
// Fixtures: video-only CFR testsrc2, zero-origin PTS, BT.709 SDR, square SAR.
// Benchmark excludes opening, pixel checks, seeking and Metal; there is no pacing,
// audio-device setup or playback clock. Wall/process CPU are measurements only.
#include "../Sources/Core/player.cpp"
#import <AppKit/AppKit.h>
#import <MetalKit/MetalKit.h>
extern "C" {
#include <libavutil/pixdesc.h>
}
#include <cstdlib>
#include <ctime>
static void require(bool condition, const char *message) {
if (!condition) { std::fprintf(stderr, "FAIL: %s\n", message); std::exit(1); }
}
static Frame retain(const AVFrame *source) {
Frame frame(av_frame_clone(source));
if (!frame) throw std::bad_alloc();
return frame;
}
static void metadata(Player &player, const AVFrame *frame) {
const auto *parameters = player.tracks[1].stream()->codecpar;
require(frame->width == parameters->width && frame->height == parameters->height,
"readable frame preserves coded dimensions");
require(frame->color_primaries == AVCOL_PRI_BT709 && frame->color_trc == AVCOL_TRC_BT709 &&
frame->colorspace == AVCOL_SPC_BT709, "readable frame preserves BT.709 SDR metadata");
require(parameters->color_range == AVCOL_RANGE_UNSPECIFIED ||
frame->color_range == parameters->color_range, "readable frame preserves color range");
const AVRational aspect = av_guess_sample_aspect_ratio(player.format,
player.tracks[1].stream(), const_cast<AVFrame *>(frame));
require(aspect.num > 0 && aspect.num == aspect.den, "consumer sees square sample aspect ratio");
const auto *description = av_pix_fmt_desc_get(static_cast<AVPixelFormat>(frame->format));
require(description && !(description->flags & AV_PIX_FMT_FLAG_HWACCEL),
"queued frames have a CPU-readable pixel format");
}
// Read every visible byte, not padding: this detects a mapped frame whose pixel
// buffer was unlocked/recycled when a decoder frame, timeline or codec was freed.
static uint64_t pixels(const AVFrame *frame) {
const auto format = static_cast<AVPixelFormat>(frame->format);
int widths[4]{};
require(av_image_fill_linesizes(widths, format, frame->width) >= 0, "readable plane widths");
ptrdiff_t strides[4]{};
for (int plane = 0; plane < 4; ++plane) strides[plane] = widths[plane];
size_t sizes[4]{};
require(av_image_fill_plane_sizes(sizes, format, frame->height, strides) >= 0,
"readable plane heights");
uint64_t hash = 14695981039346656037ULL;
unsigned low = 255, high = 0;
for (int plane = 0; plane < 4; ++plane) {
if (!sizes[plane]) continue;
require(widths[plane] > 0 && frame->data[plane] &&
std::abs(frame->linesize[plane]) >= widths[plane], "real readable pixel planes");
const size_t rows = sizes[plane] / widths[plane];
for (size_t y = 0; y < rows; ++y) {
const uint8_t *row = frame->data[plane] + ptrdiff_t(y) * frame->linesize[plane];
for (int x = 0; x < widths[plane]; ++x) {
low = std::min(low, unsigned(row[x])); high = std::max(high, unsigned(row[x]));
hash = (hash ^ row[x]) * 1099511628211ULL;
}
}
}
require(low < high, "testsrc2 contains real nonconstant pixel data");
return hash;
}
static void exactFrame(Player &player, const VideoFrame &frame, int index, int fps) {
require(bool(frame.frame), "consumer receives a video frame");
const auto timebase = player.tracks[1].stream()->time_base;
const int64_t expected = av_rescale_q(index, AVRational{1, fps}, timebase);
require(presentationTimestamp(frame.frame.get()) == expected, "exact consumer presentation timestamp");
require(std::abs(frame.pts - expected * av_q2d(timebase)) < 0.000001,
"queue PTS matches the fixture media timestamp");
require(frame.end > frame.pts, "queued frame retains a positive presentation duration");
metadata(player, frame.frame.get());
}
struct DecodedRun {
Frame first, last;
int frames = 0;
bool hardware = false;
size_t peakFrames = 0, peakBytes = 0;
};
static DecodedRun drain(Player &player, int first, int expectedFrames, int fps) {
DecodedRun result;
double previous = -std::numeric_limits<double>::infinity();
while (!player.tracks[1].drained || !player.video.empty()) {
auto *codec = player.tracks[1].codec;
result.hardware |= codec->pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX || codec->hw_frames_ctx != nullptr;
result.peakFrames = std::max(result.peakFrames, player.video.size());
result.peakBytes = std::max(result.peakBytes, player.videoBytes);
require(player.video.size() <= videoFrameLimit && player.videoBytes <= videoByteLimit,
"real decoded queue stays inside frame and byte budgets");
if (!player.video.empty() && (player.videoFull() || player.tracks[1].drained)) {
while (!player.video.empty()) {
VideoFrame frame = std::move(player.video.front());
player.video.pop_front();
player.videoBytes -= frame.bytes;
require(frame.pts > previous, "consumer observes strictly monotonic PTS including B-frame EOF");
require(result.frames < expectedFrames, "no extra decoded frames after decoder drain");
exactFrame(player, frame, first + result.frames, fps);
previous = frame.pts;
if (!result.first) result.first = retain(frame.frame.get());
result.last = std::move(frame.frame);
++result.frames;
}
} else {
require(player.decodeOne(), "local synchronous decode makes progress without a clock");
}
}
require(result.frames == expectedFrames, "all expected frames survive queueing and delayed EOF drain");
require(player.demuxEnded && player.tracks[1].sourceEnded && player.tracks[1].drainSent &&
player.tracks[1].drained && player.videoBytes == 0, "demux and codec EOF fully drain the queue");
return result;
}
static void draw(playback::Output &output, const AVFrame *frame) {
void *native = output.make_video_view();
require(native != nullptr, "real Metal view and shaders initialize");
MTKView *view = (__bridge_transfer MTKView *)native;
view.paused = YES;
NSWindow *window = [[NSWindow alloc] initWithContentRect:NSMakeRect(100, 100, 320, 180)
styleMask:NSWindowStyleMaskBorderless backing:NSBackingStoreBuffered defer:NO];
window.contentView = view;
[window orderFront:nil];
output.present(frame);
// AppKit must attach the layer before a native drawable can be acquired.
// This is an I/O watchdog, not a media clock.
const auto deadline = Clock::now() + std::chrono::seconds(5);
while (!view.currentDrawable && Clock::now() < deadline)
CFRunLoopRunInMode(kCFRunLoopDefaultMode, .005, false);
require(view.currentDrawable != nil, "real Metal surface supplies a drawable");
[view draw];
[window orderOut:nil];
}
static void fixture(const char *path, int frames, int fps, bool metal, bool preroll) {
Player player;
player.shutdown(); // Join the worker: use actual open/decode/seek synchronously.
player.stopping.store(false);
player.activeEpoch = player.requestedEpoch.load();
Command command{Action::Open, path};
player.open(command);
require(player.tracks[1].codec && !player.tracks[0].codec, "fixture is video-only (no audio device)");
require(player.canSeek && std::abs(player.duration - double(frames) / fps) < .002,
"fixture has the fixed expected media duration and is seekable");
if (preroll) {
while (player.video.empty() && !player.tracks[1].drained)
require(player.decodeOne(), "opening transport stream makes progress");
require(!player.video.empty(), "transport stream supplies an opening picture");
auto retained = retain(player.video.front().frame.get());
const auto hash = pixels(retained.get());
const double target = (frames * 3 / 5 + .4) / fps;
player.seek(target, false);
while (player.video.empty() && !player.tracks[1].drained)
require(player.decodeOne(), "dependent-packet seek makes progress");
require(!player.video.empty() && player.video.front().pts >= target &&
player.video.front().pts < player.duration, "transport seek supplies a usable preview after preroll");
metadata(player, player.video.front().frame.get());
pixels(player.video.front().frame.get());
const bool software = player.tracks[1].codec->pix_fmt != AV_PIX_FMT_VIDEOTOOLBOX;
player.release();
require(pixels(retained.get()) == hash, "mapped opening picture survives preroll decoder replacement");
std::printf("PASS native dependent-packet preroll/preview/lifetime: %s software=%s\n",
path, software ? "yes" : "no");
return;
}
const auto wallStart = std::chrono::steady_clock::now();
const auto cpuStart = std::clock();
auto decoded = drain(player, 0, frames, fps);
const double cpu = double(std::clock() - cpuStart) / CLOCKS_PER_SEC;
const double wall = std::chrono::duration<double>(std::chrono::steady_clock::now() - wallStart).count();
#ifdef TEREBI_SOFTWARE_DECODE
constexpr const char *mode = "software";
require(!decoded.hardware, "matched software baseline never negotiates hardware");
#else
constexpr const char *mode = "preferred";
#endif
std::printf("native-decode mode=%s fixture=%s codec=%s videotoolbox=%s format=%s frames=%d "
"wall=%.6f cpu=%.6f peak_frames=%zu peak_bytes=%zu\n", mode, path,
player.tracks[1].codec->codec->name, decoded.hardware ? "yes" : "no",
av_get_pix_fmt_name(static_cast<AVPixelFormat>(decoded.first->format)), decoded.frames,
wall, cpu, decoded.peakFrames, decoded.peakBytes);
const auto firstHash = pixels(decoded.first.get());
const auto lastHash = pixels(decoded.last.get());
require(firstHash != lastHash, "retained first and last frames contain distinct fixture pictures");
if (metal) draw(player.output, decoded.first.get());
// A non-keyframe seek must discard preroll and expose the exact next PTS.
const int middle = frames / 2;
const auto seekStart = Clock::now();
player.seek((middle + .4) / fps, false);
require(player.video.empty() && player.videoBytes == 0, "seek clears the old readable queue");
while (player.video.empty() && !player.tracks[1].drained)
require(player.decodeOne(), "seek preroll makes progress");
std::printf("native-seek mode=%s fixture=%s preview_ms=%.3f\n", mode, path,
std::chrono::duration<double, std::milli>(Clock::now() - seekStart).count());
require(pixels(decoded.first.get()) == firstHash && pixels(decoded.last.get()) == lastHash,
"mapped frame references survive decoder flush");
auto sought = drain(player, middle + 1, frames - middle - 1, fps);
pixels(sought.first.get());
// Seek backward after EOF, then step in both directions without a clock.
player.seek((middle + .4) / fps, false);
player.step(1);
exactFrame(player, player.presented, middle + 2, fps);
player.step(-1);
exactFrame(player, player.presented, middle + 1, fps);
player.step(1);
exactFrame(player, player.presented, middle + 2, fps);
player.seek(1.4 / fps, false);
auto restarted = drain(player, 2, frames - 2, fps);
const double lastPts = presentationTimestamp(restarted.last.get()) *
av_q2d(player.tracks[1].stream()->time_base) - player.origin;
const int lastBytes = av_image_get_buffer_size(static_cast<AVPixelFormat>(restarted.last->format),
restarted.last->width, restarted.last->height, 1);
require(lastBytes > 0, "last readable frame has a valid byte size");
player.presentVideo(VideoFrame{retain(restarted.last.get()), lastPts,
lastPts + player.videoFrameTime, static_cast<size_t>(lastBytes)});
player.step(1);
exactFrame(player, player.presented, frames - 1, fps);
player.step(-1);
exactFrame(player, player.presented, frames - 2, fps);
player.step(1);
exactFrame(player, player.presented, frames - 1, fps);
player.release(); // Retained mapped planes must also outlive the codec/device.
require(pixels(decoded.first.get()) == firstHash && pixels(decoded.last.get()) == lastHash,
"mapped frame references survive full decoder release and pool reuse");
if (metal) draw(player.output, decoded.last.get());
std::printf("PASS native readable frames/SDR/SAR/seek/exact steps/EOF/lifetimes: %s\n", path);
}
int main(int argc, char **argv) {
@autoreleasepool {
int frames = 360, fps = 30;
bool metal = false, preroll = false;
std::vector<const char *> paths;
for (int i = 1; i < argc; ++i) {
if (std::strcmp(argv[i], "--metal") == 0) { metal = true; continue; }
if (std::strcmp(argv[i], "--preroll") == 0) { preroll = true; continue; }
if (std::strcmp(argv[i], "--frames") == 0 || std::strcmp(argv[i], "--fps") == 0) {
const bool count = std::strcmp(argv[i], "--frames") == 0;
require(++i < argc, "numeric option needs a value");
char *end = nullptr;
const long value = std::strtol(argv[i], &end, 10);
require(end != argv[i] && !*end && value > 0 && value <= 1000000,
"frame count and integer frame rate must be positive and bounded");
(count ? frames : fps) = static_cast<int>(value);
} else {
require(argv[i][0] != '-', "unknown option");
paths.push_back(argv[i]);
}
}
require(!paths.empty() && frames >= 8, "provide fixtures with at least eight frames");
if (metal) {
[NSApplication sharedApplication];
[NSApp setActivationPolicy:NSApplicationActivationPolicyAccessory];
[NSApp finishLaunching];
}
try {
for (const char *path : paths) fixture(path, frames, fps, metal, preroll);
} catch (const std::exception &error) {
std::fprintf(stderr, "FAIL: %s\n", error.what());
return 1;
}
}
}