hl-valdemar/terebi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

perf: prefer VideoToolbox decoding with software fallback

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-10-17 01:11:58 +02:00
parent 599c1492ee
commit fcad96de62
8 changed files with 513 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Sources/Core/cast_media.cpp
+25 -23
View File
@@ -1,6 +1,7 @@
#include "cast_media.h"
#include "network_policy.hpp"
#include "network_input.hpp"
#include "video_decoder.hpp"
extern "C" {
#include <libavformat/avformat.h>
#include <libavcodec/avcodec.h>
@@ -40,6 +41,9 @@ using Clock = std::chrono::steady_clock;
constexpr double segmentSeconds = 6;
constexpr size_t segmentLimit = 32 * 1024 * 1024;
constexpr size_t windowSegments = 8;
#ifdef CAST_MEDIA_CHECK
std::atomic<uint64_t> hardwareVideoFrames{0}, softwareVideoFrames{0};
#endif
void check(int value) { if (value < 0) throw std::runtime_error("Cast media operation failed (private details redacted)."); }
void require(bool value) { if (!value) throw std::runtime_error("Cast media is unavailable or unsupported (private details redacted)."); }
void privateLog(void *, int level, const char *, va_list) {
@@ -55,26 +59,6 @@ Frame frame() { auto p = Frame(av_frame_alloc()); require(bool(p)); return p; }
Packet packet() { auto p = Packet(av_packet_alloc()); require(bool(p)); return p; }
int64_t aacBoundary(double seconds) { return int64_t(std::ceil(seconds * 48000 / 1024)) * 1024; }
double aacPreroll(double seconds) { return double(std::max<int64_t>(0, aacBoundary(seconds) - 12 * 1024)) / 48000; }
bool hasIDR(const AVPacket &packet, const AVCodecParameters &codec) {
const auto *bytes = packet.data;
const size_t size = size_t(std::max(0, packet.size));
if (codec.extradata_size >= 5 && codec.extradata[0] == 1) {
const size_t lengthSize = (codec.extradata[4] & 3) + 1;
for (size_t at = 0; at + lengthSize <= size;) {
uint32_t length = 0;
for (size_t i = 0; i < lengthSize; ++i) length = (length << 8) | bytes[at++];
if (!length || length > size - at) return false;
if ((bytes[at] & 31) == 5) return true;
at += length;
}
return false;
}
for (size_t at = 0; at + 3 < size; ++at) {
if (bytes[at] == 0 && bytes[at + 1] == 0 && bytes[at + 2] == 1 &&
(bytes[at + 3] & 31) == 5) return true;
}
return false;
}
struct Source {
std::string video, videoHeaders, audio, audioHeaders;
bool local, restricted, live;
@@ -234,7 +218,7 @@ struct Track {
const auto rate = av_guess_frame_rate(input->format, stream, nullptr);
const double openingGap = rate.num > 0 ? av_q2d(av_inv_q(rate)) : 0;
remux = remux && !live && pending && (pending->flags & AV_PKT_FLAG_KEY) &&
hasIDR(*pending, *stream->codecpar) &&
playback::hasIDR(*pending, *stream->codecpar) &&
(std::abs(packetTime() - start) < .001 ||
(start == 0 && packetTime() >= 0 && packetTime() <= openingGap));
if (aligned && !remux) {
@@ -243,7 +227,7 @@ struct Track {
}
}
if (!live && type == AVMEDIA_TYPE_VIDEO && pending && (pending->flags & AV_PKT_FLAG_KEY) &&
!hasIDR(*pending, *stream->codecpar) && stream->codecpar->codec_id == AV_CODEC_ID_H264) {
!playback::hasIDR(*pending, *stream->codecpar) && stream->codecpar->codec_id == AV_CODEC_ID_H264) {
// ponytail: rare intra-refresh seeks decode from the beginning; index real IDRs if this becomes slow.
input->seek(0);
nextPacket();
@@ -258,6 +242,9 @@ struct Track {
decoder->pkt_timebase = stream->time_base;
decoder->thread_count = 2;
decoder->max_pixels = 4096LL * 2160;
if (type == AVMEDIA_TYPE_VIDEO && pending &&
playback::independentVideoPacket(*pending, *stream->codecpar))
playback::preferVideoToolbox(decoder);
check(avcodec_open2(decoder, codec, nullptr));
} catch (...) { avcodec_free_context(&decoder); throw; }
}
@@ -294,6 +281,10 @@ struct Track {
require(!input->interrupted());
int result = avcodec_receive_frame(decoder, f.get());
if (result == 0) {
#ifdef CAST_MEDIA_CHECK
if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO)
(f->format == AV_PIX_FMT_VIDEOTOOLBOX ? hardwareVideoFrames : softwareVideoFrames).fetch_add(1, std::memory_order_relaxed);
#endif
decoded = std::move(f);
const double step = stream->codecpar->codec_type == AVMEDIA_TYPE_AUDIO ? double(decoded->nb_samples) / decoded->sample_rate : 1.0 / 30;
nextTime = frameTime() + step;
@@ -337,6 +328,7 @@ struct OutputTrack {
AVStream *stream = nullptr;
AVCodecContext *encoder = nullptr;
SwsContext *scale = nullptr;
Frame readable;
SwrContext *resample = nullptr;
AVAudioFifo *fifo = nullptr;
int64_t audioPTS = AV_NOPTS_VALUE;
@@ -463,8 +455,14 @@ struct OutputTrack {
auto out = frame(); out->format = encoder->pix_fmt; out->width = encoder->width; out->height = encoder->height;
out->sample_aspect_ratio = encoder->sample_aspect_ratio;
check(av_frame_get_buffer(out.get(), 32));
if (f->format == AV_PIX_FMT_VIDEOTOOLBOX) {
if (!readable) readable = frame();
check(playback::refVideoFrame(readable.get(), f));
f = readable.get();
}
scale = sws_getCachedContext(scale, f->width, f->height, AVPixelFormat(f->format), out->width, out->height, encoder->pix_fmt, SWS_BILINEAR, nullptr, nullptr, nullptr);
require(scale != nullptr); check(sws_scale(scale, f->data, f->linesize, 0, f->height, out->data, out->linesize));
if (readable) av_frame_unref(readable.get());
out->pts = int64_t(std::llround(std::max(start, time) * 90000));
out->pict_type = firstVideo ? AV_PICTURE_TYPE_I : AV_PICTURE_TYPE_NONE; firstVideo = false; encoded(out.get());
lastVideoTime = time;
@@ -816,7 +814,7 @@ struct CastMedia {
const bool independent = !video || !video->remux ||
(video->pending && std::abs(video->packetTime() - start) < .001 &&
(video->pending->flags & AV_PKT_FLAG_KEY) &&
hasIDR(*video->pending, *video->stream->codecpar));
playback::hasIDR(*video->pending, *video->stream->codecpar));
const bool drained = (vodPipeline->videoOutput && vodPipeline->videoOutput->drained) ||
(vodPipeline->audioOutput && vodPipeline->audioOutput->drained);
if (vodPipeline->nextStart != start || !independent || drained)
@@ -971,6 +969,10 @@ extern "C" bool cast_media_subtitle(CastMedia *media, int id, const void *bytes,
extern "C" void cast_media_cancel(CastMedia *media) { if (media) media->cancel(); }
extern "C" void cast_media_destroy(CastMedia *media) { delete media; }
#ifdef CAST_MEDIA_CHECK
extern "C" void cast_media_check_decode_counts(uint64_t *hardware, uint64_t *software) {
*hardware = hardwareVideoFrames.load(std::memory_order_relaxed);
*software = softwareVideoFrames.load(std::memory_order_relaxed);
}
extern "C" bool cast_media_check_advance(CastMedia *media, double seconds) {
if (!media || !media->source.live || !std::isfinite(seconds) || seconds < 6 || seconds > 3600) return false;
std::unique_lock guard(media->lock);
Sources/Core/player.cpp
+14 -4
View File
@@ -2,6 +2,7 @@
#include "player_macos.h"
#include "output.hpp"
#include "progress_reporter.hpp"
#include "video_decoder.hpp"
extern "C" {
#include <libavcodec/avcodec.h>
@@ -64,6 +65,7 @@ struct Track {
AVPacket *packet = nullptr;
bool packetPending = false;
bool receiving = false;
bool firstPacket = true;
double packetBufferedEnd = 0;
double decoderBufferedEnd = 0;
bool sourceEnded = false;
@@ -456,6 +458,7 @@ struct Player {
track.codec->pkt_timebase = source->streams[index]->time_base;
// A bounded software decoder pool; FFmpeg owns its internal frame reorder buffer.
track.codec->thread_count = 2;
if (type == AVMEDIA_TYPE_VIDEO) playback::preferVideoToolbox(track.codec);
check(avcodec_open2(track.codec, decoder, nullptr), "Opening decoder");
}
@@ -466,6 +469,7 @@ struct Player {
if (!track.codec) continue;
avcodec_flush_buffers(track.codec);
track.drainSent = track.drained = false;
track.firstPacket = true;
av_packet_unref(track.packet);
track.packetPending = track.receiving = track.sourceEnded = false;
track.packetBufferedEnd = track.decoderBufferedEnd = 0;
@@ -1086,8 +1090,11 @@ struct Player {
return;
skipPresentedTimestamp = AV_NOPTS_VALUE;
}
const int size = av_image_get_buffer_size(static_cast<AVPixelFormat>(decoded->format),
decoded->width, decoded->height, 1);
Frame frame(av_frame_alloc());
if (!frame) throw std::bad_alloc();
check(playback::refVideoFrame(frame.get(), decoded), "Mapping decoded video");
const int size = av_image_get_buffer_size(static_cast<AVPixelFormat>(frame->format),
frame->width, frame->height, 1);
check(size, "Sizing decoded video");
const auto bytes = static_cast<std::size_t>(size);
if (bytes > videoByteLimit)
@@ -1100,8 +1107,6 @@ struct Player {
videoBytes -= video.back().bytes;
video.pop_back();
}
Frame frame(av_frame_clone(decoded));
if (!frame) throw std::bad_alloc();
VideoFrame item{std::move(frame), pts, pts + frameDuration, bytes, track.timestampOccurrence};
auto insertion = std::upper_bound(video.begin(), video.end(), pts,
[](double value, const VideoFrame &entry) { return value < entry.pts; });
@@ -1137,6 +1142,11 @@ struct Player {
return true;
}
if (track.packetPending) {
if (slot == 1 && track.firstPacket) {
check(playback::prepareVideoPacket(track.codec, *track.stream()->codecpar, *track.packet),
"Preparing video decoder preroll");
track.firstPacket = false;
}
const int result = avcodec_send_packet(track.codec, track.packet);
if (result != AVERROR(EAGAIN)) {
check(result, "Sending media packet to decoder");
Sources/Core/video_decoder.hpp
+74
View File
@@ -0,0 +1,74 @@
#pragma once
#include <algorithm>
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavutil/hwcontext.h>
}
namespace playback {
inline bool hasIDR(const AVPacket &packet, const AVCodecParameters &codec) {
const auto *bytes = packet.data;
const size_t size = size_t(std::max(0, packet.size));
if (codec.extradata_size >= 5 && codec.extradata[0] == 1) {
const size_t lengthSize = (codec.extradata[4] & 3) + 1;
for (size_t at = 0; at + lengthSize <= size;) {
uint32_t length = 0;
for (size_t i = 0; i < lengthSize; ++i) length = (length << 8) | bytes[at++];
if (!length || length > size - at) return false;
if ((bytes[at] & 31) == 5) return true;
at += length;
}
return false;
}
for (size_t at = 0; at + 3 < size; ++at) {
if (bytes[at] == 0 && bytes[at + 1] == 0 && bytes[at + 2] == 1 &&
(bytes[at + 3] & 31) == 5) return true;
}
return false;
}
// FFmpeg retries software if VideoToolbox rejects the stream; HEVC may also use
// Apple's platform software fallback. Keep the normal decoder for other codecs.
static inline void preferVideoToolbox(AVCodecContext *context) {
#ifndef TEREBI_SOFTWARE_DECODE
// Prefer only the codecs measured in native playback and the Cast relay.
if (context->codec_id != AV_CODEC_ID_H264 && context->codec_id != AV_CODEC_ID_HEVC &&
context->codec_id != AV_CODEC_ID_VP9) return;
for (int index = 0; const auto *config = avcodec_get_hw_config(context->codec, index); ++index) {
if (config->device_type != AV_HWDEVICE_TYPE_VIDEOTOOLBOX ||
!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) continue;
// The default get_format prefers this device and retries software if
// VideoToolbox rejects the stream; device creation failure is also safe.
av_hwdevice_ctx_create(&context->hw_device_ctx, AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
nullptr, nullptr, 0);
break;
}
#endif
}
// Starting at a dependent packet can make VideoToolbox fail during send_packet,
// after format negotiation has already succeeded. Preserve software preroll.
inline bool independentVideoPacket(const AVPacket &packet, const AVCodecParameters &parameters) {
return (packet.flags & AV_PKT_FLAG_KEY) &&
(parameters.codec_id != AV_CODEC_ID_H264 || hasIDR(packet, parameters));
}
inline int prepareVideoPacket(AVCodecContext *&context, const AVCodecParameters &parameters,
const AVPacket &packet) {
if (!context->hw_device_ctx || independentVideoPacket(packet, parameters)) return 0;
auto *software = avcodec_alloc_context3(context->codec);
if (!software) return AVERROR(ENOMEM);
int result = avcodec_parameters_to_context(software, &parameters);
software->pkt_timebase = context->pkt_timebase;
software->thread_count = context->thread_count;
software->max_pixels = context->max_pixels;
if (result >= 0) result = avcodec_open2(software, context->codec, nullptr);
if (result < 0) { avcodec_free_context(&software); return result; }
avcodec_free_context(&context); context = software;
return 0;
}
// Mapping retains and locks the CVPixelBuffer until the last frame reference
// is released. Consumers see ordinary CPU-readable planes without a pixel copy.
inline int refVideoFrame(AVFrame *destination, const AVFrame *source) {
if (source->format == AV_PIX_FMT_VIDEOTOOLBOX)
return av_hwframe_map(destination, source, AV_HWFRAME_MAP_READ);
return av_frame_ref(destination, source);
}
}