hl-valdemar/terebi
1
0
Fork 0
Code Issues Pull Requests Actions Packages Releases Wiki Activity

feat: step to adjacent decoded video frames

Browse Source
This commit is contained in:
Valdemar Høgsvig Lorenzen
2026-10-10 17:49:04 +02:00
parent 310be5aa61
commit a99b0c1168
6 changed files with 311 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Sources/Core/player.cpp
+181 -24
View File
@@ -46,7 +46,9 @@ using Frame = std::unique_ptr<AVFrame, FrameDeleter>;
struct VideoFrame {
Frame frame;
double pts;
double end;
std::size_t bytes;
uint64_t timestampOccurrence = 0;
};
struct Track {
AVCodecContext *codec = nullptr;
@@ -54,18 +56,26 @@ struct Track {
bool drainSent = false;
bool drained = false;
double nextPts = 0;
int64_t countedTimestamp = AV_NOPTS_VALUE;
uint64_t timestampOccurrence = 0;
};
enum class Action { Open, Play, Pause, Seek, Close };
enum class Action { Open, Play, Pause, Seek, Step, Close };
struct Command {
Action action;
std::string path;
double seconds = 0;
int direction = 0;
uint64_t epoch = 0;
};
void copyError(char *destination, const char *message) noexcept {
std::snprintf(destination, 512, "%s", message ? message : "Unknown playback error");
}
int64_t presentationTimestamp(const AVFrame *frame) {
return frame->best_effort_timestamp != AV_NOPTS_VALUE ?
frame->best_effort_timestamp : frame->pts;
}
}
struct Player {
@@ -74,6 +84,7 @@ struct Player {
std::condition_variable wake;
std::deque<Command> commands;
PlayerState state{};
uint64_t publishedEpoch = 0;
std::array<char, 512> requestedError{};
bool errorRequested = false;
uint64_t errorEpoch = 0;
@@ -96,6 +107,9 @@ struct Player {
int receiving = -1;
std::deque<VideoFrame> video;
std::size_t videoBytes = 0;
VideoFrame presented{};
int64_t skipPresentedTimestamp = AV_NOPTS_VALUE;
uint64_t skipPresentedOccurrence = 0;
std::vector<float> pcm;
std::size_t pcmFrames = 0;
std::size_t pcmOffset = 0;
@@ -127,6 +141,10 @@ struct Player {
std::lock_guard<std::mutex> lock(mutex);
if (stopping.load(std::memory_order_relaxed) || commands.size() >= commandLimit)
return false;
if (command.action == Action::Step &&
(!state.has_video || publishedEpoch != requestedEpoch.load(std::memory_order_relaxed) ||
(state.status != PLAYER_PAUSED && state.status != PLAYER_PLAYING && state.status != PLAYER_ENDED)))
return false;
const bool newTimeline = command.action == Action::Open || command.action == Action::Close;
command.epoch = requestedEpoch.load(std::memory_order_relaxed) + (newTimeline ? 1 : 0);
commands.push_back(std::move(command));
@@ -137,7 +155,8 @@ struct Player {
state.status = PLAYER_OPENING;
state.revision = revision;
}
if (commands.back().action == Action::Seek) state.seek_pending = true;
if (commands.back().action == Action::Seek || commands.back().action == Action::Step)
state.seek_pending = true;
wake.notify_one();
return true;
}
@@ -175,14 +194,16 @@ struct Player {
void publish() {
std::lock_guard<std::mutex> lock(mutex);
if (activeEpoch != requestedEpoch.load(std::memory_order_relaxed)) return;
publishedEpoch = activeEpoch;
state.status = status;
state.position = position;
state.duration = duration;
state.has_audio = tracks[0].codec != nullptr;
state.has_video = tracks[1].codec != nullptr;
// The worker publishes only outside seek execution; retain pending for later queued seeks.
// Commands remain pending while executing; publication acknowledges only completed work.
state.seek_pending = std::any_of(commands.begin(), commands.end(), [this](const Command &command) {
return command.action == Action::Seek && command.epoch == activeEpoch;
return (command.action == Action::Seek || command.action == Action::Step) &&
command.epoch == activeEpoch;
});
state.video_aspect_ratio = 0;
if (const auto *video = tracks[1].codec; video && video->width > 0 && video->height > 0) {
@@ -199,6 +220,8 @@ struct Player {
output.reset();
video.clear();
videoBytes = 0;
presented = {};
skipPresentedTimestamp = AV_NOPTS_VALUE;
swr_free(&resampler);
av_channel_layout_uninit(&inputLayout);
for (auto &track : tracks) {
@@ -253,6 +276,8 @@ struct Player {
if (!track.codec) continue;
avcodec_flush_buffers(track.codec);
track.drainSent = track.drained = false;
track.countedTimestamp = AV_NOPTS_VALUE;
track.timestampOccurrence = 0;
const AVStream *stream = format->streams[track.index];
track.nextPts = stream->start_time == AV_NOPTS_VALUE ? target :
std::max(target, stream->start_time * av_q2d(stream->time_base) - origin);
@@ -265,6 +290,8 @@ struct Player {
av_frame_unref(decoded);
video.clear();
videoBytes = 0;
presented = {};
skipPresentedTimestamp = AV_NOPTS_VALUE;
pcmFrames = pcmOffset = 0;
packetPending = demuxEnded = resamplerDrained = false;
receiving = -1;
@@ -394,6 +421,108 @@ struct Player {
if (resume) start();
}
void seekVideo(int64_t timestamp, double target) {
output.pause();
check(av_seek_frame(format, tracks[1].index, timestamp, AVSEEK_FLAG_BACKWARD),
"Seeking video keyframe");
resetTimeline(target);
status = PLAYER_PAUSED;
}
void presentVideo(VideoFrame frame) {
output.present(frame.frame.get());
presented = std::move(frame);
previewShown = true;
}
VideoFrame nextSteppingFrame() {
while (video.empty() && !tracks[1].drained) {
if (interrupted()) throw Cancelled{};
if (!decodeOne(true)) {
std::unique_lock<std::mutex> lock(mutex);
wake.wait_for(lock, std::chrono::milliseconds(5), [this] { return interrupted(); });
}
}
if (video.empty()) return {};
VideoFrame frame = std::move(video.front());
video.pop_front();
videoBytes -= frame.bytes;
return frame;
}
void step(int direction) {
if (!format || !tracks[1].codec) return;
pause();
status = PLAYER_PAUSED;
// A seek followed immediately by a queued step must first establish its preview.
while (!presented.frame) {
auto frame = nextSteppingFrame();
if (!frame.frame) return;
if (frame.pts + sampleTime * 0.5 < seekTarget) continue;
presentVideo(std::move(frame));
}
VideoFrame selected = std::move(presented);
const int64_t reference = presentationTimestamp(selected.frame.get());
const uint64_t referenceOccurrence = selected.timestampOccurrence;
if (reference == AV_NOPTS_VALUE)
throw std::runtime_error("Frame stepping requires video presentation timestamps");
const AVStream *stream = format->streams[tracks[1].index];
const long double ticksPerSecond = 1.0L / av_q2d(stream->time_base);
const long double originTimestamp = std::clamp(std::floor(origin * ticksPerSecond),
static_cast<long double>(std::numeric_limits<int64_t>::min() + 1),
static_cast<long double>(std::numeric_limits<int64_t>::max()));
const int64_t beginning = std::min(reference, std::max(static_cast<int64_t>(originTimestamp),
stream->start_time == AV_NOPTS_VALUE ? static_cast<int64_t>(originTimestamp) : stream->start_time));
int64_t target = reference > beginning ? reference - 1 : reference;
long double backoff = std::max(1.0L, ticksPerSecond);
for (;;) {
// FFmpeg's decoder emits presentation order, including delayed B-frames at drain.
// ponytail: a one-GOP/unindexed file can require a prefix scan; retained frames stay bounded.
seekVideo(target, 0);
VideoFrame previous{};
bool sawReference = false;
for (;;) {
auto frame = nextSteppingFrame();
if (!frame.frame) break;
const int64_t timestamp = presentationTimestamp(frame.frame.get());
const bool atReference = timestamp == reference &&
frame.timestampOccurrence == referenceOccurrence;
const bool beforeReference = timestamp < reference ||
(timestamp == reference && frame.timestampOccurrence < referenceOccurrence);
if (atReference) sawReference = true;
if (direction > 0) {
if (!beforeReference && !atReference) {
if (sawReference) selected = std::move(frame);
break;
}
} else {
if (!beforeReference) break;
if (timestamp >= beginning) previous = std::move(frame);
}
}
if ((direction > 0 && sawReference) || previous.frame || target == beginning) {
if (previous.frame) selected = std::move(previous);
break;
}
// Some demuxers land later than requested. Expand the keyframe search, not an FPS guess.
target = static_cast<int64_t>(std::max(static_cast<long double>(beginning),
static_cast<long double>(reference) - backoff));
backoff *= 2;
}
const int64_t selectedTimestamp = presentationTimestamp(selected.frame.get());
const uint64_t selectedOccurrence = selected.timestampOccurrence;
const double selectedPts = selected.pts;
// Reuse the verified preroll anchor: timestamp seeks can land after a B-frame's PTS.
// Audio is rebuilt and trimmed to selectedPts, while video skips the displayed frame.
seekVideo(target, selectedPts);
if (interrupted()) throw Cancelled{};
videoEnd = selected.end;
presentVideo(std::move(selected));
skipPresentedTimestamp = selectedTimestamp;
skipPresentedOccurrence = selectedOccurrence;
position = selectedPts;
}
void execute(const Command &command) {
if (command.epoch != requestedEpoch.load(std::memory_order_relaxed)) return;
activeEpoch = command.epoch;
@@ -402,14 +531,14 @@ struct Player {
case Action::Play: start(); break;
case Action::Pause: pause(); break;
case Action::Seek: seek(command.seconds, status == PLAYER_PLAYING); break;
case Action::Step: step(command.direction); break;
case Action::Close: release(); break;
}
publish();
}
double framePts(Track &track, const AVFrame *frame) {
const int64_t timestamp = frame->best_effort_timestamp != AV_NOPTS_VALUE ?
frame->best_effort_timestamp : frame->pts;
const int64_t timestamp = presentationTimestamp(frame);
if (timestamp == AV_NOPTS_VALUE) return track.nextPts;
return timestamp * av_q2d(format->streams[track.index]->time_base) - origin;
}
@@ -522,17 +651,34 @@ struct Player {
(!video.empty() && videoBytes > videoByteLimit - video.back().bytes);
}
void queueVideo() {
void queueVideo(bool stepping = false) {
auto &track = tracks[1];
const auto transfer = decoded->color_trc == AVCOL_TRC_UNSPECIFIED ?
track.codec->color_trc : decoded->color_trc;
if (transfer == AVCOL_TRC_SMPTE2084 || transfer == AVCOL_TRC_ARIB_STD_B67)
throw std::runtime_error("HDR PQ/HLG video is unsupported: the native view currently supports SDR only");
const int64_t timestamp = presentationTimestamp(decoded);
if (stepping && timestamp == AV_NOPTS_VALUE)
throw std::runtime_error("Frame stepping requires video presentation timestamps");
const double pts = framePts(track, decoded);
const double frameDuration = decoded->duration > 0 ?
decoded->duration * av_q2d(format->streams[track.index]->time_base) : videoFrameTime;
track.nextPts = pts + frameDuration;
if (pts + sampleTime * 0.5 < seekTarget || (decoded->flags & AV_FRAME_FLAG_DISCARD)) return;
if (decoded->flags & AV_FRAME_FLAG_DISCARD) return;
if (timestamp != track.countedTimestamp) {
track.countedTimestamp = timestamp;
track.timestampOccurrence = 0;
} else {
++track.timestampOccurrence;
}
if (!stepping && pts + sampleTime * 0.5 < seekTarget) return;
if (!stepping && skipPresentedTimestamp != AV_NOPTS_VALUE) {
if (timestamp != AV_NOPTS_VALUE &&
(timestamp < skipPresentedTimestamp ||
(timestamp == skipPresentedTimestamp && track.timestampOccurrence <= skipPresentedOccurrence)))
return;
skipPresentedTimestamp = AV_NOPTS_VALUE;
}
const int size = av_image_get_buffer_size(static_cast<AVPixelFormat>(decoded->format),
decoded->width, decoded->height, 1);
check(size, "Sizing decoded video");
@@ -548,16 +694,20 @@ struct Player {
}
Frame frame(av_frame_clone(decoded));
if (!frame) throw std::bad_alloc();
VideoFrame item{std::move(frame), pts, bytes};
VideoFrame item{std::move(frame), pts, pts + frameDuration, bytes, track.timestampOccurrence};
auto insertion = std::upper_bound(video.begin(), video.end(), pts,
[](double value, const VideoFrame &entry) { return value < entry.pts; });
video.insert(insertion, std::move(item));
videoBytes += bytes;
}
bool decodeOne() {
if (pcmOffset < pcmFrames) return false;
bool decodeOne(bool stepping = false) {
if (!stepping && pcmOffset < pcmFrames) return false;
if (receiving >= 0) {
if (stepping && receiving == 0) {
receiving = -1;
return true;
}
auto &track = tracks[receiving];
if (receiving == 1 && videoFull() &&
(!tracks[0].codec || output.queued() >= 1024 || status != PLAYER_PLAYING))
@@ -573,11 +723,16 @@ struct Player {
return true;
}
check(result, "Decoding media");
if (receiving == 0) convertAudio(); else queueVideo();
if (receiving == 0) convertAudio(); else queueVideo(stepping);
av_frame_unref(decoded);
return true;
}
if (packetPending) {
if (stepping && packet->stream_index == tracks[0].index) {
av_packet_unref(packet);
packetPending = false;
return true;
}
const int slot = packet->stream_index == tracks[0].index ? 0 : 1;
const int result = avcodec_send_packet(tracks[slot].codec, packet);
if (result == AVERROR(EAGAIN)) {
@@ -593,7 +748,7 @@ struct Player {
if (demuxEnded) {
for (int slot = 0; slot < 2; ++slot) {
auto &track = tracks[slot];
if (!track.codec || track.drainSent) continue;
if (!track.codec || track.drainSent || (stepping && slot == 0)) continue;
const int result = avcodec_send_packet(track.codec, nullptr);
if (result == AVERROR(EAGAIN)) {
receiving = slot;
@@ -604,7 +759,7 @@ struct Player {
receiving = slot;
return true;
}
if ((!tracks[0].codec || tracks[0].drained) && !resamplerDrained) {
if (!stepping && (!tracks[0].codec || tracks[0].drained) && !resamplerDrained) {
drainResampler();
return true;
}
@@ -617,7 +772,8 @@ struct Player {
}
if (result == AVERROR(EAGAIN)) return false;
check(result, "Reading media packet");
if (packet->stream_index != tracks[0].index && packet->stream_index != tracks[1].index) {
if ((packet->stream_index != tracks[0].index && packet->stream_index != tracks[1].index) ||
(stepping && packet->stream_index == tracks[0].index)) {
av_packet_unref(packet);
} else {
packetPending = true;
@@ -626,16 +782,13 @@ struct Player {
}
void renderVideo(double now) {
Frame latest;
VideoFrame latest{};
while (!video.empty() && video.front().pts <= now + 0.002) {
videoBytes -= video.front().bytes;
latest = std::move(video.front().frame);
latest = std::move(video.front());
video.pop_front();
}
if (latest) {
output.present(latest.get());
previewShown = true;
}
if (latest.frame) presentVideo(std::move(latest));
}
bool tick() {
@@ -643,10 +796,10 @@ struct Player {
bool progressed = pumpAudio();
if (status == PLAYER_PAUSED) {
if (!previewShown && !video.empty()) {
output.present(video.front().frame.get());
videoBytes -= video.front().bytes;
auto frame = std::move(video.front());
videoBytes -= frame.bytes;
video.pop_front();
previewShown = true;
presentVideo(std::move(frame));
progressed = true;
}
const bool needPreview = tracks[1].codec && !previewShown;
@@ -756,6 +909,10 @@ bool player_seek(Player *player, double seconds) {
if (!std::isfinite(seconds)) return false;
return submit(player, [=] { return player->enqueue({Action::Seek, {}, seconds}); });
}
bool player_step_frame(Player *player, int direction) {
if (direction != -1 && direction != 1) return false;
return submit(player, [=] { return player->enqueue({Action::Step, {}, 0, direction}); });
}
bool player_close(Player *player) {
return submit(player, [=] { return player->enqueue({Action::Close, {}}); });
}