From a99b0c116891fa7aacae14d5c17f324cbdba8708 Mon Sep 17 00:00:00 2001 From: hl-valdemar Date: Sat, 10 Oct 2026 17:49:04 +0200 Subject: [PATCH] feat: step to adjacent decoded video frames --- Checks/playback.mm | 101 ++++++++++++++++++ Makefile | 2 + README.md | 13 ++- Sources/App/PlayerApp.swift | 14 +++ Sources/Core/player.cpp | 205 +++++++++++++++++++++++++++++++----- Sources/Core/player.h | 5 +- 6 files changed, 311 insertions(+), 29 deletions(-) diff --git a/Checks/playback.mm b/Checks/playback.mm index 15c888e..25f4721 100644 --- a/Checks/playback.mm +++ b/Checks/playback.mm @@ -31,12 +31,109 @@ static PlayerState until(Player *player, const std::function std::exit(1); } +static PlayerState frameCompleted(Player *player, uint64_t revision, double pts) { + auto state = until(player, [revision](auto s) { + return !s.seek_pending && s.revision != revision && s.status == PLAYER_PAUSED; + }); + if (std::abs(state.position - pts) >= .0001) { + std::fprintf(stderr, "Frame PTS: expected %.4f, got %.4f\n", pts, state.position); + require(false, "step selects exact adjacent presentation timestamp"); + } + return state; +} + +static PlayerState step(Player *player, int direction, double pts) { + auto revision = player_get_state(player).revision; + require(player_step_frame(player, direction), "frame step accepted"); + return frameCompleted(player, revision, pts); +} + +static void frameSteps(Player *player, MTKView *view, const std::string &directory) { + // Independent literals from the fixture's integer-millisecond setpts input. + constexpr double pts[] = {0, .1, .2, .3, .4, .65, .9, 1.15, 1.4, 1.5, 1.6, 1.7}; + require(player_open(player, (directory + "/frame-step.mp4").c_str()), "VFR open accepted"); + until(player, [](auto s) { return !s.seek_pending && s.status == PLAYER_PAUSED; }); + require(!player_step_frame(player, 0) && !player_step_frame(player, 2) && + !player_step_frame(player, -2), "only adjacent frame directions accepted"); + step(player, -1, 0); // First boundary also ensures the initial preview is ready. + for (unsigned i = 1; i < sizeof(pts) / sizeof(pts[0]); ++i) { + step(player, 1, pts[i]); + [view draw]; + } + step(player, 1, 1.7); + for (int i = 10; i >= 0; --i) step(player, -1, pts[i]); + step(player, -1, 0); + + auto revision = player_get_state(player).revision; + require(player_step_frame(player, 1) && player_step_frame(player, 1) && + player_step_frame(player, 1) && player_step_frame(player, 1), + "repeated queued steps accepted"); + frameCompleted(player, revision, .4); + revision = player_get_state(player).revision; + require(player_step_frame(player, 1) && player_step_frame(player, 1) && + player_step_frame(player, -1), "mixed queued steps accepted"); + frameCompleted(player, revision, .65); + + revision = player_get_state(player).revision; + require(player_seek(player, .72), "non-keyframe seek accepted"); + until(player, [revision](auto s) { + return !s.seek_pending && s.revision != revision && s.status == PLAYER_PAUSED; + }); + // Seek previews .9, not the .72 authoritative seek clock and not keyframe zero. + step(player, 1, 1.15); + step(player, -1, .9); + step(player, -1, .65); + revision = player_get_state(player).revision; + require(player_seek(player, .72) && player_step_frame(player, 1) && + player_step_frame(player, -1), "queued seek and steps accepted"); + frameCompleted(player, revision, .9); + step(player, -1, .65); + + require(player_play(player), "play before frame step accepted"); + until(player, [](auto s) { return s.status == PLAYER_PLAYING && s.position > .67 && s.position < .8; }); + auto frozen = step(player, 1, .9); + [view draw]; + for (int i = 0; i < 20; ++i) + CFRunLoopRunInMode(kCFRunLoopDefaultMode, .005, false); + auto paused = player_get_state(player); + require(paused.status == PLAYER_PAUSED && !paused.seek_pending && + std::abs(paused.position - frozen.position) < .0001, + "stepping during playback pauses and freezes audio/video timeline"); + require(player_play(player), "resume stepped timeline accepted"); + until(player, [](auto s) { return s.status == PLAYER_PLAYING && s.position > 1.19 && s.position < 1.32; }); + require(player_pause(player), "pause resumed timeline accepted"); + until(player, [](auto s) { return s.status == PLAYER_PAUSED; }); + // Resumed presentation has advanced from .9 to 1.15; no pre-step frame remains. + step(player, -1, .9); + require(player_play(player), "play stepped media through EOF"); + until(player, [](auto s) { return s.status == PLAYER_ENDED; }); + step(player, 1, 1.7); + step(player, -1, 1.6); + step(player, 1, 1.7); + step(player, 1, 1.7); + require(player_open(player, (directory + "/video.mp4").c_str()), "video-only stepping open"); + until(player, [](auto s) { return s.status == PLAYER_PAUSED; }); + revision = player_get_state(player).revision; + require(player_seek(player, 3.9), "seek near video-only last frame"); + until(player, [revision](auto s) { return !s.seek_pending && s.revision != revision; }); + step(player, 1, 95.0 / 24); + require(player_play(player), "resume video-only last frame"); + auto ended = until(player, [](auto s) { return s.status == PLAYER_ENDED; }); + require(std::abs(ended.position - 4) < .0001, "last stepped frame retains its presentation duration"); + step(player, -1, 94.0 / 24); + require(player_close(player), "close stepped media accepted"); + until(player, [](auto s) { return s.status == PLAYER_EMPTY; }); + std::puts("PASS exact VFR/B-frame steps/queued steps/seek reference/pause/resume/boundaries"); +} + int main(int argc, char **argv) { @autoreleasepool { require(argc == 2, "fixture directory argument required"); [NSApplication sharedApplication]; std::unique_ptr player(player_create(), player_destroy); require(bool(player), "player creation"); + require(!player_step_frame(nullptr, 1) && !player_step_frame(player.get(), 1), + "null and empty players reject frame steps"); void *native = player_macos_make_video_view(player.get()); require(native != nullptr, "Metal view and shaders initialize"); MTKView *view = (__bridge_transfer MTKView *)native; @@ -56,6 +153,9 @@ int main(int argc, char **argv) { "audio track selection"); double aspect = !opened.has_video ? 0 : std::string(fixture) == "anamorphic.mp4" ? 8.0 / 3 : 16.0 / 9; require(std::abs(opened.video_aspect_ratio - aspect) < .0001, "display aspect includes non-square pixels"); + if (!opened.has_video) + require(!player_step_frame(player.get(), 1) && !player_step_frame(player.get(), -1), + "audio-only media rejects frame steps"); require(!player_seek(player.get(), NAN), "non-finite seek rejected"); require(player_play(player.get()), "play accepted"); until(player.get(), [](auto s) { return s.status == PLAYER_PLAYING && s.position > .35; }); @@ -98,6 +198,7 @@ int main(int argc, char **argv) { "close clears media state and pending seeks"); std::printf("PASS %s: open/play/pause/seek/end/replay/close\n", fixture); } + frameSteps(player.get(), view, argv[1]); require(player_open(player.get(), (std::string(argv[1]) + "/hdr.mkv").c_str()), "HDR open accepted"); auto hdr = until(player.get(), [](auto s) { return s.status == PLAYER_ERROR; }, true); require(std::string(hdr.error).find("HDR") != std::string::npos, "HDR reports unsupported output"); diff --git a/Makefile b/Makefile index e061a22..2fa11af 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,8 @@ check: .build/playback-check check-audio ffmpeg -hide_banner -loglevel error -y -i .build/fixtures/av.mp4 -an -c:v copy .build/fixtures/video.mp4 ffmpeg -hide_banner -loglevel error -y -i .build/fixtures/av.mp4 -an -vf setsar=3/2 -c:v mpeg4 .build/fixtures/anamorphic.mp4 ffmpeg -hide_banner -loglevel error -y -i .build/fixtures/av.mp4 -vn -c:a pcm_s16le .build/fixtures/audio.wav + # Literal frame PTS: 0,.1,.2,.3,.4,.65,.9,1.15,1.4,1.5,1.6,1.7; reordered B-frames across short GOPs. + ffmpeg -hide_banner -loglevel error -y -f lavfi -i testsrc2=size=320x180:rate=10:duration=1.2 -f lavfi -i sine=frequency=440:sample_rate=44100:duration=2 -vf "settb=1/1000,setpts='if(lt(N,5),N*100,if(lt(N,9),400+(N-4)*250,1400+(N-8)*100))'" -fps_mode vfr -enc_time_base:v 1:1000 -c:v mpeg4 -g 4 -bf 2 -video_track_timescale 1000 -c:a aac .build/fixtures/frame-step.mp4 ffmpeg -hide_banner -loglevel error -y -f lavfi -i testsrc2=size=160x90:rate=12 -t 1 -vf 'format=yuv420p10le,setparams=color_primaries=bt2020:color_trc=smpte2084:colorspace=bt2020nc' -c:v ffv1 .build/fixtures/hdr.mkv .build/playback-check .build/fixtures diff --git a/README.md b/README.md index 46d4d91..f07e771 100644 --- a/README.md +++ b/README.md @@ -27,12 +27,15 @@ Opening video fits the window to its display aspect ratio, including non-square | `K` / `Space` | Toggle playback | | `M` | Toggle mute, restoring the previous audible level | | `J` / `L` | Seek backward / forward ten seconds | +| `,` / `.` | Step to the previous / next video frame; leave playback paused | | `F` | Toggle fullscreen | | `P` | Toggle keep-on-top (windowed mode) | | `O` | Open the file picker to switch files | | `Q` | Quit through normal application shutdown | -Single-key shortcuts apply only to the playback window and do not intercept text editing, file-picker input, or Command/Control/Option combinations. Held toggle keys do not repeatedly toggle; held `J`/`L` can continue seeking. `,` / `.` frame stepping and `C` captions are unbound until supported. +Single-key shortcuts apply only to the playback window and do not intercept text editing, file-picker input, or Command/Control/Option combinations. Held toggle keys do not repeatedly toggle; held `J`/`L` continue seeking and held `,`/`.` continue stepping. `C` captions remains unbound. + +Frame stepping uses decoded presentation timestamps, not an average-FPS time offset, including variable-frame-rate video and reordered B-frames. Steps start from the displayed frame, stop at the first/last frame, and keep the timeline paused. Audio-only files disable stepping; normal playback resumes with audio synchronized to the selected frame. - Existing shortcuts remain: **⌘O** opens files, **←/→** seeks five seconds, **↑/↓** adjusts volume by five percent, **⌃⌘F** toggles fullscreen, and **Esc** leaves fullscreen. Double-clicking video also toggles fullscreen. - The timeline slider seeks on release and holds the requested position until queued seeking finishes. Both slider tracks span the native thumb-center travel range; the orange fill ends at the current volume thumb center. The volume thumb is opaque light gray. @@ -43,8 +46,8 @@ Only implemented actions have controls; there are no placeholder playlist, subti ## Where to change things -- `Sources/Core/player.h`: the C interface and authoritative state snapshot. Playback commands enqueue work; poll state for completion. `seek_pending` is set synchronously when a seek is accepted and clears when all queued seeks on that timeline finish. Volume set/get is synchronous and uses a lock-free target gain. Callers own the opaque handle and must stop using it before destruction. -- `Sources/Core/player.cpp`: single demux/decode worker, bounded buffers, timestamps, audio-master synchronization, paused previews, decoder/resampler drain, seeking, errors, and shutdown. A seek stops output and flushes all old timeline state before resuming. +- `Sources/Core/player.h`: the C interface and authoritative state snapshot. Playback commands enqueue work; poll state for completion. `seek_pending` is set synchronously when a seek or frame step is accepted and clears when all queued seeks/steps on that timeline finish. `player_step_frame` accepts only `-1` or `+1`, requires ready video, and publishes the selected frame's timestamp as `position`. Volume set/get is synchronous and uses a lock-free target gain. Callers own the opaque handle and must stop using it before destruction. +- `Sources/Core/player.cpp`: single demux/decode worker, bounded buffers, timestamps, audio-master synchronization, paused previews, exact frame stepping, decoder/resampler drain, seeking, errors, and shutdown. A seek stops output and flushes all old timeline state before resuming. - `Sources/Core/output.hpp` and `Sources/macOS/Output.mm`: native output contract, allocation-free/nonblocking Core Audio callback and bounded audio ring, latency-aware presentation clock, retained frame mailbox, and Metal YUV conversion. The view can outlive the player safely. - `Sources/App/PlayerApp.swift`: MainActor state polling, file panel/Finder/drop handling, video host, AppKit glass and native slider cells, floating controls, visibility, video-fitted sizing, and native window integration. - `Makefile`: compiler/linker commands and app bundle construction. No generated Xcode project or additional build dependency. @@ -55,7 +58,7 @@ Only implemented actions have controls; there are no placeholder playlist, subti make check ``` -Generates short fixtures and exercises the real playback interface and native outputs: audio/video, video-only, audio-only, non-square-pixel display aspect, pause, paused/playing and queued seek acknowledgement, completion, replay, close, HDR rejection, failed-open recovery, and destruction while playing. It initializes the actual Metal shaders and uses an offscreen native window. It requires a working audio device and graphical macOS session, and plays short sine-wave audio. It is a runtime smoke check, not a mocked clock/device test. +Generates short fixtures and exercises the real playback interface and native outputs: audio/video, video-only, audio-only, non-square-pixel display aspect, pause, paused/playing and queued seek acknowledgement, exact bidirectional VFR/B-frame stepping across keyframes, queued/mixed steps, seek-to-step transitions, first/last-frame boundaries, stepping pause/resume, completion, replay, close, HDR rejection, failed-open recovery, and destruction while playing. It initializes the actual Metal shaders and uses an offscreen native window. It requires a working audio device and graphical macOS session, and plays short sine-wave audio. It is a runtime smoke check, not a mocked clock/device test. `make check-audio` runs just the audio-quality check. A check-only hook captures actual Core Audio callback PCM into preallocated storage and compares every sample against FFmpeg's reference output for PCM/AAC, 44.1/48 kHz, and coarse container timestamps. It checks fractional gain, silent muted startup, smooth live volume changes, and unchanged playback completion. A separate gapped fixture verifies that real timestamp gaps remain silent. Release builds contain no capture hook. @@ -69,4 +72,6 @@ Visual correctness, perceived A/V synchronization, and control usability still n Local regular files, automatically selected best audio/video tracks, software decoding, and SDR output. PQ/HLG HDR is rejected rather than rendered incorrectly. No subtitles, selectable tracks, VideoToolbox decoding, rotation/deinterlacing, or network playback yet. Mid-track audio format changes report an error. +Frame stepping re-decodes from an earlier keyframe and rebuilds audio at the selected timestamp. Long-GOP or unindexed files can therefore step slowly; frames are not cached across the entire file. Stepping requires usable decoded presentation timestamps and reports an error rather than inventing frame times when they are absent. + Audio buffering is bounded to one second plus one decoded block (up to four seconds); video is bounded to 64 frames/128 MiB. Extremely interleaved files can drop far-future video frames to avoid starving audio; the ceiling and upgrade path are marked at `queueVideo()`. Device/stream latency is accounted for, but device changes during playback are not handled yet. diff --git a/Sources/App/PlayerApp.swift b/Sources/App/PlayerApp.swift index 2979147..ee96af3 100644 --- a/Sources/App/PlayerApp.swift +++ b/Sources/App/PlayerApp.swift @@ -52,6 +52,7 @@ final class PlayerModel: ObservableObject { (state.status == PLAYER_PAUSED || state.status == PLAYER_PLAYING || state.status == PLAYER_ENDED) } var canSeek: Bool { canTogglePlayback && state.duration.isFinite && state.duration > 0 } + var canStepFrame: Bool { canTogglePlayback && state.has_video && !isSeeking } var canOpen: Bool { handle != nil && !isBusy } var canClose: Bool { handle != nil && !isBusy && state.status != PLAYER_EMPTY } var canSetVolume: Bool { handle != nil && !isBusy && state.has_audio && state.status != PLAYER_ERROR } @@ -155,6 +156,14 @@ final class PlayerModel: ObservableObject { revealControls() } + func stepFrame(_ direction: Int32) { + guard canStepFrame, let handle else { return } + let accepted = player_step_frame(handle, direction) + if accepted { seekPending = true } + commandError = accepted ? nil : "Could not queue the frame step. Please try again." + revealControls() + } + func setVolume(_ value: Double) { guard let handle else { return } if player_set_volume(handle, Float(value)) { @@ -697,6 +706,8 @@ private struct WindowChrome: NSViewRepresentable { switch key { case "j": model.seekRelative(-10) case "l": model.seekRelative(10) + case ",": model.stepFrame(-1) + case ".": model.stepFrame(1) case "k", " ": if !event.isARepeat { model.togglePlayback() } case "m": @@ -825,6 +836,9 @@ struct PlayerApp: App { Button("Seek Back 10 Seconds") { model.seekRelative(-10) }.disabled(!model.canSeek) Button("Seek Forward 10 Seconds") { model.seekRelative(10) }.disabled(!model.canSeek) Divider() + Button("Previous Frame") { model.stepFrame(-1) }.disabled(!model.canStepFrame) + Button("Next Frame") { model.stepFrame(1) }.disabled(!model.canStepFrame) + Divider() Button("Volume Up") { model.setVolume(min(1, model.volume + 0.05)) } .keyboardShortcut(.upArrow, modifiers: []).disabled(!model.canSetVolume) Button("Volume Down") { model.setVolume(max(0, model.volume - 0.05)) } diff --git a/Sources/Core/player.cpp b/Sources/Core/player.cpp index 624ae46..56c50f1 100644 --- a/Sources/Core/player.cpp +++ b/Sources/Core/player.cpp @@ -46,7 +46,9 @@ using Frame = std::unique_ptr; struct VideoFrame { Frame frame; double pts; + double end; std::size_t bytes; + uint64_t timestampOccurrence = 0; }; struct Track { AVCodecContext *codec = nullptr; @@ -54,18 +56,26 @@ struct Track { bool drainSent = false; bool drained = false; double nextPts = 0; + int64_t countedTimestamp = AV_NOPTS_VALUE; + uint64_t timestampOccurrence = 0; }; -enum class Action { Open, Play, Pause, Seek, Close }; +enum class Action { Open, Play, Pause, Seek, Step, Close }; struct Command { Action action; std::string path; double seconds = 0; + int direction = 0; uint64_t epoch = 0; }; void copyError(char *destination, const char *message) noexcept { std::snprintf(destination, 512, "%s", message ? message : "Unknown playback error"); } + +int64_t presentationTimestamp(const AVFrame *frame) { + return frame->best_effort_timestamp != AV_NOPTS_VALUE ? + frame->best_effort_timestamp : frame->pts; +} } struct Player { @@ -74,6 +84,7 @@ struct Player { std::condition_variable wake; std::deque commands; PlayerState state{}; + uint64_t publishedEpoch = 0; std::array requestedError{}; bool errorRequested = false; uint64_t errorEpoch = 0; @@ -96,6 +107,9 @@ struct Player { int receiving = -1; std::deque video; std::size_t videoBytes = 0; + VideoFrame presented{}; + int64_t skipPresentedTimestamp = AV_NOPTS_VALUE; + uint64_t skipPresentedOccurrence = 0; std::vector pcm; std::size_t pcmFrames = 0; std::size_t pcmOffset = 0; @@ -127,6 +141,10 @@ struct Player { std::lock_guard lock(mutex); if (stopping.load(std::memory_order_relaxed) || commands.size() >= commandLimit) return false; + if (command.action == Action::Step && + (!state.has_video || publishedEpoch != requestedEpoch.load(std::memory_order_relaxed) || + (state.status != PLAYER_PAUSED && state.status != PLAYER_PLAYING && state.status != PLAYER_ENDED))) + return false; const bool newTimeline = command.action == Action::Open || command.action == Action::Close; command.epoch = requestedEpoch.load(std::memory_order_relaxed) + (newTimeline ? 1 : 0); commands.push_back(std::move(command)); @@ -137,7 +155,8 @@ struct Player { state.status = PLAYER_OPENING; state.revision = revision; } - if (commands.back().action == Action::Seek) state.seek_pending = true; + if (commands.back().action == Action::Seek || commands.back().action == Action::Step) + state.seek_pending = true; wake.notify_one(); return true; } @@ -175,14 +194,16 @@ struct Player { void publish() { std::lock_guard lock(mutex); if (activeEpoch != requestedEpoch.load(std::memory_order_relaxed)) return; + publishedEpoch = activeEpoch; state.status = status; state.position = position; state.duration = duration; state.has_audio = tracks[0].codec != nullptr; state.has_video = tracks[1].codec != nullptr; - // The worker publishes only outside seek execution; retain pending for later queued seeks. + // Commands remain pending while executing; publication acknowledges only completed work. state.seek_pending = std::any_of(commands.begin(), commands.end(), [this](const Command &command) { - return command.action == Action::Seek && command.epoch == activeEpoch; + return (command.action == Action::Seek || command.action == Action::Step) && + command.epoch == activeEpoch; }); state.video_aspect_ratio = 0; if (const auto *video = tracks[1].codec; video && video->width > 0 && video->height > 0) { @@ -199,6 +220,8 @@ struct Player { output.reset(); video.clear(); videoBytes = 0; + presented = {}; + skipPresentedTimestamp = AV_NOPTS_VALUE; swr_free(&resampler); av_channel_layout_uninit(&inputLayout); for (auto &track : tracks) { @@ -253,6 +276,8 @@ struct Player { if (!track.codec) continue; avcodec_flush_buffers(track.codec); track.drainSent = track.drained = false; + track.countedTimestamp = AV_NOPTS_VALUE; + track.timestampOccurrence = 0; const AVStream *stream = format->streams[track.index]; track.nextPts = stream->start_time == AV_NOPTS_VALUE ? target : std::max(target, stream->start_time * av_q2d(stream->time_base) - origin); @@ -265,6 +290,8 @@ struct Player { av_frame_unref(decoded); video.clear(); videoBytes = 0; + presented = {}; + skipPresentedTimestamp = AV_NOPTS_VALUE; pcmFrames = pcmOffset = 0; packetPending = demuxEnded = resamplerDrained = false; receiving = -1; @@ -394,6 +421,108 @@ struct Player { if (resume) start(); } + void seekVideo(int64_t timestamp, double target) { + output.pause(); + check(av_seek_frame(format, tracks[1].index, timestamp, AVSEEK_FLAG_BACKWARD), + "Seeking video keyframe"); + resetTimeline(target); + status = PLAYER_PAUSED; + } + + void presentVideo(VideoFrame frame) { + output.present(frame.frame.get()); + presented = std::move(frame); + previewShown = true; + } + + VideoFrame nextSteppingFrame() { + while (video.empty() && !tracks[1].drained) { + if (interrupted()) throw Cancelled{}; + if (!decodeOne(true)) { + std::unique_lock lock(mutex); + wake.wait_for(lock, std::chrono::milliseconds(5), [this] { return interrupted(); }); + } + } + if (video.empty()) return {}; + VideoFrame frame = std::move(video.front()); + video.pop_front(); + videoBytes -= frame.bytes; + return frame; + } + + void step(int direction) { + if (!format || !tracks[1].codec) return; + pause(); + status = PLAYER_PAUSED; + // A seek followed immediately by a queued step must first establish its preview. + while (!presented.frame) { + auto frame = nextSteppingFrame(); + if (!frame.frame) return; + if (frame.pts + sampleTime * 0.5 < seekTarget) continue; + presentVideo(std::move(frame)); + } + VideoFrame selected = std::move(presented); + const int64_t reference = presentationTimestamp(selected.frame.get()); + const uint64_t referenceOccurrence = selected.timestampOccurrence; + if (reference == AV_NOPTS_VALUE) + throw std::runtime_error("Frame stepping requires video presentation timestamps"); + const AVStream *stream = format->streams[tracks[1].index]; + const long double ticksPerSecond = 1.0L / av_q2d(stream->time_base); + const long double originTimestamp = std::clamp(std::floor(origin * ticksPerSecond), + static_cast(std::numeric_limits::min() + 1), + static_cast(std::numeric_limits::max())); + const int64_t beginning = std::min(reference, std::max(static_cast(originTimestamp), + stream->start_time == AV_NOPTS_VALUE ? static_cast(originTimestamp) : stream->start_time)); + int64_t target = reference > beginning ? reference - 1 : reference; + long double backoff = std::max(1.0L, ticksPerSecond); + for (;;) { + // FFmpeg's decoder emits presentation order, including delayed B-frames at drain. + // ponytail: a one-GOP/unindexed file can require a prefix scan; retained frames stay bounded. + seekVideo(target, 0); + VideoFrame previous{}; + bool sawReference = false; + for (;;) { + auto frame = nextSteppingFrame(); + if (!frame.frame) break; + const int64_t timestamp = presentationTimestamp(frame.frame.get()); + const bool atReference = timestamp == reference && + frame.timestampOccurrence == referenceOccurrence; + const bool beforeReference = timestamp < reference || + (timestamp == reference && frame.timestampOccurrence < referenceOccurrence); + if (atReference) sawReference = true; + if (direction > 0) { + if (!beforeReference && !atReference) { + if (sawReference) selected = std::move(frame); + break; + } + } else { + if (!beforeReference) break; + if (timestamp >= beginning) previous = std::move(frame); + } + } + if ((direction > 0 && sawReference) || previous.frame || target == beginning) { + if (previous.frame) selected = std::move(previous); + break; + } + // Some demuxers land later than requested. Expand the keyframe search, not an FPS guess. + target = static_cast(std::max(static_cast(beginning), + static_cast(reference) - backoff)); + backoff *= 2; + } + const int64_t selectedTimestamp = presentationTimestamp(selected.frame.get()); + const uint64_t selectedOccurrence = selected.timestampOccurrence; + const double selectedPts = selected.pts; + // Reuse the verified preroll anchor: timestamp seeks can land after a B-frame's PTS. + // Audio is rebuilt and trimmed to selectedPts, while video skips the displayed frame. + seekVideo(target, selectedPts); + if (interrupted()) throw Cancelled{}; + videoEnd = selected.end; + presentVideo(std::move(selected)); + skipPresentedTimestamp = selectedTimestamp; + skipPresentedOccurrence = selectedOccurrence; + position = selectedPts; + } + void execute(const Command &command) { if (command.epoch != requestedEpoch.load(std::memory_order_relaxed)) return; activeEpoch = command.epoch; @@ -402,14 +531,14 @@ struct Player { case Action::Play: start(); break; case Action::Pause: pause(); break; case Action::Seek: seek(command.seconds, status == PLAYER_PLAYING); break; + case Action::Step: step(command.direction); break; case Action::Close: release(); break; } publish(); } double framePts(Track &track, const AVFrame *frame) { - const int64_t timestamp = frame->best_effort_timestamp != AV_NOPTS_VALUE ? - frame->best_effort_timestamp : frame->pts; + const int64_t timestamp = presentationTimestamp(frame); if (timestamp == AV_NOPTS_VALUE) return track.nextPts; return timestamp * av_q2d(format->streams[track.index]->time_base) - origin; } @@ -522,17 +651,34 @@ struct Player { (!video.empty() && videoBytes > videoByteLimit - video.back().bytes); } - void queueVideo() { + void queueVideo(bool stepping = false) { auto &track = tracks[1]; const auto transfer = decoded->color_trc == AVCOL_TRC_UNSPECIFIED ? track.codec->color_trc : decoded->color_trc; if (transfer == AVCOL_TRC_SMPTE2084 || transfer == AVCOL_TRC_ARIB_STD_B67) throw std::runtime_error("HDR PQ/HLG video is unsupported: the native view currently supports SDR only"); + const int64_t timestamp = presentationTimestamp(decoded); + if (stepping && timestamp == AV_NOPTS_VALUE) + throw std::runtime_error("Frame stepping requires video presentation timestamps"); const double pts = framePts(track, decoded); const double frameDuration = decoded->duration > 0 ? decoded->duration * av_q2d(format->streams[track.index]->time_base) : videoFrameTime; track.nextPts = pts + frameDuration; - if (pts + sampleTime * 0.5 < seekTarget || (decoded->flags & AV_FRAME_FLAG_DISCARD)) return; + if (decoded->flags & AV_FRAME_FLAG_DISCARD) return; + if (timestamp != track.countedTimestamp) { + track.countedTimestamp = timestamp; + track.timestampOccurrence = 0; + } else { + ++track.timestampOccurrence; + } + if (!stepping && pts + sampleTime * 0.5 < seekTarget) return; + if (!stepping && skipPresentedTimestamp != AV_NOPTS_VALUE) { + if (timestamp != AV_NOPTS_VALUE && + (timestamp < skipPresentedTimestamp || + (timestamp == skipPresentedTimestamp && track.timestampOccurrence <= skipPresentedOccurrence))) + return; + skipPresentedTimestamp = AV_NOPTS_VALUE; + } const int size = av_image_get_buffer_size(static_cast(decoded->format), decoded->width, decoded->height, 1); check(size, "Sizing decoded video"); @@ -548,16 +694,20 @@ struct Player { } Frame frame(av_frame_clone(decoded)); if (!frame) throw std::bad_alloc(); - VideoFrame item{std::move(frame), pts, bytes}; + VideoFrame item{std::move(frame), pts, pts + frameDuration, bytes, track.timestampOccurrence}; auto insertion = std::upper_bound(video.begin(), video.end(), pts, [](double value, const VideoFrame &entry) { return value < entry.pts; }); video.insert(insertion, std::move(item)); videoBytes += bytes; } - bool decodeOne() { - if (pcmOffset < pcmFrames) return false; + bool decodeOne(bool stepping = false) { + if (!stepping && pcmOffset < pcmFrames) return false; if (receiving >= 0) { + if (stepping && receiving == 0) { + receiving = -1; + return true; + } auto &track = tracks[receiving]; if (receiving == 1 && videoFull() && (!tracks[0].codec || output.queued() >= 1024 || status != PLAYER_PLAYING)) @@ -573,11 +723,16 @@ struct Player { return true; } check(result, "Decoding media"); - if (receiving == 0) convertAudio(); else queueVideo(); + if (receiving == 0) convertAudio(); else queueVideo(stepping); av_frame_unref(decoded); return true; } if (packetPending) { + if (stepping && packet->stream_index == tracks[0].index) { + av_packet_unref(packet); + packetPending = false; + return true; + } const int slot = packet->stream_index == tracks[0].index ? 0 : 1; const int result = avcodec_send_packet(tracks[slot].codec, packet); if (result == AVERROR(EAGAIN)) { @@ -593,7 +748,7 @@ struct Player { if (demuxEnded) { for (int slot = 0; slot < 2; ++slot) { auto &track = tracks[slot]; - if (!track.codec || track.drainSent) continue; + if (!track.codec || track.drainSent || (stepping && slot == 0)) continue; const int result = avcodec_send_packet(track.codec, nullptr); if (result == AVERROR(EAGAIN)) { receiving = slot; @@ -604,7 +759,7 @@ struct Player { receiving = slot; return true; } - if ((!tracks[0].codec || tracks[0].drained) && !resamplerDrained) { + if (!stepping && (!tracks[0].codec || tracks[0].drained) && !resamplerDrained) { drainResampler(); return true; } @@ -617,7 +772,8 @@ struct Player { } if (result == AVERROR(EAGAIN)) return false; check(result, "Reading media packet"); - if (packet->stream_index != tracks[0].index && packet->stream_index != tracks[1].index) { + if ((packet->stream_index != tracks[0].index && packet->stream_index != tracks[1].index) || + (stepping && packet->stream_index == tracks[0].index)) { av_packet_unref(packet); } else { packetPending = true; @@ -626,16 +782,13 @@ struct Player { } void renderVideo(double now) { - Frame latest; + VideoFrame latest{}; while (!video.empty() && video.front().pts <= now + 0.002) { videoBytes -= video.front().bytes; - latest = std::move(video.front().frame); + latest = std::move(video.front()); video.pop_front(); } - if (latest) { - output.present(latest.get()); - previewShown = true; - } + if (latest.frame) presentVideo(std::move(latest)); } bool tick() { @@ -643,10 +796,10 @@ struct Player { bool progressed = pumpAudio(); if (status == PLAYER_PAUSED) { if (!previewShown && !video.empty()) { - output.present(video.front().frame.get()); - videoBytes -= video.front().bytes; + auto frame = std::move(video.front()); + videoBytes -= frame.bytes; video.pop_front(); - previewShown = true; + presentVideo(std::move(frame)); progressed = true; } const bool needPreview = tracks[1].codec && !previewShown; @@ -756,6 +909,10 @@ bool player_seek(Player *player, double seconds) { if (!std::isfinite(seconds)) return false; return submit(player, [=] { return player->enqueue({Action::Seek, {}, seconds}); }); } +bool player_step_frame(Player *player, int direction) { + if (direction != -1 && direction != 1) return false; + return submit(player, [=] { return player->enqueue({Action::Step, {}, 0, direction}); }); +} bool player_close(Player *player) { return submit(player, [=] { return player->enqueue({Action::Close, {}}); }); } diff --git a/Sources/Core/player.h b/Sources/Core/player.h index 4bb6ea0..a205aac 100644 --- a/Sources/Core/player.h +++ b/Sources/Core/player.h @@ -17,7 +17,7 @@ typedef struct PlayerState { double duration; bool has_audio; bool has_video; - // True from accepted seek until all queued seeks on this timeline finish. + // True until all accepted seeks/frame steps on this timeline finish. bool seek_pending; // Display width / height, including sample aspect ratio; zero without video. double video_aspect_ratio; @@ -32,6 +32,9 @@ bool player_open(Player *player, const char *path); bool player_play(Player *player); bool player_pause(Player *player); bool player_seek(Player *player, double seconds); +// Direction must be -1 (previous) or +1 (next). Requires ready video media. +// Pauses playback and selects the adjacent decoded presentation frame. +bool player_step_frame(Player *player, int direction); bool player_close(Player *player); // Synchronous, thread-safe gain; survives open, seek, and close. Zero mutes. // Rejects null handles and non-finite/out-of-range values without changing gain.