feat: add native local-file playback
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
.build/
|
||||
.DS_Store
|
||||
@@ -0,0 +1,102 @@
|
||||
#include "../Sources/Core/player_macos.h"
|
||||
#import <AppKit/AppKit.h>
|
||||
#import <MetalKit/MetalKit.h>
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <functional>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
static void require(bool condition, const char *message) {
|
||||
if (!condition) { std::fprintf(stderr, "FAIL: %s\n", message); std::exit(1); }
|
||||
}
|
||||
// Runtime smoke: the actual audio device and monotonic playback clock are exercised.
|
||||
// No calendar dates, sleeps standing in for command completion, or mocked outputs.
|
||||
static PlayerState until(Player *player, const std::function<bool(PlayerState)> &ready, bool expectError = false) {
|
||||
auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10);
|
||||
while (std::chrono::steady_clock::now() < deadline) {
|
||||
auto state = player_get_state(player);
|
||||
if (state.status == PLAYER_ERROR && !expectError) {
|
||||
std::fprintf(stderr, "Playback error: %s\n", state.error);
|
||||
std::exit(1);
|
||||
}
|
||||
if (ready(state)) return state;
|
||||
CFRunLoopRunInMode(kCFRunLoopDefaultMode, .005, false);
|
||||
}
|
||||
auto state = player_get_state(player);
|
||||
std::fprintf(stderr, "Timeout: status=%d position=%.4f duration=%.4f\n",
|
||||
state.status, state.position, state.duration);
|
||||
std::exit(1);
|
||||
}
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
@autoreleasepool {
|
||||
require(argc == 2, "fixture directory argument required");
|
||||
[NSApplication sharedApplication];
|
||||
std::unique_ptr<Player, decltype(&player_destroy)> player(player_create(), player_destroy);
|
||||
require(bool(player), "player creation");
|
||||
void *native = player_macos_make_video_view(player.get());
|
||||
require(native != nullptr, "Metal view and shaders initialize");
|
||||
MTKView *view = (__bridge_transfer MTKView *)native;
|
||||
// Attach the real renderer to an offscreen native window, not a browser.
|
||||
NSWindow *window = [[NSWindow alloc] initWithContentRect:NSMakeRect(-2000, -2000, 320, 180)
|
||||
styleMask:NSWindowStyleMaskBorderless backing:NSBackingStoreBuffered defer:NO];
|
||||
window.contentView = view;
|
||||
[window orderBack:nil];
|
||||
|
||||
for (const char *fixture : {"av.mp4", "video.mp4", "audio.wav"}) {
|
||||
std::string path = std::string(argv[1]) + "/" + fixture;
|
||||
require(player_open(player.get(), path.c_str()), "open accepted");
|
||||
auto opened = until(player.get(), [](auto s) { return s.status == PLAYER_PAUSED; });
|
||||
require(opened.duration > 3.9 && opened.duration < 4.2, "discovered fixture duration");
|
||||
require(opened.has_video == (std::string(fixture) != "audio.wav"), "video track selection");
|
||||
require(opened.has_audio == (std::string(fixture) != "video.mp4"), "audio track selection");
|
||||
require(!player_seek(player.get(), NAN), "non-finite seek rejected");
|
||||
require(player_play(player.get()), "play accepted");
|
||||
until(player.get(), [](auto s) { return s.status == PLAYER_PLAYING && s.position > .35; });
|
||||
[view draw];
|
||||
require(player_pause(player.get()), "pause accepted");
|
||||
auto paused = until(player.get(), [](auto s) { return s.status == PLAYER_PAUSED; });
|
||||
// Observe the device through several run-loop turns while paused.
|
||||
for (int i = 0; i < 20; ++i)
|
||||
CFRunLoopRunInMode(kCFRunLoopDefaultMode, .005, false);
|
||||
require(std::abs(player_get_state(player.get()).position - paused.position) < .002,
|
||||
"pause freezes authoritative position");
|
||||
uint64_t revision = paused.revision;
|
||||
require(player_seek(player.get(), 1.5), "paused seek accepted");
|
||||
auto sought = until(player.get(), [revision](auto s) {
|
||||
return s.revision != revision && s.status == PLAYER_PAUSED && std::abs(s.position - 1.5) < .03;
|
||||
});
|
||||
[view draw];
|
||||
require(sought.status == PLAYER_PAUSED, "seek preserves paused state");
|
||||
require(player_play(player.get()), "resume accepted");
|
||||
until(player.get(), [](auto s) { return s.status == PLAYER_PLAYING && s.position > 1.7; });
|
||||
require(player_seek(player.get(), 3.2), "playing seek accepted");
|
||||
until(player.get(), [](auto s) { return s.status == PLAYER_PLAYING && s.position >= 3.2; });
|
||||
auto ended = until(player.get(), [](auto s) { return s.status == PLAYER_ENDED; });
|
||||
require(ended.position >= 3.8, "EOF drains media before completion");
|
||||
require(player_play(player.get()), "replay accepted");
|
||||
until(player.get(), [](auto s) { return s.status == PLAYER_PLAYING && s.position > .1 && s.position < 1; });
|
||||
require(player_close(player.get()), "close accepted while playing");
|
||||
auto closed = until(player.get(), [](auto s) { return s.status == PLAYER_EMPTY; });
|
||||
require(closed.position == 0 && !closed.has_audio && !closed.has_video, "close clears media state");
|
||||
std::printf("PASS %s: open/play/pause/seek/end/replay/close\n", fixture);
|
||||
}
|
||||
require(player_open(player.get(), (std::string(argv[1]) + "/hdr.mkv").c_str()), "HDR open accepted");
|
||||
auto hdr = until(player.get(), [](auto s) { return s.status == PLAYER_ERROR; }, true);
|
||||
require(std::string(hdr.error).find("HDR") != std::string::npos, "HDR reports unsupported output");
|
||||
require(player_open(player.get(), "/nonexistent/personal-player-smoke.mp4"), "error open accepted");
|
||||
auto failed = until(player.get(), [](auto s) { return s.status == PLAYER_ERROR; }, true);
|
||||
require(failed.error[0], "failed open reports an error");
|
||||
require(player_open(player.get(), (std::string(argv[1]) + "/av.mp4").c_str()), "recover after error");
|
||||
until(player.get(), [](auto s) { return s.status == PLAYER_PAUSED; });
|
||||
require(player_play(player.get()), "play before destruction");
|
||||
until(player.get(), [](auto s) { return s.status == PLAYER_PLAYING && s.position > .1; });
|
||||
player.reset();
|
||||
[view draw]; // Native view may outlive the player without a dangling engine pointer.
|
||||
[window orderOut:nil];
|
||||
std::puts("PASS HDR rejection/errors/recovery/destroy while playing/native view lifetime");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
CXX := xcrun clang++
|
||||
SWIFT := xcrun swiftc
|
||||
FFMPEG := libavformat libavcodec libavutil libswresample libswscale
|
||||
FF_CFLAGS := $(shell pkg-config --cflags $(FFMPEG))
|
||||
FF_LIBS := $(shell pkg-config --libs $(FFMPEG))
|
||||
ARCH := $(shell uname -m)
|
||||
CXXFLAGS := -std=c++20 -mmacosx-version-min=15.0 -O2 -g -Wall -Wextra -Wno-unused-parameter $(FF_CFLAGS)
|
||||
FRAMEWORKS := -framework AppKit -framework AudioToolbox -framework CoreAudio -framework Metal -framework MetalKit -framework QuartzCore
|
||||
OBJECTS := .build/player.o .build/Output.o
|
||||
APP := .build/Player.app
|
||||
|
||||
.PHONY: all run check
|
||||
all: $(APP)/Contents/MacOS/Player $(APP)/Contents/Info.plist
|
||||
|
||||
.build:
|
||||
mkdir -p $@
|
||||
|
||||
.build/player.o: Sources/Core/player.cpp Sources/Core/player.h Sources/Core/player_macos.h Sources/Core/output.hpp Makefile | .build
|
||||
$(CXX) $(CXXFLAGS) -c $< -o $@
|
||||
|
||||
.build/Output.o: Sources/macOS/Output.mm Sources/Core/output.hpp Makefile | .build
|
||||
$(CXX) $(CXXFLAGS) -fobjc-arc -c $< -o $@
|
||||
|
||||
$(APP)/Contents/MacOS/Player: Sources/App/PlayerApp.swift $(OBJECTS) Makefile
|
||||
mkdir -p $(APP)/Contents/MacOS
|
||||
$(SWIFT) -target $(ARCH)-apple-macosx15.0 -swift-version 6 -parse-as-library -O -g -import-objc-header Sources/Core/player_macos.h $< $(OBJECTS) $(FF_LIBS) $(FRAMEWORKS) -lc++ -o $@
|
||||
|
||||
$(APP)/Contents/Info.plist: Resources/Info.plist
|
||||
mkdir -p $(APP)/Contents
|
||||
cp $< $@
|
||||
|
||||
run: all
|
||||
open $(APP)
|
||||
|
||||
.build/playback-check: Checks/playback.mm $(OBJECTS) Makefile
|
||||
$(CXX) $(CXXFLAGS) -fobjc-arc $< $(OBJECTS) $(FF_LIBS) $(FRAMEWORKS) -o $@
|
||||
|
||||
check: .build/playback-check
|
||||
mkdir -p .build/fixtures
|
||||
ffmpeg -hide_banner -loglevel error -y -f lavfi -i testsrc2=size=320x180:rate=24 -f lavfi -i sine=frequency=440:sample_rate=44100 -t 4 -c:v mpeg4 -bf 2 -c:a aac .build/fixtures/av.mp4
|
||||
ffmpeg -hide_banner -loglevel error -y -i .build/fixtures/av.mp4 -an -c:v copy .build/fixtures/video.mp4
|
||||
ffmpeg -hide_banner -loglevel error -y -i .build/fixtures/av.mp4 -vn -c:a pcm_s16le .build/fixtures/audio.wav
|
||||
ffmpeg -hide_banner -loglevel error -y -f lavfi -i testsrc2=size=160x90:rate=12 -t 1 -vf 'format=yuv420p10le,setparams=color_primaries=bt2020:color_trc=smpte2084:colorspace=bt2020nc' -c:v ffv1 .build/fixtures/hdr.mkv
|
||||
.build/playback-check .build/fixtures
|
||||
@@ -0,0 +1,41 @@
|
||||
# Player
|
||||
|
||||
A personal native macOS player whose playback code is small enough to own and modify. FFmpeg reads and decodes media; our C++ worker owns playback. SwiftUI owns the controls, Core Audio owns audio output, and Metal presents video.
|
||||
|
||||
## Build and run
|
||||
|
||||
Requires macOS 15+, Xcode's command-line tools with Swift 6, and Homebrew FFmpeg/pkg-config:
|
||||
|
||||
```sh
|
||||
brew install ffmpeg pkg-config
|
||||
make
|
||||
make run
|
||||
```
|
||||
|
||||
The local app is `.build/Player.app`. It links the installed FFmpeg libraries; it is not a standalone distributable bundle. After upgrading FFmpeg, rebuild with `make -B`.
|
||||
|
||||
Open a local file with **⌘O**, play/pause with **Space**, and seek with the slider. Files open paused. **⇧⌘W** closes the file; closing the window quits and joins the worker.
|
||||
|
||||
## Where to change things
|
||||
|
||||
- `Sources/Core/player.h`: the C interface and authoritative state snapshot. Commands enqueue work; poll state for completion. Callers own the opaque handle and must stop using it before destruction.
|
||||
- `Sources/Core/player.cpp`: single demux/decode worker, bounded buffers, timestamps, audio-master synchronization, paused previews, decoder/resampler drain, seeking, errors, and shutdown. A seek stops output and flushes all old timeline state before resuming.
|
||||
- `Sources/Core/output.hpp` and `Sources/macOS/Output.mm`: native output contract, allocation-free/nonblocking Core Audio callback and bounded audio ring, latency-aware presentation clock, retained frame mailbox, and Metal YUV conversion. The view can outlive the player safely.
|
||||
- `Sources/App/PlayerApp.swift`: MainActor state polling, file panel, video host, and controls.
|
||||
- `Makefile`: compiler/linker commands and app bundle construction. No generated Xcode project or additional build dependency.
|
||||
|
||||
## Runnable check
|
||||
|
||||
```sh
|
||||
make check
|
||||
```
|
||||
|
||||
Generates short fixtures and exercises the real playback interface and native outputs: audio/video, video-only, audio-only, pause, paused/playing seek, completion, replay, close, HDR rejection, failed-open recovery, and destruction while playing. It initializes the actual Metal shaders and uses an offscreen native window. It requires a working audio device and graphical macOS session, and plays short sine-wave audio. It is a runtime smoke check, not a mocked clock/device test.
|
||||
|
||||
Visual correctness, perceived A/V synchronization, and control usability still need a check in the visible app.
|
||||
|
||||
## Current limits
|
||||
|
||||
Local regular files, automatically selected best audio/video tracks, software decoding, and SDR output. PQ/HLG HDR is rejected rather than rendered incorrectly. No subtitles, selectable tracks, VideoToolbox decoding, rotation/deinterlacing, or network playback yet. Mid-track audio format changes report an error.
|
||||
|
||||
Audio buffering is bounded to one second plus one decoded block (up to four seconds); video is bounded to 64 frames/128 MiB. Extremely interleaved files can drop far-future video frames to avoid starving audio; the ceiling and upgrade path are marked at `queueVideo()`. Device/stream latency is accounted for, but device changes during playback are not handled yet.
|
||||
@@ -0,0 +1,14 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>CFBundleExecutable</key><string>Player</string>
|
||||
<key>CFBundleIdentifier</key><string>local.personal.player</string>
|
||||
<key>CFBundleName</key><string>Player</string>
|
||||
<key>CFBundlePackageType</key><string>APPL</string>
|
||||
<key>CFBundleShortVersionString</key><string>0.1</string>
|
||||
<key>LSMinimumSystemVersion</key><string>15.0</string>
|
||||
<key>NSHighResolutionCapable</key><true/>
|
||||
<key>NSPrincipalClass</key><string>NSApplication</string>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -0,0 +1,253 @@
|
||||
import AppKit
|
||||
import SwiftUI
|
||||
|
||||
@MainActor
|
||||
final class PlayerModel: ObservableObject {
|
||||
static let shared = PlayerModel()
|
||||
|
||||
@Published private(set) var state = PlayerState()
|
||||
@Published private(set) var fileName: String?
|
||||
@Published private(set) var commandError: String?
|
||||
@Published private(set) var pendingRevision: UInt64?
|
||||
@Published var seekPosition = 0.0
|
||||
@Published var isSeeking = false
|
||||
|
||||
let videoView: NSView?
|
||||
private var handle: OpaquePointer?
|
||||
private var timer: Timer?
|
||||
|
||||
private init() {
|
||||
let player = player_create()
|
||||
handle = player
|
||||
if let player, let view = player_macos_make_video_view(player) {
|
||||
videoView = Unmanaged<NSView>.fromOpaque(view).takeRetainedValue()
|
||||
state = player_get_state(player)
|
||||
} else {
|
||||
videoView = nil
|
||||
commandError = "Could not create the media player."
|
||||
}
|
||||
timer = Timer.scheduledTimer(withTimeInterval: 0.05, repeats: true) { [weak self] _ in
|
||||
Task { @MainActor [weak self] in
|
||||
self?.poll()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var isBusy: Bool { pendingRevision != nil || state.status == PLAYER_OPENING }
|
||||
var canTogglePlayback: Bool {
|
||||
handle != nil && !isBusy &&
|
||||
(state.status == PLAYER_PAUSED || state.status == PLAYER_PLAYING || state.status == PLAYER_ENDED)
|
||||
}
|
||||
var canSeek: Bool { canTogglePlayback && state.duration.isFinite && state.duration > 0 }
|
||||
var canOpen: Bool { handle != nil && !isBusy }
|
||||
var canClose: Bool { handle != nil && !isBusy && state.status != PLAYER_EMPTY }
|
||||
var duration: Double { state.duration.isFinite ? max(0, state.duration) : 0 }
|
||||
var statusText: String {
|
||||
if isBusy { return "Opening…" }
|
||||
switch state.status {
|
||||
case PLAYER_EMPTY: return "Open a local audio or video file to begin."
|
||||
case PLAYER_OPENING: return "Opening…"
|
||||
case PLAYER_PAUSED: return "Paused"
|
||||
case PLAYER_PLAYING: return "Playing"
|
||||
case PLAYER_ENDED: return "Finished"
|
||||
case PLAYER_ERROR: return "Unable to play this file"
|
||||
default: return "Unavailable"
|
||||
}
|
||||
}
|
||||
var errorText: String? {
|
||||
if let commandError { return commandError }
|
||||
guard state.status == PLAYER_ERROR else { return nil }
|
||||
var error = state.error
|
||||
return withUnsafeBytes(of: &error) { bytes in
|
||||
let message = String(decoding: bytes.prefix { $0 != 0 }, as: UTF8.self)
|
||||
return message.isEmpty ? "An unknown playback error occurred." : message
|
||||
}
|
||||
}
|
||||
|
||||
func openPanel() {
|
||||
guard canOpen else { return }
|
||||
let panel = NSOpenPanel()
|
||||
panel.canChooseFiles = true
|
||||
panel.canChooseDirectories = false
|
||||
panel.allowsMultipleSelection = false
|
||||
panel.prompt = "Open"
|
||||
guard panel.runModal() == .OK, let url = panel.url else { return }
|
||||
open(url)
|
||||
}
|
||||
|
||||
private func open(_ url: URL) {
|
||||
guard let handle else { return }
|
||||
let revision = player_get_state(handle).revision
|
||||
let accepted = url.withUnsafeFileSystemRepresentation { path in
|
||||
guard let path else { return false }
|
||||
return player_open(handle, path)
|
||||
}
|
||||
guard accepted else {
|
||||
commandError = "Could not queue the file to open. Please try again."
|
||||
return
|
||||
}
|
||||
// Open is always paused; never queue play against the previous file.
|
||||
pendingRevision = revision
|
||||
fileName = url.lastPathComponent
|
||||
commandError = nil
|
||||
isSeeking = false
|
||||
seekPosition = 0
|
||||
}
|
||||
|
||||
func togglePlayback() {
|
||||
guard let handle, canTogglePlayback else { return }
|
||||
let accepted = state.status == PLAYER_PLAYING ? player_pause(handle) : player_play(handle)
|
||||
commandError = accepted ? nil : "Could not queue playback. Please try again."
|
||||
}
|
||||
|
||||
func finishSeeking() {
|
||||
isSeeking = false
|
||||
guard let handle, canSeek else { return }
|
||||
let position = min(max(0, seekPosition), duration)
|
||||
commandError = player_seek(handle, position) ? nil : "Could not queue the seek. Please try again."
|
||||
}
|
||||
|
||||
func closeFile() {
|
||||
guard let handle, canClose else { return }
|
||||
guard player_close(handle) else {
|
||||
commandError = "Could not queue closing the file. Please try again."
|
||||
return
|
||||
}
|
||||
fileName = nil
|
||||
commandError = nil
|
||||
isSeeking = false
|
||||
}
|
||||
|
||||
private func poll() {
|
||||
guard let handle else { return }
|
||||
let next = player_get_state(handle)
|
||||
if let revision = pendingRevision {
|
||||
guard next.revision != revision, next.status != PLAYER_OPENING else { return }
|
||||
pendingRevision = nil
|
||||
}
|
||||
state = next
|
||||
if !isSeeking {
|
||||
seekPosition = min(max(0, next.position.isFinite ? next.position : 0), duration)
|
||||
}
|
||||
}
|
||||
|
||||
func shutdown() {
|
||||
timer?.invalidate()
|
||||
timer = nil
|
||||
guard let player = handle else { return }
|
||||
handle = nil
|
||||
player_destroy(player)
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private struct VideoSurface: NSViewRepresentable {
|
||||
let view: NSView
|
||||
|
||||
func makeNSView(context: Context) -> NSView { view }
|
||||
func updateNSView(_ nsView: NSView, context: Context) {}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private struct PlayerContent: View {
|
||||
@ObservedObject var model: PlayerModel
|
||||
|
||||
private func time(_ seconds: Double) -> String {
|
||||
let value = Int(min(max(0, seconds.isFinite ? seconds : 0), Double(Int.max / 2)))
|
||||
let hours = value / 3600
|
||||
return hours > 0
|
||||
? String(format: "%d:%02d:%02d", hours, value / 60 % 60, value % 60)
|
||||
: String(format: "%d:%02d", value / 60, value % 60)
|
||||
}
|
||||
|
||||
var body: some View {
|
||||
VStack(spacing: 0) {
|
||||
ZStack {
|
||||
Color.black
|
||||
if let view = model.videoView {
|
||||
VideoSurface(view: view)
|
||||
}
|
||||
if model.fileName == nil {
|
||||
VStack(spacing: 12) {
|
||||
Image(systemName: "play.rectangle").font(.system(size: 48))
|
||||
Text("Open a local audio or video file")
|
||||
Button("Open File…", action: model.openPanel)
|
||||
.disabled(!model.canOpen)
|
||||
}
|
||||
.foregroundStyle(.white)
|
||||
} else if !model.isBusy && !model.state.has_video && model.state.has_audio {
|
||||
Image(systemName: "waveform").font(.system(size: 64)).foregroundStyle(.white)
|
||||
}
|
||||
}
|
||||
.frame(maxWidth: .infinity, maxHeight: .infinity)
|
||||
.accessibilityLabel("Video")
|
||||
|
||||
VStack(alignment: .leading, spacing: 10) {
|
||||
HStack {
|
||||
Button(action: model.togglePlayback) {
|
||||
Image(systemName: model.state.status == PLAYER_PLAYING ? "pause.fill" : "play.fill")
|
||||
.frame(width: 22)
|
||||
}
|
||||
.keyboardShortcut(.space, modifiers: [])
|
||||
.accessibilityLabel(model.state.status == PLAYER_PLAYING ? "Pause" : "Play")
|
||||
.disabled(!model.canTogglePlayback)
|
||||
Text(time(model.seekPosition)).monospacedDigit().frame(minWidth: 52, alignment: .trailing)
|
||||
Slider(value: $model.seekPosition, in: 0...max(model.duration, 0.001), onEditingChanged: { editing in
|
||||
if editing { model.isSeeking = true } else { model.finishSeeking() }
|
||||
})
|
||||
.disabled(!model.canSeek)
|
||||
.accessibilityLabel("Playback position")
|
||||
Text(time(model.duration)).monospacedDigit().frame(minWidth: 52, alignment: .leading)
|
||||
Button("Open…", action: model.openPanel)
|
||||
.accessibilityLabel("Open media file")
|
||||
.disabled(!model.canOpen)
|
||||
}
|
||||
Text(model.fileName ?? "Player").font(.headline).lineLimit(1).help(model.fileName ?? "Player")
|
||||
Text(model.statusText).font(.caption).foregroundStyle(.secondary)
|
||||
if let error = model.errorText {
|
||||
Text(error).font(.callout).foregroundStyle(.red).textSelection(.enabled)
|
||||
}
|
||||
}
|
||||
.padding()
|
||||
}
|
||||
.frame(minWidth: 560, minHeight: 360)
|
||||
.navigationTitle(model.fileName ?? "Player")
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
final class AppDelegate: NSObject, NSApplicationDelegate {
|
||||
func applicationDidFinishLaunching(_ notification: Notification) {
|
||||
NSApp.setActivationPolicy(.regular)
|
||||
NSApp.activate(ignoringOtherApps: true)
|
||||
}
|
||||
|
||||
func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { true }
|
||||
|
||||
func applicationWillTerminate(_ notification: Notification) {
|
||||
PlayerModel.shared.shutdown()
|
||||
}
|
||||
}
|
||||
|
||||
@main
|
||||
@MainActor
|
||||
struct PlayerApp: App {
|
||||
@NSApplicationDelegateAdaptor(AppDelegate.self) private var delegate
|
||||
@StateObject private var model = PlayerModel.shared
|
||||
|
||||
var body: some Scene {
|
||||
Window("Player", id: "player") {
|
||||
PlayerContent(model: model)
|
||||
}
|
||||
.defaultSize(width: 900, height: 600)
|
||||
.commands {
|
||||
CommandGroup(replacing: .newItem) {
|
||||
Button("Open File…", action: model.openPanel).keyboardShortcut("o")
|
||||
.disabled(!model.canOpen)
|
||||
Button("Close File", action: model.closeFile)
|
||||
.keyboardShortcut("w", modifiers: [.command, .shift])
|
||||
.disabled(!model.canClose)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
#pragma once
|
||||
#include <cstddef>
|
||||
#include <memory>
|
||||
#include <string>
|
||||
extern "C" {
|
||||
#include <libavutil/frame.h>
|
||||
}
|
||||
|
||||
namespace playback {
|
||||
// One decoding worker produces output. The native audio callback is the consumer.
|
||||
// Audio is interleaved stereo float at 48 kHz; write() never blocks.
|
||||
class Output {
|
||||
public:
|
||||
Output();
|
||||
~Output();
|
||||
Output(const Output &) = delete;
|
||||
Output &operator=(const Output &) = delete;
|
||||
bool configure_audio(std::string &error);
|
||||
bool start(std::string &error);
|
||||
void pause();
|
||||
// Called with audio stopped. Clears audio and video from the old timeline.
|
||||
void reset();
|
||||
std::size_t write(const float *samples, std::size_t frames, double pts);
|
||||
std::size_t queued() const;
|
||||
// Latency-aware presented position; NaN until an audio callback has run.
|
||||
double clock() const;
|
||||
// Retains the frame's buffers; decoder may reuse its AVFrame after return.
|
||||
void present(const AVFrame *frame);
|
||||
// Returns a retained NSView. The view remains safe after Output is destroyed.
|
||||
void *make_video_view();
|
||||
private:
|
||||
struct Impl;
|
||||
std::unique_ptr<Impl> impl;
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,781 @@
|
||||
#include "player.h"
|
||||
#include "player_macos.h"
|
||||
#include "output.hpp"
|
||||
|
||||
extern "C" {
|
||||
#include <libavcodec/avcodec.h>
|
||||
#include <libavformat/avformat.h>
|
||||
#include <libavutil/channel_layout.h>
|
||||
#include <libavutil/error.h>
|
||||
#include <libavutil/imgutils.h>
|
||||
#include <libswresample/swresample.h>
|
||||
}
|
||||
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
#include <condition_variable>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <deque>
|
||||
#include <filesystem>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
constexpr int sampleRate = 48000;
|
||||
constexpr std::size_t commandLimit = 32;
|
||||
constexpr std::size_t videoFrameLimit = 64;
|
||||
constexpr std::size_t videoByteLimit = 128 * 1024 * 1024;
|
||||
constexpr int audioFrameLimit = sampleRate * 4;
|
||||
constexpr double sampleTime = 1.0 / sampleRate;
|
||||
using Clock = std::chrono::steady_clock;
|
||||
|
||||
struct Cancelled {};
|
||||
struct FrameDeleter {
|
||||
void operator()(AVFrame *frame) const { av_frame_free(&frame); }
|
||||
};
|
||||
using Frame = std::unique_ptr<AVFrame, FrameDeleter>;
|
||||
struct VideoFrame {
|
||||
Frame frame;
|
||||
double pts;
|
||||
std::size_t bytes;
|
||||
};
|
||||
struct Track {
|
||||
AVCodecContext *codec = nullptr;
|
||||
int index = -1;
|
||||
bool drainSent = false;
|
||||
bool drained = false;
|
||||
double nextPts = 0;
|
||||
};
|
||||
enum class Action { Open, Play, Pause, Seek, Close };
|
||||
struct Command {
|
||||
Action action;
|
||||
std::string path;
|
||||
double seconds = 0;
|
||||
uint64_t epoch = 0;
|
||||
};
|
||||
|
||||
void copyError(char *destination, const char *message) noexcept {
|
||||
std::snprintf(destination, 512, "%s", message ? message : "Unknown playback error");
|
||||
}
|
||||
}
|
||||
|
||||
struct Player {
|
||||
playback::Output output;
|
||||
std::mutex mutex;
|
||||
std::condition_variable wake;
|
||||
std::deque<Command> commands;
|
||||
PlayerState state{};
|
||||
std::array<char, 512> requestedError{};
|
||||
bool errorRequested = false;
|
||||
uint64_t errorEpoch = 0;
|
||||
std::atomic<bool> stopping{false};
|
||||
std::atomic<uint64_t> requestedEpoch{0};
|
||||
uint64_t activeEpoch = 0;
|
||||
|
||||
// Everything below, including FFmpeg and Output audio operations, is worker-owned.
|
||||
AVFormatContext *format = nullptr;
|
||||
std::array<Track, 2> tracks;
|
||||
AVPacket *packet = nullptr;
|
||||
AVFrame *decoded = nullptr;
|
||||
SwrContext *resampler = nullptr;
|
||||
AVChannelLayout inputLayout{};
|
||||
int inputRate = 0;
|
||||
AVSampleFormat inputFormat = AV_SAMPLE_FMT_NONE;
|
||||
bool resamplerDrained = false;
|
||||
bool packetPending = false;
|
||||
bool demuxEnded = false;
|
||||
int receiving = -1;
|
||||
std::deque<VideoFrame> video;
|
||||
std::size_t videoBytes = 0;
|
||||
std::vector<float> pcm;
|
||||
std::size_t pcmFrames = 0;
|
||||
std::size_t pcmOffset = 0;
|
||||
double pcmPts = 0;
|
||||
double resampleNextPts = 0;
|
||||
double origin = 0;
|
||||
double duration = 0;
|
||||
double position = 0;
|
||||
double seekTarget = 0;
|
||||
double audioWriteEnd = 0;
|
||||
double videoEnd = 0;
|
||||
double videoFrameTime = 1.0 / 25;
|
||||
double steadyPosition = 0;
|
||||
Clock::time_point steadyAnchor = Clock::now();
|
||||
PlayerStatus status = PLAYER_EMPTY;
|
||||
bool previewShown = false;
|
||||
std::array<char, 512> error{};
|
||||
std::thread worker;
|
||||
|
||||
Player() : worker([this] { run(); }) {}
|
||||
~Player() {
|
||||
stopping.store(true, std::memory_order_relaxed);
|
||||
requestedEpoch.fetch_add(1, std::memory_order_relaxed);
|
||||
wake.notify_one();
|
||||
if (worker.joinable()) worker.join();
|
||||
}
|
||||
|
||||
bool enqueue(Command command) {
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
if (stopping.load(std::memory_order_relaxed) || commands.size() >= commandLimit)
|
||||
return false;
|
||||
const bool newTimeline = command.action == Action::Open || command.action == Action::Close;
|
||||
command.epoch = requestedEpoch.load(std::memory_order_relaxed) + (newTimeline ? 1 : 0);
|
||||
commands.push_back(std::move(command));
|
||||
if (newTimeline) requestedEpoch.store(commands.back().epoch, std::memory_order_relaxed);
|
||||
if (commands.back().action == Action::Open) {
|
||||
const auto revision = state.revision + 1;
|
||||
state = {};
|
||||
state.status = PLAYER_OPENING;
|
||||
state.revision = revision;
|
||||
}
|
||||
wake.notify_one();
|
||||
return true;
|
||||
}
|
||||
|
||||
void apiError(const char *message) noexcept {
|
||||
try {
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
copyError(requestedError.data(), message);
|
||||
errorRequested = true;
|
||||
errorEpoch = requestedEpoch.fetch_add(1, std::memory_order_relaxed) + 1;
|
||||
state.status = PLAYER_ERROR;
|
||||
copyError(state.error, message);
|
||||
++state.revision;
|
||||
wake.notify_one();
|
||||
} catch (...) {
|
||||
// No exception may cross the C boundary, including a failed mutex operation.
|
||||
}
|
||||
}
|
||||
|
||||
bool interrupted() const {
|
||||
return stopping.load(std::memory_order_relaxed) ||
|
||||
requestedEpoch.load(std::memory_order_relaxed) != activeEpoch;
|
||||
}
|
||||
static int interrupt(void *opaque) {
|
||||
return static_cast<Player *>(opaque)->interrupted();
|
||||
}
|
||||
void check(int result, const char *operation) {
|
||||
if (result >= 0) return;
|
||||
if (interrupted()) throw Cancelled{};
|
||||
char details[AV_ERROR_MAX_STRING_SIZE];
|
||||
av_strerror(result, details, sizeof(details));
|
||||
throw std::runtime_error(std::string(operation) + ": " + details);
|
||||
}
|
||||
|
||||
void publish() {
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
if (activeEpoch != requestedEpoch.load(std::memory_order_relaxed)) return;
|
||||
state.status = status;
|
||||
state.position = position;
|
||||
state.duration = duration;
|
||||
state.has_audio = tracks[0].codec != nullptr;
|
||||
state.has_video = tracks[1].codec != nullptr;
|
||||
std::memcpy(state.error, error.data(), error.size());
|
||||
++state.revision;
|
||||
}
|
||||
|
||||
void release() {
|
||||
output.pause();
|
||||
output.reset();
|
||||
video.clear();
|
||||
videoBytes = 0;
|
||||
swr_free(&resampler);
|
||||
av_channel_layout_uninit(&inputLayout);
|
||||
for (auto &track : tracks) {
|
||||
avcodec_free_context(&track.codec);
|
||||
track = {};
|
||||
}
|
||||
av_packet_free(&packet);
|
||||
av_frame_free(&decoded);
|
||||
avformat_close_input(&format);
|
||||
inputRate = 0;
|
||||
inputFormat = AV_SAMPLE_FMT_NONE;
|
||||
pcmFrames = pcmOffset = 0;
|
||||
packetPending = demuxEnded = resamplerDrained = false;
|
||||
receiving = -1;
|
||||
origin = duration = position = seekTarget = audioWriteEnd = videoEnd = resampleNextPts = 0;
|
||||
previewShown = false;
|
||||
error.fill(0);
|
||||
status = PLAYER_EMPTY;
|
||||
}
|
||||
|
||||
void fail(const char *message) {
|
||||
// Preserve messages that point into one of this object's own state buffers.
|
||||
std::array<char, 512> saved{};
|
||||
copyError(saved.data(), message);
|
||||
release();
|
||||
error = saved;
|
||||
status = PLAYER_ERROR;
|
||||
publish();
|
||||
}
|
||||
|
||||
void openTrack(int slot, AVMediaType type, int related) {
|
||||
const AVCodec *decoder = nullptr;
|
||||
int index = av_find_best_stream(format, type, -1, related, &decoder, 0);
|
||||
if (index == AVERROR_STREAM_NOT_FOUND) return;
|
||||
check(index, type == AVMEDIA_TYPE_AUDIO ? "Finding audio decoder" : "Finding video decoder");
|
||||
auto &track = tracks[slot];
|
||||
track.index = index;
|
||||
track.codec = avcodec_alloc_context3(decoder);
|
||||
if (!track.codec) throw std::bad_alloc();
|
||||
check(avcodec_parameters_to_context(track.codec, format->streams[index]->codecpar),
|
||||
"Reading codec parameters");
|
||||
track.codec->pkt_timebase = format->streams[index]->time_base;
|
||||
// A bounded software decoder pool; FFmpeg owns its internal frame reorder buffer.
|
||||
track.codec->thread_count = 2;
|
||||
check(avcodec_open2(track.codec, decoder, nullptr), "Opening decoder");
|
||||
}
|
||||
|
||||
void resetTimeline(double target) {
|
||||
output.pause();
|
||||
output.reset();
|
||||
for (auto &track : tracks) {
|
||||
if (!track.codec) continue;
|
||||
avcodec_flush_buffers(track.codec);
|
||||
track.drainSent = track.drained = false;
|
||||
const AVStream *stream = format->streams[track.index];
|
||||
track.nextPts = stream->start_time == AV_NOPTS_VALUE ? target :
|
||||
std::max(target, stream->start_time * av_q2d(stream->time_base) - origin);
|
||||
}
|
||||
swr_free(&resampler);
|
||||
av_channel_layout_uninit(&inputLayout);
|
||||
inputRate = 0;
|
||||
inputFormat = AV_SAMPLE_FMT_NONE;
|
||||
av_packet_unref(packet);
|
||||
av_frame_unref(decoded);
|
||||
video.clear();
|
||||
videoBytes = 0;
|
||||
pcmFrames = pcmOffset = 0;
|
||||
packetPending = demuxEnded = resamplerDrained = false;
|
||||
receiving = -1;
|
||||
position = seekTarget = audioWriteEnd = videoEnd = resampleNextPts = target;
|
||||
previewShown = false;
|
||||
steadyPosition = target;
|
||||
steadyAnchor = Clock::now();
|
||||
}
|
||||
|
||||
void open(const std::string &path) {
|
||||
release();
|
||||
status = PLAYER_OPENING;
|
||||
publish();
|
||||
std::error_code filesystemError;
|
||||
if (!std::filesystem::is_regular_file(path, filesystemError))
|
||||
throw std::runtime_error(filesystemError ? filesystemError.message() :
|
||||
"Open requires a local regular media file");
|
||||
const std::string localPath = std::filesystem::absolute(path).string();
|
||||
format = avformat_alloc_context();
|
||||
if (!format) throw std::bad_alloc();
|
||||
format->interrupt_callback = {interrupt, this};
|
||||
AVDictionary *options = nullptr;
|
||||
const int optionResult = av_dict_set(&options, "protocol_whitelist", "file", 0);
|
||||
if (optionResult < 0) {
|
||||
av_dict_free(&options);
|
||||
check(optionResult, "Restricting input to local files");
|
||||
}
|
||||
const int result = avformat_open_input(&format, localPath.c_str(), nullptr, &options);
|
||||
av_dict_free(&options);
|
||||
check(result, "Opening media");
|
||||
check(avformat_find_stream_info(format, nullptr), "Reading media information");
|
||||
openTrack(1, AVMEDIA_TYPE_VIDEO, -1);
|
||||
openTrack(0, AVMEDIA_TYPE_AUDIO, tracks[1].index);
|
||||
if (!tracks[0].codec && !tracks[1].codec)
|
||||
throw std::runtime_error("The file contains no playable audio or video track");
|
||||
for (unsigned i = 0; i < format->nb_streams; ++i) {
|
||||
if (static_cast<int>(i) != tracks[0].index && static_cast<int>(i) != tracks[1].index)
|
||||
format->streams[i]->discard = AVDISCARD_ALL;
|
||||
}
|
||||
if (format->start_time != AV_NOPTS_VALUE) {
|
||||
origin = static_cast<double>(format->start_time) / AV_TIME_BASE;
|
||||
} else {
|
||||
double first = std::numeric_limits<double>::infinity();
|
||||
for (const auto &track : tracks) {
|
||||
if (!track.codec) continue;
|
||||
const AVStream *stream = format->streams[track.index];
|
||||
if (stream->start_time != AV_NOPTS_VALUE)
|
||||
first = std::min(first, stream->start_time * av_q2d(stream->time_base));
|
||||
}
|
||||
origin = std::isfinite(first) ? first : 0;
|
||||
}
|
||||
if (format->duration != AV_NOPTS_VALUE && format->duration > 0)
|
||||
duration = static_cast<double>(format->duration) / AV_TIME_BASE;
|
||||
if (duration == 0) {
|
||||
for (const auto &track : tracks) {
|
||||
if (!track.codec) continue;
|
||||
const AVStream *stream = format->streams[track.index];
|
||||
if (stream->duration == AV_NOPTS_VALUE || stream->duration <= 0) continue;
|
||||
const double start = stream->start_time == AV_NOPTS_VALUE ? origin :
|
||||
stream->start_time * av_q2d(stream->time_base);
|
||||
duration = std::max(duration, start - origin + stream->duration * av_q2d(stream->time_base));
|
||||
}
|
||||
}
|
||||
if (tracks[1].codec) {
|
||||
const AVRational rate = av_guess_frame_rate(format, format->streams[tracks[1].index], nullptr);
|
||||
videoFrameTime = rate.num > 0 && rate.den > 0 ? av_q2d(av_inv_q(rate)) : 1.0 / 25;
|
||||
}
|
||||
if (tracks[0].codec) {
|
||||
std::string deviceError;
|
||||
if (!output.configure_audio(deviceError))
|
||||
throw std::runtime_error(deviceError.empty() ? "Cannot configure audio output" : deviceError);
|
||||
}
|
||||
packet = av_packet_alloc();
|
||||
decoded = av_frame_alloc();
|
||||
if (!packet || !decoded) throw std::bad_alloc();
|
||||
resetTimeline(0);
|
||||
status = PLAYER_PAUSED;
|
||||
publish();
|
||||
}
|
||||
|
||||
double clockPosition() const {
|
||||
if (status != PLAYER_PLAYING) return position;
|
||||
if (tracks[0].codec) {
|
||||
const double audioClock = output.clock();
|
||||
return std::isfinite(audioClock) ? std::max(seekTarget, audioClock) : position;
|
||||
}
|
||||
return steadyPosition + std::chrono::duration<double>(Clock::now() - steadyAnchor).count();
|
||||
}
|
||||
|
||||
void start() {
|
||||
if (!format || status == PLAYER_ERROR || status == PLAYER_EMPTY) return;
|
||||
if (status == PLAYER_ENDED) seek(0, false);
|
||||
if (status == PLAYER_PLAYING) return;
|
||||
if (tracks[0].codec) {
|
||||
std::string deviceError;
|
||||
if (!output.start(deviceError))
|
||||
throw std::runtime_error(deviceError.empty() ? "Cannot start audio output" : deviceError);
|
||||
}
|
||||
steadyPosition = position;
|
||||
steadyAnchor = Clock::now();
|
||||
status = PLAYER_PLAYING;
|
||||
}
|
||||
|
||||
void pause() {
|
||||
if (status != PLAYER_PLAYING) return;
|
||||
position = clockPosition();
|
||||
output.pause();
|
||||
if (tracks[0].codec) {
|
||||
const double frozen = output.clock();
|
||||
if (std::isfinite(frozen)) position = std::max(seekTarget, frozen);
|
||||
}
|
||||
status = PLAYER_PAUSED;
|
||||
}
|
||||
|
||||
void seek(double seconds, bool resume) {
|
||||
if (!format) return;
|
||||
const double target = duration > 0 ? std::clamp(seconds, 0.0, duration) : std::max(0.0, seconds);
|
||||
const long double absolute = std::round((static_cast<long double>(target) + origin) * AV_TIME_BASE);
|
||||
if (absolute < -0x1p63L || absolute >= 0x1p63L)
|
||||
throw std::runtime_error("Seek target is outside the supported timestamp range");
|
||||
output.pause();
|
||||
const int64_t timestamp = static_cast<int64_t>(absolute);
|
||||
check(avformat_seek_file(format, -1, std::numeric_limits<int64_t>::min(),
|
||||
timestamp, timestamp, 0), "Seeking media");
|
||||
resetTimeline(target);
|
||||
status = PLAYER_PAUSED;
|
||||
if (resume) start();
|
||||
}
|
||||
|
||||
void execute(const Command &command) {
|
||||
if (command.epoch != requestedEpoch.load(std::memory_order_relaxed)) return;
|
||||
activeEpoch = command.epoch;
|
||||
switch (command.action) {
|
||||
case Action::Open: open(command.path); break;
|
||||
case Action::Play: start(); break;
|
||||
case Action::Pause: pause(); break;
|
||||
case Action::Seek: seek(command.seconds, status == PLAYER_PLAYING); break;
|
||||
case Action::Close: release(); break;
|
||||
}
|
||||
publish();
|
||||
}
|
||||
|
||||
double framePts(Track &track, const AVFrame *frame) {
|
||||
const int64_t timestamp = frame->best_effort_timestamp != AV_NOPTS_VALUE ?
|
||||
frame->best_effort_timestamp : frame->pts;
|
||||
if (timestamp == AV_NOPTS_VALUE) return track.nextPts;
|
||||
return timestamp * av_q2d(format->streams[track.index]->time_base) - origin;
|
||||
}
|
||||
|
||||
void convertAudio() {
|
||||
auto &track = tracks[0];
|
||||
const int rate = decoded->sample_rate > 0 ? decoded->sample_rate : track.codec->sample_rate;
|
||||
const AVChannelLayout *layout = decoded->ch_layout.nb_channels > 0 ?
|
||||
&decoded->ch_layout : &track.codec->ch_layout;
|
||||
const auto sampleFormat = static_cast<AVSampleFormat>(decoded->format);
|
||||
if (rate <= 0 || layout->nb_channels <= 0 || decoded->nb_samples < 0)
|
||||
throw std::runtime_error("Invalid decoded audio format");
|
||||
if (!resampler) {
|
||||
AVChannelLayout stereo = AV_CHANNEL_LAYOUT_STEREO;
|
||||
check(swr_alloc_set_opts2(&resampler, &stereo, AV_SAMPLE_FMT_FLT, sampleRate,
|
||||
layout, sampleFormat, rate, 0, nullptr), "Allocating audio resampler");
|
||||
check(swr_init(resampler), "Initializing audio resampler");
|
||||
check(av_channel_layout_copy(&inputLayout, layout), "Saving audio channel layout");
|
||||
inputRate = rate;
|
||||
inputFormat = sampleFormat;
|
||||
} else if (inputRate != rate || inputFormat != sampleFormat ||
|
||||
av_channel_layout_compare(&inputLayout, layout) != 0) {
|
||||
throw std::runtime_error("Audio format changes within a track are not supported");
|
||||
}
|
||||
const double timestamp = framePts(track, decoded);
|
||||
track.nextPts = timestamp + static_cast<double>(decoded->nb_samples) / rate;
|
||||
pcmPts = timestamp - static_cast<double>(swr_get_delay(resampler, rate)) / rate;
|
||||
const int capacity = swr_get_out_samples(resampler, decoded->nb_samples);
|
||||
check(capacity, "Sizing resampled audio");
|
||||
if (capacity > audioFrameLimit)
|
||||
throw std::runtime_error("Decoded audio block exceeds the four-second buffer limit");
|
||||
const auto samples = static_cast<std::size_t>(capacity) * 2;
|
||||
if (pcm.size() < samples) pcm.resize(samples);
|
||||
uint8_t *destination = reinterpret_cast<uint8_t *>(pcm.data());
|
||||
const int count = swr_convert(resampler, &destination, capacity,
|
||||
const_cast<const uint8_t **>(decoded->extended_data), decoded->nb_samples);
|
||||
check(count, "Resampling audio");
|
||||
resampleNextPts = pcmPts + static_cast<double>(count) / sampleRate;
|
||||
pcmFrames = static_cast<std::size_t>(count);
|
||||
pcmOffset = 0;
|
||||
trimAudio();
|
||||
}
|
||||
|
||||
void trimAudio() {
|
||||
if (pcmPts < seekTarget) {
|
||||
const double skip = std::ceil((seekTarget - pcmPts) * sampleRate - 1e-6);
|
||||
pcmOffset = static_cast<std::size_t>(std::min(skip, static_cast<double>(pcmFrames)));
|
||||
}
|
||||
}
|
||||
|
||||
void drainResampler() {
|
||||
if (!resampler) {
|
||||
resamplerDrained = true;
|
||||
return;
|
||||
}
|
||||
const int capacity = swr_get_out_samples(resampler, 0);
|
||||
check(capacity, "Sizing final audio");
|
||||
if (capacity > audioFrameLimit)
|
||||
throw std::runtime_error("Resampler drain exceeds the audio buffer limit");
|
||||
// A nonzero destination also lets a resampler with no delayed samples signal completion.
|
||||
const auto samples = static_cast<std::size_t>(std::max(capacity, 32)) * 2;
|
||||
if (pcm.size() < samples) pcm.resize(samples);
|
||||
uint8_t *destination = reinterpret_cast<uint8_t *>(pcm.data());
|
||||
const int count = swr_convert(resampler, &destination, std::max(capacity, 32), nullptr, 0);
|
||||
check(count, "Draining audio resampler");
|
||||
pcmPts = resampleNextPts;
|
||||
resampleNextPts += static_cast<double>(count) / sampleRate;
|
||||
pcmFrames = static_cast<std::size_t>(count);
|
||||
pcmOffset = 0;
|
||||
resamplerDrained = count == 0;
|
||||
trimAudio();
|
||||
}
|
||||
|
||||
bool writeSilence(double end) {
|
||||
if (end <= audioWriteEnd + sampleTime * 0.5) return false;
|
||||
static constexpr std::array<float, 4096> silence{};
|
||||
const double remaining = std::round((end - audioWriteEnd) * sampleRate);
|
||||
const std::size_t frames = static_cast<std::size_t>(std::min(remaining, 2048.0));
|
||||
if (frames == 0) return false;
|
||||
const std::size_t accepted = output.write(silence.data(), frames, audioWriteEnd);
|
||||
audioWriteEnd += static_cast<double>(accepted) / sampleRate;
|
||||
return accepted != 0;
|
||||
}
|
||||
|
||||
bool pumpAudio() {
|
||||
if (pcmOffset >= pcmFrames) return false;
|
||||
const double start = pcmPts + static_cast<double>(pcmOffset) / sampleRate;
|
||||
if (start > audioWriteEnd + sampleTime * 0.5) return writeSilence(start);
|
||||
if (start < audioWriteEnd - sampleTime * 0.5) {
|
||||
const double overlap = std::round((audioWriteEnd - start) * sampleRate);
|
||||
const auto skip = static_cast<std::size_t>(std::min(overlap, static_cast<double>(pcmFrames - pcmOffset)));
|
||||
pcmOffset += skip;
|
||||
if (pcmOffset >= pcmFrames) return true;
|
||||
}
|
||||
const std::size_t accepted = output.write(pcm.data() + pcmOffset * 2,
|
||||
pcmFrames - pcmOffset, audioWriteEnd);
|
||||
pcmOffset += accepted;
|
||||
audioWriteEnd += static_cast<double>(accepted) / sampleRate;
|
||||
return accepted != 0;
|
||||
}
|
||||
|
||||
bool videoFull() const {
|
||||
return video.size() >= videoFrameLimit ||
|
||||
(!video.empty() && videoBytes > videoByteLimit - video.back().bytes);
|
||||
}
|
||||
|
||||
void queueVideo() {
|
||||
auto &track = tracks[1];
|
||||
const auto transfer = decoded->color_trc == AVCOL_TRC_UNSPECIFIED ?
|
||||
track.codec->color_trc : decoded->color_trc;
|
||||
if (transfer == AVCOL_TRC_SMPTE2084 || transfer == AVCOL_TRC_ARIB_STD_B67)
|
||||
throw std::runtime_error("HDR PQ/HLG video is unsupported: the native view currently supports SDR only");
|
||||
const double pts = framePts(track, decoded);
|
||||
const double frameDuration = decoded->duration > 0 ?
|
||||
decoded->duration * av_q2d(format->streams[track.index]->time_base) : videoFrameTime;
|
||||
track.nextPts = pts + frameDuration;
|
||||
if (pts + sampleTime * 0.5 < seekTarget || (decoded->flags & AV_FRAME_FLAG_DISCARD)) return;
|
||||
const int size = av_image_get_buffer_size(static_cast<AVPixelFormat>(decoded->format),
|
||||
decoded->width, decoded->height, 1);
|
||||
check(size, "Sizing decoded video");
|
||||
const auto bytes = static_cast<std::size_t>(size);
|
||||
if (bytes > videoByteLimit)
|
||||
throw std::runtime_error("A decoded video frame exceeds the 128 MiB buffer limit");
|
||||
videoEnd = std::max(videoEnd, pts + frameDuration);
|
||||
// ponytail: extreme interleaving can starve audio behind a full video queue.
|
||||
// Keep nearest frames plus the newest; packet spooling is the upgrade if that matters.
|
||||
while (!video.empty() && (video.size() >= videoFrameLimit || videoBytes + bytes > videoByteLimit)) {
|
||||
videoBytes -= video.back().bytes;
|
||||
video.pop_back();
|
||||
}
|
||||
Frame frame(av_frame_clone(decoded));
|
||||
if (!frame) throw std::bad_alloc();
|
||||
VideoFrame item{std::move(frame), pts, bytes};
|
||||
auto insertion = std::upper_bound(video.begin(), video.end(), pts,
|
||||
[](double value, const VideoFrame &entry) { return value < entry.pts; });
|
||||
video.insert(insertion, std::move(item));
|
||||
videoBytes += bytes;
|
||||
}
|
||||
|
||||
bool decodeOne() {
|
||||
if (pcmOffset < pcmFrames) return false;
|
||||
if (receiving >= 0) {
|
||||
auto &track = tracks[receiving];
|
||||
if (receiving == 1 && videoFull() &&
|
||||
(!tracks[0].codec || output.queued() >= 1024 || status != PLAYER_PLAYING))
|
||||
return false;
|
||||
const int result = avcodec_receive_frame(track.codec, decoded);
|
||||
if (result == AVERROR(EAGAIN)) {
|
||||
receiving = -1;
|
||||
return true;
|
||||
}
|
||||
if (result == AVERROR_EOF) {
|
||||
track.drained = true;
|
||||
receiving = -1;
|
||||
return true;
|
||||
}
|
||||
check(result, "Decoding media");
|
||||
if (receiving == 0) convertAudio(); else queueVideo();
|
||||
av_frame_unref(decoded);
|
||||
return true;
|
||||
}
|
||||
if (packetPending) {
|
||||
const int slot = packet->stream_index == tracks[0].index ? 0 : 1;
|
||||
const int result = avcodec_send_packet(tracks[slot].codec, packet);
|
||||
if (result == AVERROR(EAGAIN)) {
|
||||
receiving = slot;
|
||||
return true;
|
||||
}
|
||||
check(result, "Sending media packet to decoder");
|
||||
av_packet_unref(packet);
|
||||
packetPending = false;
|
||||
receiving = slot;
|
||||
return true;
|
||||
}
|
||||
if (demuxEnded) {
|
||||
for (int slot = 0; slot < 2; ++slot) {
|
||||
auto &track = tracks[slot];
|
||||
if (!track.codec || track.drainSent) continue;
|
||||
const int result = avcodec_send_packet(track.codec, nullptr);
|
||||
if (result == AVERROR(EAGAIN)) {
|
||||
receiving = slot;
|
||||
return true;
|
||||
}
|
||||
if (result != AVERROR_EOF) check(result, "Draining decoder");
|
||||
track.drainSent = true;
|
||||
receiving = slot;
|
||||
return true;
|
||||
}
|
||||
if ((!tracks[0].codec || tracks[0].drained) && !resamplerDrained) {
|
||||
drainResampler();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
const int result = av_read_frame(format, packet);
|
||||
if (result == AVERROR_EOF) {
|
||||
demuxEnded = true;
|
||||
return true;
|
||||
}
|
||||
if (result == AVERROR(EAGAIN)) return false;
|
||||
check(result, "Reading media packet");
|
||||
if (packet->stream_index != tracks[0].index && packet->stream_index != tracks[1].index) {
|
||||
av_packet_unref(packet);
|
||||
} else {
|
||||
packetPending = true;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
void renderVideo(double now) {
|
||||
Frame latest;
|
||||
while (!video.empty() && video.front().pts <= now + 0.002) {
|
||||
videoBytes -= video.front().bytes;
|
||||
latest = std::move(video.front().frame);
|
||||
video.pop_front();
|
||||
}
|
||||
if (latest) {
|
||||
output.present(latest.get());
|
||||
previewShown = true;
|
||||
}
|
||||
}
|
||||
|
||||
bool tick() {
|
||||
if (!format || interrupted() || status == PLAYER_ENDED || status == PLAYER_ERROR) return false;
|
||||
bool progressed = pumpAudio();
|
||||
if (status == PLAYER_PAUSED) {
|
||||
if (!previewShown && !video.empty()) {
|
||||
output.present(video.front().frame.get());
|
||||
videoBytes -= video.front().bytes;
|
||||
video.pop_front();
|
||||
previewShown = true;
|
||||
progressed = true;
|
||||
}
|
||||
const bool needPreview = tracks[1].codec && !previewShown;
|
||||
const bool needAudio = tracks[0].codec && output.queued() < 4800;
|
||||
if (needPreview || needAudio) progressed = decodeOne() || progressed;
|
||||
return progressed;
|
||||
}
|
||||
position = clockPosition();
|
||||
renderVideo(position);
|
||||
progressed = decodeOne() || progressed;
|
||||
if (demuxEnded && (!tracks[0].codec || tracks[0].drained) &&
|
||||
resamplerDrained && pcmOffset >= pcmFrames && tracks[0].codec)
|
||||
progressed = writeSilence(videoEnd) || progressed;
|
||||
const bool drained = demuxEnded && resamplerDrained && pcmOffset >= pcmFrames &&
|
||||
(!tracks[0].codec || tracks[0].drained) && (!tracks[1].codec || tracks[1].drained);
|
||||
if (drained && video.empty()) {
|
||||
const double end = std::max(audioWriteEnd, videoEnd);
|
||||
if (position + sampleTime >= end && (!tracks[0].codec || output.queued() == 0)) {
|
||||
output.pause();
|
||||
position = end;
|
||||
if (duration <= 0) duration = end;
|
||||
status = PLAYER_ENDED;
|
||||
}
|
||||
}
|
||||
publish();
|
||||
return progressed;
|
||||
}
|
||||
|
||||
void run() noexcept {
|
||||
while (!stopping.load(std::memory_order_relaxed)) {
|
||||
try {
|
||||
Command command{Action::Pause, {}};
|
||||
bool haveCommand = false;
|
||||
std::array<char, 512> apiFailure{};
|
||||
bool haveFailure = false;
|
||||
uint64_t failureEpoch = 0;
|
||||
{
|
||||
std::lock_guard<std::mutex> lock(mutex);
|
||||
if (errorRequested) {
|
||||
apiFailure = requestedError;
|
||||
failureEpoch = errorEpoch;
|
||||
errorRequested = false;
|
||||
haveFailure = true;
|
||||
} else if (!commands.empty()) {
|
||||
command = std::move(commands.front());
|
||||
commands.pop_front();
|
||||
haveCommand = true;
|
||||
}
|
||||
}
|
||||
if (haveFailure) {
|
||||
if (failureEpoch == requestedEpoch.load(std::memory_order_relaxed)) {
|
||||
activeEpoch = failureEpoch;
|
||||
fail(apiFailure.data());
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (haveCommand) {
|
||||
execute(command);
|
||||
continue;
|
||||
}
|
||||
if (tick()) continue;
|
||||
std::unique_lock<std::mutex> lock(mutex);
|
||||
wake.wait_for(lock, std::chrono::milliseconds(status == PLAYER_PLAYING ? 5 : 20),
|
||||
[this] { return stopping.load(std::memory_order_relaxed) || errorRequested || !commands.empty(); });
|
||||
} catch (const Cancelled &) {
|
||||
// An accepted open/close or destruction superseded the current I/O.
|
||||
} catch (const std::exception &exception) {
|
||||
try { fail(exception.what()); } catch (...) {}
|
||||
} catch (...) {
|
||||
try { fail("Unexpected playback failure"); } catch (...) {}
|
||||
}
|
||||
}
|
||||
try { release(); } catch (...) {}
|
||||
}
|
||||
};
|
||||
|
||||
namespace {
|
||||
template <typename Function>
|
||||
bool submit(Player *player, Function function) noexcept {
|
||||
if (!player) return false;
|
||||
try {
|
||||
return function();
|
||||
} catch (const std::exception &exception) {
|
||||
player->apiError(exception.what());
|
||||
} catch (...) {
|
||||
player->apiError("Unexpected command failure");
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
extern "C" {
|
||||
Player *player_create(void) {
|
||||
try { return new Player; } catch (...) { return nullptr; }
|
||||
}
|
||||
bool player_open(Player *player, const char *path) {
|
||||
if (!path || !*path) return false;
|
||||
return submit(player, [=] { return player->enqueue({Action::Open, path}); });
|
||||
}
|
||||
bool player_play(Player *player) {
|
||||
return submit(player, [=] { return player->enqueue({Action::Play, {}}); });
|
||||
}
|
||||
bool player_pause(Player *player) {
|
||||
return submit(player, [=] { return player->enqueue({Action::Pause, {}}); });
|
||||
}
|
||||
bool player_seek(Player *player, double seconds) {
|
||||
if (!std::isfinite(seconds)) return false;
|
||||
return submit(player, [=] { return player->enqueue({Action::Seek, {}, seconds}); });
|
||||
}
|
||||
bool player_close(Player *player) {
|
||||
return submit(player, [=] { return player->enqueue({Action::Close, {}}); });
|
||||
}
|
||||
PlayerState player_get_state(Player *player) {
|
||||
try {
|
||||
if (player) {
|
||||
std::lock_guard<std::mutex> lock(player->mutex);
|
||||
return player->state;
|
||||
}
|
||||
} catch (const std::exception &exception) {
|
||||
PlayerState result{};
|
||||
result.status = PLAYER_ERROR;
|
||||
copyError(result.error, exception.what());
|
||||
return result;
|
||||
} catch (...) {
|
||||
PlayerState result{};
|
||||
result.status = PLAYER_ERROR;
|
||||
copyError(result.error, "Cannot read playback state");
|
||||
return result;
|
||||
}
|
||||
PlayerState result{};
|
||||
result.status = PLAYER_ERROR;
|
||||
copyError(result.error, "Invalid player handle");
|
||||
return result;
|
||||
}
|
||||
void *player_macos_make_video_view(Player *player) {
|
||||
if (!player) return nullptr;
|
||||
try {
|
||||
return player->output.make_video_view();
|
||||
} catch (const std::exception &exception) {
|
||||
player->apiError(exception.what());
|
||||
} catch (...) {
|
||||
player->apiError("Cannot create the native video view");
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
void player_destroy(Player *player) {
|
||||
try { delete player; } catch (...) {}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
#ifndef PERSONAL_PLAYER_H
|
||||
#define PERSONAL_PLAYER_H
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
typedef struct Player Player;
|
||||
typedef enum PlayerStatus {
|
||||
PLAYER_EMPTY, PLAYER_OPENING, PLAYER_PAUSED, PLAYER_PLAYING,
|
||||
PLAYER_ENDED, PLAYER_ERROR
|
||||
} PlayerStatus;
|
||||
typedef struct PlayerState {
|
||||
PlayerStatus status;
|
||||
double position;
|
||||
double duration;
|
||||
bool has_audio;
|
||||
bool has_video;
|
||||
uint64_t revision;
|
||||
char error[512];
|
||||
} PlayerState;
|
||||
|
||||
// Commands are asynchronous. False means invalid input or a full command queue.
|
||||
// Paths are copied before returning. All other calls require a live handle.
|
||||
Player *player_create(void);
|
||||
bool player_open(Player *player, const char *path);
|
||||
bool player_play(Player *player);
|
||||
bool player_pause(Player *player);
|
||||
bool player_seek(Player *player, double seconds);
|
||||
bool player_close(Player *player);
|
||||
PlayerState player_get_state(Player *player);
|
||||
// Joins the worker and stops audio; never invoke from an audio callback.
|
||||
void player_destroy(Player *player);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
@@ -0,0 +1,12 @@
|
||||
#ifndef PERSONAL_PLAYER_MACOS_H
|
||||
#define PERSONAL_PLAYER_MACOS_H
|
||||
#include "player.h"
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
// Returns an owned NSView (+1). Swift must consume it with takeRetainedValue().
|
||||
void *player_macos_make_video_view(Player *player);
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
@@ -0,0 +1,399 @@
|
||||
#include "../Core/output.hpp"
|
||||
#import <AppKit/AppKit.h>
|
||||
#import <AudioToolbox/AudioToolbox.h>
|
||||
#import <CoreAudio/CoreAudio.h>
|
||||
#import <MetalKit/MetalKit.h>
|
||||
#import <mach/mach_time.h>
|
||||
#import <simd/simd.h>
|
||||
extern "C" {
|
||||
#include <libswscale/swscale.h>
|
||||
}
|
||||
#include <algorithm>
|
||||
#include <array>
|
||||
#include <atomic>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <mutex>
|
||||
#include <stdexcept>
|
||||
#include <vector>
|
||||
|
||||
namespace {
|
||||
constexpr size_t capacity = 48000;
|
||||
double host_seconds(uint64_t ticks) {
|
||||
static const double factor = [] {
|
||||
mach_timebase_info_data_t info;
|
||||
mach_timebase_info(&info);
|
||||
return double(info.numer) / double(info.denom) / 1e9;
|
||||
}();
|
||||
return ticks * factor;
|
||||
}
|
||||
struct VideoMailbox {
|
||||
std::mutex mutex;
|
||||
AVFrame *frame = av_frame_alloc();
|
||||
uint64_t version = 0;
|
||||
~VideoMailbox() { av_frame_free(&frame); }
|
||||
};
|
||||
struct Uniforms { simd_float4 red, green, blue, parameters; };
|
||||
NSString *const shader = @R"metal(
|
||||
#include <metal_stdlib>
|
||||
using namespace metal;
|
||||
struct U { float4 red, green, blue, parameters; };
|
||||
struct V { float4 position [[position]]; float2 uv; };
|
||||
vertex V vertex_main(uint index [[vertex_id]], constant U &u [[buffer(0)]]) {
|
||||
const float2 p[] = { {-1,-1}, {1,-1}, {-1,1}, {1,1} };
|
||||
V v;
|
||||
v.position = float4(p[index] * u.parameters.xy, 0, 1);
|
||||
v.uv = float2((p[index].x + 1) / 2, (1 - p[index].y) / 2);
|
||||
return v;
|
||||
}
|
||||
fragment float4 fragment_main(V v [[stage_in]], constant U &u [[buffer(0)]],
|
||||
texture2d<float> y [[texture(0)]], texture2d<float> cb [[texture(1)]],
|
||||
texture2d<float> cr [[texture(2)]]) {
|
||||
constexpr sampler s(address::clamp_to_edge, filter::linear);
|
||||
if (u.parameters.w == 2) return float4(y.sample(s, v.uv).rgb, 1);
|
||||
float luma = y.sample(s, v.uv).r * u.parameters.z;
|
||||
float2 chroma = u.parameters.w == 1 ? cb.sample(s, v.uv).rg :
|
||||
float2(cb.sample(s, v.uv).r, cr.sample(s, v.uv).r);
|
||||
float4 values = float4(luma, chroma * u.parameters.z, 1);
|
||||
return float4(dot(u.red, values), dot(u.green, values), dot(u.blue, values), 1);
|
||||
}
|
||||
)metal";
|
||||
}
|
||||
|
||||
@interface PlaybackVideoView : MTKView <MTKViewDelegate> {
|
||||
std::shared_ptr<VideoMailbox> mailbox;
|
||||
uint64_t seen;
|
||||
id<MTLCommandQueue> commands;
|
||||
id<MTLRenderPipelineState> pipeline;
|
||||
id<MTLTexture> planes[3];
|
||||
Uniforms uniforms;
|
||||
int imageWidth, imageHeight;
|
||||
double imageAspect;
|
||||
SwsContext *conversion;
|
||||
AVFrame *converted;
|
||||
std::shared_ptr<std::atomic_bool> inFlight;
|
||||
CFStringRef colorName;
|
||||
}
|
||||
- (instancetype)initWithMailbox:(std::shared_ptr<VideoMailbox>)source;
|
||||
@end
|
||||
|
||||
@implementation PlaybackVideoView
|
||||
- (instancetype)initWithMailbox:(std::shared_ptr<VideoMailbox>)source {
|
||||
id<MTLDevice> device = MTLCreateSystemDefaultDevice();
|
||||
if (!device) throw std::runtime_error("Metal is unavailable on this Mac");
|
||||
self = [super initWithFrame:NSMakeRect(0, 0, 960, 540) device:device];
|
||||
if (!self) throw std::runtime_error("Could not create the video view");
|
||||
mailbox = std::move(source);
|
||||
seen = UINT64_MAX;
|
||||
inFlight = std::make_shared<std::atomic_bool>(false);
|
||||
self.colorPixelFormat = MTLPixelFormatBGRA8Unorm;
|
||||
self.clearColor = MTLClearColorMake(0, 0, 0, 1);
|
||||
self.preferredFramesPerSecond = 60;
|
||||
self.delegate = self;
|
||||
self.layer.contentsScale = NSScreen.mainScreen.backingScaleFactor;
|
||||
commands = [device newCommandQueue];
|
||||
NSError *error = nil;
|
||||
id<MTLLibrary> library = [device newLibraryWithSource:shader options:nil error:&error];
|
||||
if (!library) throw std::runtime_error(error.localizedDescription.UTF8String);
|
||||
MTLRenderPipelineDescriptor *description = [MTLRenderPipelineDescriptor new];
|
||||
description.vertexFunction = [library newFunctionWithName:@"vertex_main"];
|
||||
description.fragmentFunction = [library newFunctionWithName:@"fragment_main"];
|
||||
description.colorAttachments[0].pixelFormat = self.colorPixelFormat;
|
||||
pipeline = [device newRenderPipelineStateWithDescriptor:description error:&error];
|
||||
if (!pipeline || !commands) throw std::runtime_error("Could not initialize Metal rendering");
|
||||
converted = av_frame_alloc();
|
||||
if (!converted) throw std::bad_alloc();
|
||||
return self;
|
||||
}
|
||||
- (void)dealloc {
|
||||
self.delegate = nil;
|
||||
sws_freeContext(conversion);
|
||||
av_frame_free(&converted);
|
||||
}
|
||||
- (void)mtkView:(MTKView *)view drawableSizeWillChange:(CGSize)size {}
|
||||
- (void)upload:(const AVFrame *)source {
|
||||
const AVFrame *frame = source;
|
||||
AVPixelFormat format = static_cast<AVPixelFormat>(frame->format);
|
||||
bool planar = format == AV_PIX_FMT_YUV420P || format == AV_PIX_FMT_YUVJ420P ||
|
||||
format == AV_PIX_FMT_YUV420P10LE;
|
||||
bool nv12 = format == AV_PIX_FMT_NV12;
|
||||
bool rgba = format == AV_PIX_FMT_RGBA || format == AV_PIX_FMT_BGRA;
|
||||
bool negativeStride = frame->linesize[0] < 0 || (!rgba && frame->linesize[1] < 0);
|
||||
if ((!planar && !nv12 && !rgba) || negativeStride) {
|
||||
conversion = sws_getCachedContext(conversion, frame->width, frame->height, format,
|
||||
frame->width, frame->height, AV_PIX_FMT_BGRA, SWS_BILINEAR, nullptr, nullptr, nullptr);
|
||||
if (!conversion) throw std::runtime_error("Pixel conversion is unavailable");
|
||||
int space = frame->colorspace == AVCOL_SPC_BT709 ? SWS_CS_ITU709 :
|
||||
frame->colorspace == AVCOL_SPC_BT2020_NCL ? SWS_CS_BT2020 : SWS_CS_ITU601;
|
||||
const int *coefficients = sws_getCoefficients(space);
|
||||
sws_setColorspaceDetails(conversion, coefficients, frame->color_range == AVCOL_RANGE_JPEG,
|
||||
coefficients, 1, 0, 1 << 16, 1 << 16);
|
||||
if (!converted->buf[0] || converted->width != frame->width || converted->height != frame->height) {
|
||||
av_frame_unref(converted);
|
||||
converted->format = AV_PIX_FMT_BGRA;
|
||||
converted->width = frame->width;
|
||||
converted->height = frame->height;
|
||||
if (av_frame_get_buffer(converted, 32) < 0) throw std::bad_alloc();
|
||||
}
|
||||
if (av_frame_make_writable(converted) < 0) throw std::bad_alloc();
|
||||
sws_scale(conversion, frame->data, frame->linesize, 0, frame->height,
|
||||
converted->data, converted->linesize);
|
||||
frame = converted;
|
||||
format = AV_PIX_FMT_BGRA;
|
||||
rgba = true; planar = nv12 = false;
|
||||
}
|
||||
bool ten = format == AV_PIX_FMT_YUV420P10LE;
|
||||
int count = rgba ? 1 : nv12 ? 2 : 3;
|
||||
for (int i = 0; i < count; ++i) {
|
||||
NSUInteger width = i == 0 ? frame->width : (frame->width + 1) / 2;
|
||||
NSUInteger height = i == 0 ? frame->height : (frame->height + 1) / 2;
|
||||
MTLPixelFormat pixel = rgba ? (format == AV_PIX_FMT_BGRA ? MTLPixelFormatBGRA8Unorm : MTLPixelFormatRGBA8Unorm) :
|
||||
ten ? MTLPixelFormatR16Unorm : nv12 && i == 1 ? MTLPixelFormatRG8Unorm : MTLPixelFormatR8Unorm;
|
||||
if (!planes[i] || planes[i].width != width || planes[i].height != height || planes[i].pixelFormat != pixel) {
|
||||
MTLTextureDescriptor *description = [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:pixel
|
||||
width:width height:height mipmapped:NO];
|
||||
description.usage = MTLTextureUsageShaderRead;
|
||||
description.storageMode = MTLStorageModeShared;
|
||||
planes[i] = [self.device newTextureWithDescriptor:description];
|
||||
if (!planes[i]) throw std::runtime_error("Could not allocate video texture");
|
||||
}
|
||||
[planes[i] replaceRegion:MTLRegionMake2D(0, 0, width, height) mipmapLevel:0
|
||||
withBytes:frame->data[i] bytesPerRow:frame->linesize[i]];
|
||||
}
|
||||
// Unused slots still need valid textures for the shared fragment function.
|
||||
if (count < 3) planes[2] = planes[count - 1];
|
||||
if (count == 1) planes[1] = planes[0];
|
||||
imageWidth = frame->width; imageHeight = frame->height;
|
||||
AVRational aspect = source->sample_aspect_ratio;
|
||||
imageAspect = double(imageWidth) / imageHeight * (aspect.num > 0 && aspect.den > 0 ? double(aspect.num) / aspect.den : 1);
|
||||
CFStringRef targetColor = source->color_primaries == AVCOL_PRI_BT2020 ? kCGColorSpaceITUR_2020 :
|
||||
source->color_trc == AVCOL_TRC_IEC61966_2_1 ? kCGColorSpaceSRGB : kCGColorSpaceITUR_709;
|
||||
if (colorName != targetColor) {
|
||||
CGColorSpaceRef space = CGColorSpaceCreateWithName(targetColor);
|
||||
self.colorspace = space;
|
||||
CGColorSpaceRelease(space);
|
||||
colorName = targetColor;
|
||||
}
|
||||
double kr = .299, kb = .114;
|
||||
if (source->colorspace == AVCOL_SPC_BT709 ||
|
||||
(source->colorspace == AVCOL_SPC_UNSPECIFIED && source->height >= 720)) { kr = .2126; kb = .0722; }
|
||||
if (source->colorspace == AVCOL_SPC_BT2020_NCL) { kr = .2627; kb = .0593; }
|
||||
bool full = source->color_range == AVCOL_RANGE_JPEG || format == AV_PIX_FMT_YUVJ420P;
|
||||
double ys = full ? 1 : ten ? 1023. / 876 : 255. / 219;
|
||||
double cs = full ? 1 : ten ? 1023. / 896 : 255. / 224;
|
||||
double offset = full ? 0 : ten ? 64. / 1023 : 16. / 255;
|
||||
double center = ten ? 512. / 1023 : 128. / 255;
|
||||
double r = 2 * (1 - kr) * cs, b = 2 * (1 - kb) * cs;
|
||||
double gc = -2 * kb * (1 - kb) / (1 - kr - kb) * cs;
|
||||
double gr = -2 * kr * (1 - kr) / (1 - kr - kb) * cs;
|
||||
uniforms.red = {float(ys), 0, float(r), float(-ys * offset - r * center)};
|
||||
uniforms.green = {float(ys), float(gc), float(gr), float(-ys * offset - (gc + gr) * center)};
|
||||
uniforms.blue = {float(ys), float(b), 0, float(-ys * offset - b * center)};
|
||||
uniforms.parameters = {1, 1, ten ? float(65535. / 1023) : 1.f, rgba ? 2.f : nv12 ? 1.f : 0.f};
|
||||
}
|
||||
- (void)drawInMTKView:(MTKView *)view {
|
||||
// Do not overwrite shared textures while the GPU is still sampling them.
|
||||
if (inFlight->exchange(true, std::memory_order_acquire)) return;
|
||||
// Keep the mailbox lock only for retaining a frame, never during GPU work.
|
||||
AVFrame *frame = nullptr;
|
||||
{
|
||||
std::lock_guard lock(mailbox->mutex);
|
||||
if (seen != mailbox->version) {
|
||||
seen = mailbox->version;
|
||||
if (mailbox->frame->buf[0]) frame = av_frame_clone(mailbox->frame);
|
||||
else { planes[0] = planes[1] = planes[2] = nil; imageWidth = imageHeight = 0; }
|
||||
}
|
||||
}
|
||||
if (frame) {
|
||||
try { [self upload:frame]; }
|
||||
catch (const std::exception &error) { NSLog(@"Video rendering failed: %s", error.what()); }
|
||||
av_frame_free(&frame);
|
||||
}
|
||||
MTLRenderPassDescriptor *pass = self.currentRenderPassDescriptor;
|
||||
id<CAMetalDrawable> drawable = self.currentDrawable;
|
||||
if (!pass || !drawable) { inFlight->store(false, std::memory_order_release); return; }
|
||||
id<MTLCommandBuffer> command = [commands commandBuffer];
|
||||
id<MTLRenderCommandEncoder> encoder = [command renderCommandEncoderWithDescriptor:pass];
|
||||
if (planes[0] && imageWidth > 0) {
|
||||
double viewAspect = self.drawableSize.width / std::max(1., self.drawableSize.height);
|
||||
uniforms.parameters.x = imageAspect < viewAspect ? imageAspect / viewAspect : 1;
|
||||
uniforms.parameters.y = imageAspect > viewAspect ? viewAspect / imageAspect : 1;
|
||||
[encoder setRenderPipelineState:pipeline];
|
||||
[encoder setVertexBytes:&uniforms length:sizeof(uniforms) atIndex:0];
|
||||
[encoder setFragmentBytes:&uniforms length:sizeof(uniforms) atIndex:0];
|
||||
for (int i = 0; i < 3; ++i) [encoder setFragmentTexture:planes[i] atIndex:i];
|
||||
[encoder drawPrimitives:MTLPrimitiveTypeTriangleStrip vertexStart:0 vertexCount:4];
|
||||
}
|
||||
[encoder endEncoding];
|
||||
auto flight = inFlight;
|
||||
[command addCompletedHandler:^(id<MTLCommandBuffer>) {
|
||||
flight->store(false, std::memory_order_release);
|
||||
}];
|
||||
[command presentDrawable:drawable];
|
||||
[command commit];
|
||||
}
|
||||
@end
|
||||
|
||||
namespace playback {
|
||||
struct Output::Impl {
|
||||
AudioUnit unit = nullptr;
|
||||
std::array<float, capacity * 2> samples{};
|
||||
std::atomic<uint64_t> read{0}, written{0};
|
||||
double origin = 0;
|
||||
double latency = 0;
|
||||
std::atomic<uint64_t> clockVersion{0};
|
||||
std::atomic<double> anchorPTS{NAN}, anchorHost{0}, endPTS{0}, frozen{NAN};
|
||||
bool running = false;
|
||||
std::shared_ptr<VideoMailbox> video = std::make_shared<VideoMailbox>();
|
||||
|
||||
static OSStatus render(void *context, AudioUnitRenderActionFlags *, const AudioTimeStamp *time,
|
||||
UInt32, UInt32 count, AudioBufferList *buffers) {
|
||||
auto &self = *static_cast<Impl *>(context);
|
||||
for (UInt32 i = 0; i < buffers->mNumberBuffers; ++i)
|
||||
std::memset(buffers->mBuffers[i].mData, 0, buffers->mBuffers[i].mDataByteSize);
|
||||
uint64_t r = self.read.load(std::memory_order_relaxed);
|
||||
uint64_t w = self.written.load(std::memory_order_acquire);
|
||||
size_t frames = std::min<uint64_t>(count, w - r);
|
||||
float *destination = static_cast<float *>(buffers->mBuffers[0].mData);
|
||||
size_t first = std::min(frames, capacity - size_t(r % capacity));
|
||||
std::memcpy(destination, self.samples.data() + (r % capacity) * 2, first * 2 * sizeof(float));
|
||||
std::memcpy(destination + first * 2, self.samples.data(), (frames - first) * 2 * sizeof(float));
|
||||
if (frames) {
|
||||
self.clockVersion.fetch_add(1, std::memory_order_acq_rel);
|
||||
self.anchorPTS.store(self.origin + double(r) / 48000, std::memory_order_relaxed);
|
||||
uint64_t host = time->mFlags & kAudioTimeStampHostTimeValid ? time->mHostTime : mach_absolute_time();
|
||||
self.anchorHost.store(host_seconds(host) + self.latency, std::memory_order_relaxed);
|
||||
self.endPTS.store(self.origin + double(r + frames) / 48000, std::memory_order_relaxed);
|
||||
self.clockVersion.fetch_add(1, std::memory_order_release);
|
||||
}
|
||||
self.read.store(r + frames, std::memory_order_release);
|
||||
return noErr;
|
||||
}
|
||||
~Impl() {
|
||||
if (unit) {
|
||||
AudioOutputUnitStop(unit);
|
||||
AudioUnitUninitialize(unit);
|
||||
AudioComponentInstanceDispose(unit);
|
||||
}
|
||||
}
|
||||
};
|
||||
Output::Output() : impl(std::make_unique<Impl>()) { host_seconds(mach_absolute_time()); }
|
||||
Output::~Output() = default;
|
||||
bool Output::configure_audio(std::string &error) {
|
||||
if (impl->unit) return true;
|
||||
AudioComponentDescription description{ kAudioUnitType_Output, kAudioUnitSubType_DefaultOutput,
|
||||
kAudioUnitManufacturer_Apple, 0, 0 };
|
||||
AudioComponent component = AudioComponentFindNext(nullptr, &description);
|
||||
OSStatus result = component ? AudioComponentInstanceNew(component, &impl->unit) : -1;
|
||||
AudioStreamBasicDescription format{};
|
||||
format.mSampleRate = 48000;
|
||||
format.mFormatID = kAudioFormatLinearPCM;
|
||||
format.mFormatFlags = kAudioFormatFlagsNativeFloatPacked;
|
||||
format.mBytesPerPacket = format.mBytesPerFrame = 2 * sizeof(float);
|
||||
format.mFramesPerPacket = 1; format.mChannelsPerFrame = 2; format.mBitsPerChannel = 32;
|
||||
AURenderCallbackStruct callback{Impl::render, impl.get()};
|
||||
if (result == noErr) result = AudioUnitSetProperty(impl->unit, kAudioUnitProperty_StreamFormat,
|
||||
kAudioUnitScope_Input, 0, &format, sizeof(format));
|
||||
if (result == noErr) result = AudioUnitSetProperty(impl->unit, kAudioUnitProperty_SetRenderCallback,
|
||||
kAudioUnitScope_Input, 0, &callback, sizeof(callback));
|
||||
if (result == noErr) result = AudioUnitInitialize(impl->unit);
|
||||
if (result != noErr) {
|
||||
error = "Core Audio initialization failed (" + std::to_string(result) + ")";
|
||||
if (impl->unit) AudioComponentInstanceDispose(impl->unit);
|
||||
impl->unit = nullptr;
|
||||
return false;
|
||||
}
|
||||
AudioDeviceID device = 0;
|
||||
UInt32 size = sizeof(device);
|
||||
if (AudioUnitGetProperty(impl->unit, kAudioOutputUnitProperty_CurrentDevice,
|
||||
kAudioUnitScope_Global, 0, &device, &size) == noErr) {
|
||||
AudioObjectPropertyAddress address{ kAudioDevicePropertyLatency, kAudioObjectPropertyScopeOutput,
|
||||
kAudioObjectPropertyElementMain };
|
||||
UInt32 latency = 0; size = sizeof(latency);
|
||||
Float64 rate = 48000;
|
||||
AudioObjectGetPropertyData(device, &address, 0, nullptr, &size, &latency);
|
||||
address.mSelector = kAudioDevicePropertyNominalSampleRate; address.mScope = kAudioObjectPropertyScopeGlobal;
|
||||
size = sizeof(rate);
|
||||
AudioObjectGetPropertyData(device, &address, 0, nullptr, &size, &rate);
|
||||
address = {kAudioDevicePropertyStreams, kAudioObjectPropertyScopeOutput, kAudioObjectPropertyElementMain};
|
||||
if (AudioObjectGetPropertyDataSize(device, &address, 0, nullptr, &size) == noErr) {
|
||||
std::vector<AudioStreamID> streams(size / sizeof(AudioStreamID));
|
||||
if (AudioObjectGetPropertyData(device, &address, 0, nullptr, &size, streams.data()) == noErr) {
|
||||
UInt32 streamLatency = 0;
|
||||
for (AudioStreamID stream : streams) {
|
||||
AudioObjectPropertyAddress property{kAudioStreamPropertyLatency,
|
||||
kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyElementMain};
|
||||
UInt32 value = 0, bytes = sizeof(value);
|
||||
if (AudioObjectGetPropertyData(stream, &property, 0, nullptr, &bytes, &value) == noErr)
|
||||
streamLatency = std::max(streamLatency, value);
|
||||
}
|
||||
latency += streamLatency;
|
||||
}
|
||||
}
|
||||
impl->latency = rate > 0 ? latency / rate : 0;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
bool Output::start(std::string &error) {
|
||||
if (!impl->unit) return true; // Video-only playback does not open an audio device.
|
||||
if (impl->running) return true;
|
||||
OSStatus status = AudioOutputUnitStart(impl->unit);
|
||||
if (status != noErr) { error = "Could not start Core Audio (" + std::to_string(status) + ")"; return false; }
|
||||
impl->running = true;
|
||||
return true;
|
||||
}
|
||||
void Output::pause() {
|
||||
if (!impl->running) return;
|
||||
double position = clock();
|
||||
AudioOutputUnitStop(impl->unit);
|
||||
impl->running = false;
|
||||
impl->frozen.store(position, std::memory_order_release);
|
||||
}
|
||||
void Output::reset() {
|
||||
impl->read.store(0); impl->written.store(0);
|
||||
impl->origin = 0;
|
||||
impl->anchorPTS.store(NAN); impl->frozen.store(NAN); impl->endPTS.store(0);
|
||||
std::lock_guard lock(impl->video->mutex);
|
||||
av_frame_unref(impl->video->frame);
|
||||
++impl->video->version;
|
||||
}
|
||||
size_t Output::write(const float *samples, size_t frames, double pts) {
|
||||
uint64_t w = impl->written.load(std::memory_order_relaxed);
|
||||
uint64_t r = impl->read.load(std::memory_order_acquire);
|
||||
frames = std::min(frames, capacity - size_t(w - r));
|
||||
if (!frames) return 0;
|
||||
if (w == 0) impl->origin = pts;
|
||||
size_t first = std::min(frames, capacity - size_t(w % capacity));
|
||||
std::memcpy(impl->samples.data() + (w % capacity) * 2, samples, first * 2 * sizeof(float));
|
||||
std::memcpy(impl->samples.data(), samples + first * 2, (frames - first) * 2 * sizeof(float));
|
||||
impl->written.store(w + frames, std::memory_order_release);
|
||||
return frames;
|
||||
}
|
||||
size_t Output::queued() const {
|
||||
return impl->written.load(std::memory_order_acquire) - impl->read.load(std::memory_order_acquire);
|
||||
}
|
||||
double Output::clock() const {
|
||||
if (!impl->running) return impl->frozen.load(std::memory_order_acquire);
|
||||
for (int attempt = 0; attempt < 3; ++attempt) {
|
||||
uint64_t version = impl->clockVersion.load(std::memory_order_acquire);
|
||||
if (version & 1) continue;
|
||||
double pts = impl->anchorPTS.load(std::memory_order_relaxed);
|
||||
double host = impl->anchorHost.load(std::memory_order_relaxed);
|
||||
double end = impl->endPTS.load(std::memory_order_relaxed);
|
||||
if (version != impl->clockVersion.load(std::memory_order_acquire)) continue;
|
||||
if (!std::isfinite(pts)) return NAN;
|
||||
return std::clamp(pts + host_seconds(mach_absolute_time()) - host, pts, end);
|
||||
}
|
||||
return NAN;
|
||||
}
|
||||
void Output::present(const AVFrame *frame) {
|
||||
std::lock_guard lock(impl->video->mutex);
|
||||
av_frame_unref(impl->video->frame);
|
||||
if (av_frame_ref(impl->video->frame, frame) < 0) throw std::bad_alloc();
|
||||
++impl->video->version;
|
||||
}
|
||||
void *Output::make_video_view() {
|
||||
PlaybackVideoView *view = [[PlaybackVideoView alloc] initWithMailbox:impl->video];
|
||||
return (__bridge_retained void *)view;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user