From ebc1916b346a8f2cebe6a83e9cfc54386e3d85b0 Mon Sep 17 00:00:00 2001 From: Dextinfire <> Date: Sat, 21 Dec 2024 01:28:37 -0800 Subject: [PATCH 1/5] Fix video playing faster than audio --- src/audio/openal/ffmpegaudioplayer.cpp | 8 +++---- src/audio/openal/ffmpegaudioplayer.h | 2 +- src/video/clock.cpp | 27 +++++++++-------------- src/video/clock.h | 17 ++++++++------- src/video/ffmpegplayer.cpp | 30 +++++++++++++++----------- 5 files changed, 42 insertions(+), 42 deletions(-) diff --git a/src/audio/openal/ffmpegaudioplayer.cpp b/src/audio/openal/ffmpegaudioplayer.cpp index 43477a68..4e2b1ef4 100644 --- a/src/audio/openal/ffmpegaudioplayer.cpp +++ b/src/audio/openal/ffmpegaudioplayer.cpp @@ -153,10 +153,10 @@ void FFmpegAudioPlayer::Process() { int samplePosition = BufferStartPositions[currentlyPlayingBuffer] + offset; int sampleRate = Player->AudioStream->CodecContext.sampleRate(); samplePosition = std::min(samplePosition, Player->AudioStream->Duration); - ImpLogSlow(LL_Trace, LC_Video, "samplePosition: %f\n", - samplePosition / sampleRate); - - AudioClock.Set(av::Timestamp(samplePosition, av::Rational(sampleRate)), 0); + auto audioTime = av::Timestamp(samplePosition, av::Rational(1, sampleRate)); + double audioS = audioTime.seconds(); + ImpLogSlow(LL_Trace, LC_Video, "samplePosition: %f\n", audioS); + AudioClock.Set(audioTime, 0); FillAudioBuffers(); ALint sourceState; diff --git a/src/audio/openal/ffmpegaudioplayer.h b/src/audio/openal/ffmpegaudioplayer.h index 0bab5fb2..2bc76665 100644 --- a/src/audio/openal/ffmpegaudioplayer.h +++ b/src/audio/openal/ffmpegaudioplayer.h @@ -30,7 +30,7 @@ class FFmpegAudioPlayer : public Audio::FFmpegAudioPlayer { uint8_t HostBuffer[AudioBufferSize]; int FirstFreeBuffer = 0; int FreeBufferCount = AudioBufferCount; - int BufferStartPositions[AudioBufferCount]; + int BufferStartPositions[AudioBufferCount] = {}; bool First = true; }; diff --git a/src/video/clock.cpp b/src/video/clock.cpp index 6c7bcdd9..be986ad6 100644 --- a/src/video/clock.cpp +++ b/src/video/clock.cpp @@ -2,22 +2,12 @@ namespace Impacto::Video { -Clock::Clock() { - Speed = 1.0; - Paused = true; - Pts = MonotonicClock::duration::min(); - LastUpdated = MonotonicClock::duration::min(); - PtsDrift = MonotonicClock::duration::min(); - Serial = -1; -} - void Clock::SyncTo(Clock const& target) { - DoubleSeconds clock = Get(); - DoubleSeconds targetClock = target.Get(); - if (targetClock == MonotonicClock::duration::min() && - (clock != MonotonicClock::duration::min() || - std::chrono::abs(clock - targetClock) > DoubleSeconds(10.0))) { - Set(targetClock, target.Serial); + std::optional clock = Get(); + std::optional targetClock = target.Get(); + if ((!targetClock && clock) || + std::chrono::abs(*clock - *targetClock) > DoubleSeconds(10.0)) { + Set(*targetClock, target.Serial); Paused = false; } } @@ -31,8 +21,11 @@ void Clock::Set(av::Timestamp const& pts, int serial) { Serial = serial; } -Clock::DoubleSeconds Clock::Get() const { - if (Paused || Pts == MonotonicClock::duration::min()) { +std::optional Clock::Get() const { + if (Pts == DoubleSeconds::zero()) { + return std::nullopt; + } + if (Paused) { return Pts; } DoubleSeconds time = MonotonicClock::now().time_since_epoch(); diff --git a/src/video/clock.h b/src/video/clock.h index 83e4d464..be587b47 100644 --- a/src/video/clock.h +++ b/src/video/clock.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include namespace Impacto::Video { @@ -9,18 +10,18 @@ class Clock { public: using MonotonicClock = std::chrono::steady_clock; using DoubleSeconds = std::chrono::duration; - DoubleSeconds Pts; + DoubleSeconds Pts{}; // duration between monotonic clock and frame timestamp - DoubleSeconds PtsDrift; - DoubleSeconds LastUpdated; - double Speed; - int Serial; - bool Paused; + DoubleSeconds PtsDrift{}; + DoubleSeconds LastUpdated{}; + double Speed = 1.0; + int Serial = -1; + bool Paused = true; + bool Init = false; - Clock(); void SyncTo(Clock const& target); void Set(av::Timestamp const& pts, int serial); - DoubleSeconds Get() const; + std::optional Get() const; }; } // namespace Impacto::Video \ No newline at end of file diff --git a/src/video/ffmpegplayer.cpp b/src/video/ffmpegplayer.cpp index 4a2ddf40..dbfd1d18 100644 --- a/src/video/ffmpegplayer.cpp +++ b/src/video/ffmpegplayer.cpp @@ -471,21 +471,21 @@ void FFmpegPlayer::Update(float dt) { } return; } + Clock::DoubleSeconds time = Clock::MonotonicClock::now().time_since_epoch(); if (FrameTimer == FrameTimer.zero()) { - FrameTimer = Clock::MonotonicClock::now().time_since_epoch(); + FrameTimer = time; } - auto time = Clock::MonotonicClock::now().time_since_epoch(); std::chrono::duration duration{}; if (PreviousFrameTimestamp.isValid()) { auto inverseFrameRate = av::Rational(1, 30); - int frameNum = av_rescale_q(frame.Timestamp.timestamp(), - frame.Timestamp.timebase().getValue(), - inverseFrameRate.getValue()); + size_t frameNum = av_rescale_q(frame.Timestamp.timestamp(), + frame.Timestamp.timebase().getValue(), + inverseFrameRate.getValue()); // This isn't the place for it but I can't think of // anything right now - ScrWork[SW_MOVIEFRAME] = frameNum; + ScrWork[SW_MOVIEFRAME] = (int)frameNum; duration = std::chrono::duration( - (frame.Timestamp - PreviousFrameTimestamp).seconds()); + (frame.Timestamp.seconds() - PreviousFrameTimestamp.seconds())); } if (AudioStream) { @@ -533,13 +533,19 @@ void FFmpegPlayer::Render(float videoAlpha) { Clock::DoubleSeconds FFmpegPlayer::GetTargetDelay( Clock::DoubleSeconds duration) { - auto diff = (VideoClock.Get() - MasterClock->Get()); - auto sync_threshold = std::max(Clock::DoubleSeconds(0.04), - std::min(Clock::DoubleSeconds(0.1), duration)); + using std::chrono_literals::operator""s; // Double Seconds + auto videoTime = VideoClock.Get(); + auto masterTime = MasterClock->Get(); + if (!videoTime || !masterTime) { + return duration; + } + auto diff = (*videoTime - *masterTime); + + auto sync_threshold = std::clamp(duration, 0.04s, 0.1s); if (!isnan(diff.count()) && std::chrono::abs(diff) < MaxFrameDuration) { if (diff <= -sync_threshold) - duration = std::max(Clock::DoubleSeconds(0.0), duration + diff); - else if (diff >= sync_threshold && duration > Clock::DoubleSeconds(0.1)) + duration = std::max(0.0s, duration + diff); + else if (diff >= sync_threshold && duration > 0.1s) duration = duration + diff; else if (diff >= sync_threshold) duration = 2 * duration; From cafd6477fcf487056d050a90acb4c8a905504c40 Mon Sep 17 00:00:00 2001 From: Dextinfire <> Date: Sat, 21 Dec 2024 03:35:37 -0800 Subject: [PATCH 2/5] Revert to old synch code --- src/audio/openal/ffmpegaudioplayer.cpp | 2 +- src/video/clock.cpp | 42 +++++---- src/video/clock.h | 25 ++--- src/video/ffmpegplayer.cpp | 126 ++++++++++++------------- src/video/ffmpegplayer.h | 9 +- 5 files changed, 100 insertions(+), 104 deletions(-) diff --git a/src/audio/openal/ffmpegaudioplayer.cpp b/src/audio/openal/ffmpegaudioplayer.cpp index 4e2b1ef4..abfd365d 100644 --- a/src/audio/openal/ffmpegaudioplayer.cpp +++ b/src/audio/openal/ffmpegaudioplayer.cpp @@ -156,7 +156,7 @@ void FFmpegAudioPlayer::Process() { auto audioTime = av::Timestamp(samplePosition, av::Rational(1, sampleRate)); double audioS = audioTime.seconds(); ImpLogSlow(LL_Trace, LC_Video, "samplePosition: %f\n", audioS); - AudioClock.Set(audioTime, 0); + AudioClock.Set(audioS, 0); FillAudioBuffers(); ALint sourceState; diff --git a/src/video/clock.cpp b/src/video/clock.cpp index be986ad6..67fa4577 100644 --- a/src/video/clock.cpp +++ b/src/video/clock.cpp @@ -1,35 +1,41 @@ #include "clock.h" +extern "C" { +#include +} + namespace Impacto::Video { -void Clock::SyncTo(Clock const& target) { - std::optional clock = Get(); - std::optional targetClock = target.Get(); - if ((!targetClock && clock) || - std::chrono::abs(*clock - *targetClock) > DoubleSeconds(10.0)) { - Set(*targetClock, target.Serial); - Paused = false; - } +Clock::Clock() + : Pts(NAN), + PtsDrift(NAN), + LastUpdated(av_gettime_relative() / 1000000.0), + Speed(1.0), + Serial(-1), + Paused(false) {} + +void Clock::SyncTo(Clock* target) { + double clock = Get(); + double targetClock = target->Get(); + if (!isnan(targetClock) && (isnan(clock) || fabs(clock - targetClock) > 10.0)) + Set(targetClock, target->Serial); } -void Clock::Set(av::Timestamp const& pts, int serial) { - Paused = false; - DoubleSeconds time = MonotonicClock::now().time_since_epoch(); - Pts = DoubleSeconds{pts.seconds()}; +void Clock::Set(double pts, int serial) { + double time = av_gettime_relative() / 1000000.0; + Pts = pts; LastUpdated = time; PtsDrift = Pts - time; Serial = serial; } -std::optional Clock::Get() const { - if (Pts == DoubleSeconds::zero()) { - return std::nullopt; - } +double Clock::Get() { if (Paused) { return Pts; + } else { + double time = av_gettime_relative() / 1000000.0; + return PtsDrift + time - (time - LastUpdated) * (1.0 - Speed); } - DoubleSeconds time = MonotonicClock::now().time_since_epoch(); - return PtsDrift + time - (time - LastUpdated) * (1.0 - Speed); } } // namespace Impacto::Video \ No newline at end of file diff --git a/src/video/clock.h b/src/video/clock.h index be587b47..9db53366 100644 --- a/src/video/clock.h +++ b/src/video/clock.h @@ -2,26 +2,21 @@ #include #include -#include namespace Impacto::Video { class Clock { public: - using MonotonicClock = std::chrono::steady_clock; - using DoubleSeconds = std::chrono::duration; - DoubleSeconds Pts{}; - // duration between monotonic clock and frame timestamp - DoubleSeconds PtsDrift{}; - DoubleSeconds LastUpdated{}; - double Speed = 1.0; - int Serial = -1; - bool Paused = true; - bool Init = false; + double Pts; + double PtsDrift; + double LastUpdated; + double Speed; + int Serial; + bool Paused; - void SyncTo(Clock const& target); - void Set(av::Timestamp const& pts, int serial); - std::optional Get() const; + Clock(); + void SyncTo(Clock* target); + void Set(double pts, int serial); + double Get(); }; - } // namespace Impacto::Video \ No newline at end of file diff --git a/src/video/ffmpegplayer.cpp b/src/video/ffmpegplayer.cpp index dbfd1d18..c4bd7d92 100644 --- a/src/video/ffmpegplayer.cpp +++ b/src/video/ffmpegplayer.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include @@ -127,7 +129,6 @@ template void FFmpegPlayer::OpenCodec( void FFmpegPlayer::Play(Io::Stream* stream, bool looping, bool alpha) { // Don't do anything if we don't have the video system if (!IsInit) return; - if (stream == nullptr) { ImpLog(LL_Error, LC_Video, "Stream was a nullptr! This means the caller is buggy. Backing " @@ -199,7 +200,7 @@ void FFmpegPlayer::Play(Io::Stream* stream, bool looping, bool alpha) { VideoClock = Clock(); MasterClock = &VideoClock; - MaxFrameDuration = Clock::DoubleSeconds{ + MaxFrameDuration = { FormatContext.inputFormat().flags() & AVFMT_TS_DISCONT ? 10.0 : 3600.0}; // Danger zone @@ -218,17 +219,18 @@ void FFmpegPlayer::Play(Io::Stream* stream, bool looping, bool alpha) { } bool FFmpegPlayer::QueuesHaveEnoughPackets() { - size_t videoQueueSize = [this]() { + const size_t videoQueueSize = [this]() { std::lock_guard videoPacketLock(VideoStream->PacketLock); return VideoStream->PacketQueue.size(); }(); - size_t audioQueueSize = 0; - if (AudioStream) { - std::lock_guard audioPacketLock(AudioStream->PacketLock); - audioQueueSize = AudioStream->PacketQueue.size(); - } - + const size_t audioQueueSize = [this]() { + if (AudioStream) { + std::lock_guard audioPacketLock(AudioStream->PacketLock); + return AudioStream->PacketQueue.size(); + } + return 26ull; + }(); return (videoQueueSize > 25 && audioQueueSize > 25); } @@ -252,8 +254,8 @@ void FFmpegPlayer::HandleSeekRequest() { AudioStream->FlushFrameQueue(); } - FrameTimer = FrameTimer.min(); - PreviousFrameTimestamp = {}; + FrameTimer = 0; + PreviousFrameTimestamp = -1; } void FFmpegPlayer::Read() { @@ -304,7 +306,6 @@ void FFmpegPlayer::Read() { template void FFmpegPlayer::Decode() { - int ret = 0; std::mutex waitMutex; FFmpegStream* stream; if constexpr (MediaType == AVMEDIA_TYPE_VIDEO) @@ -450,71 +451,74 @@ void FFmpegPlayer::Seek(int64_t pos) { void FFmpegPlayer::Update(float dt) { if (IsPlaying) { if (AudioStream) AudioPlayer->Process(); - AVFrameItem frame; + double duration{}; + double time; { - std::unique_lock frameLock(VideoStream->FrameLock); - if (VideoStream->FrameQueue.empty()) { + std::unique_lock frameLock{VideoStream->FrameLock}; + size_t frameQueueSize = VideoStream->FrameQueue.size(); + if (frameQueueSize == 0) { + frameLock.unlock(); VideoStream->DecodeCond.notify_one(); return; } + AVFrameItem const& frame = + VideoStream->FrameQueue.front(); + ImpLogSlow(LL_Trace, LC_Video, "VideoStream->FrameQueue.size() = %d\n", - VideoStream->FrameQueue.size()); - frame = VideoStream->FrameQueue.front(); + frameQueueSize); SetFlag(SF_MOVIE_DRAWWAIT, false); - } - if (frame.Serial == INT32_MIN) { - if (Looping) { - Seek(0); - } else { - Stop(); + if (frame.Serial == INT32_MIN) { + if (Looping) { + Seek(0); + } else { + Stop(); + } + return; + } + time = av_gettime_relative() / 1000000.0; + if (!FrameTimer) { + FrameTimer = time; + } + if (PreviousFrameTimestamp != -1) { + auto inverseFrameRate = av::Rational(1, 30); + size_t frameNum = av_rescale_q(frame.Timestamp.timestamp(), + frame.Timestamp.timebase().getValue(), + inverseFrameRate.getValue()); + // This isn't the place for it but I can't think of + // anything right now + ScrWork[SW_MOVIEFRAME] = (int)frameNum; + duration = (frame.Timestamp.seconds() - PreviousFrameTimestamp); } - return; - } - Clock::DoubleSeconds time = Clock::MonotonicClock::now().time_since_epoch(); - if (FrameTimer == FrameTimer.zero()) { - FrameTimer = time; - } - std::chrono::duration duration{}; - if (PreviousFrameTimestamp.isValid()) { - auto inverseFrameRate = av::Rational(1, 30); - size_t frameNum = av_rescale_q(frame.Timestamp.timestamp(), - frame.Timestamp.timebase().getValue(), - inverseFrameRate.getValue()); - // This isn't the place for it but I can't think of - // anything right now - ScrWork[SW_MOVIEFRAME] = (int)frameNum; - duration = std::chrono::duration( - (frame.Timestamp.seconds() - PreviousFrameTimestamp.seconds())); } if (AudioStream) { duration = GetTargetDelay(duration); } else { - duration = Clock::DoubleSeconds( - (double)VideoStream->stream.averageFrameRate().getDenominator() / - VideoStream->stream.averageFrameRate().getNumerator()); + duration = + ((double)VideoStream->stream.averageFrameRate().getDenominator() / + VideoStream->stream.averageFrameRate().getNumerator()); } if (time < FrameTimer + duration) { return; } - FrameTimer += std::chrono::duration_cast(duration); - if (duration.count() > 0 && - (time - FrameTimer) > std::chrono::duration(0.1)) { + VideoStream->FrameLock.lock(); + AVFrameItem frame = + std::move(VideoStream->FrameQueue.front()); + VideoStream->FrameQueue.pop(); + VideoStream->FrameLock.unlock(); + FrameTimer += duration; + if (duration > 0 && (time - FrameTimer) > 0.1) { FrameTimer = time; } PreviousFrameTimestamp = frame.Timestamp; - VideoClock.Set(frame.Timestamp, frame.Serial); + VideoClock.Set(frame.Timestamp.seconds(), frame.Serial); VideoTexture->Submit(frame.Frame.data(0), frame.Frame.data(1), frame.Frame.data(2)); PlaybackStarted = true; - { - std::lock_guard frameLock(VideoStream->FrameLock); - VideoStream->FrameQueue.pop(); - } VideoStream->DecodeCond.notify_one(); } } @@ -531,21 +535,13 @@ void FFmpegPlayer::Render(float videoAlpha) { } } -Clock::DoubleSeconds FFmpegPlayer::GetTargetDelay( - Clock::DoubleSeconds duration) { - using std::chrono_literals::operator""s; // Double Seconds - auto videoTime = VideoClock.Get(); - auto masterTime = MasterClock->Get(); - if (!videoTime || !masterTime) { - return duration; - } - auto diff = (*videoTime - *masterTime); - - auto sync_threshold = std::clamp(duration, 0.04s, 0.1s); - if (!isnan(diff.count()) && std::chrono::abs(diff) < MaxFrameDuration) { +double FFmpegPlayer::GetTargetDelay(double duration) { + double diff = VideoClock.Get() - MasterClock->Get(); + double sync_threshold = std::max(0.04, std::min(0.1, duration)); + if (!isnan(diff) && fabs(diff) < MaxFrameDuration) { if (diff <= -sync_threshold) - duration = std::max(0.0s, duration + diff); - else if (diff >= sync_threshold && duration > 0.1s) + duration = std::max(0.0, duration + diff); + else if (diff >= sync_threshold && duration > 0.1) duration = duration + diff; else if (diff >= sync_threshold) duration = 2 * duration; diff --git a/src/video/ffmpegplayer.h b/src/video/ffmpegplayer.h index ed6b9396..a0bd42b8 100644 --- a/src/video/ffmpegplayer.h +++ b/src/video/ffmpegplayer.h @@ -57,8 +57,7 @@ class FFmpegPlayer : public VideoPlayer { private: void FillAudioBuffers(); - std::chrono::duration GetTargetDelay( - std::chrono::duration duration); + double GetTargetDelay(double duration); bool QueuesHaveEnoughPackets(); void HandleSeekRequest(); @@ -91,9 +90,9 @@ class FFmpegPlayer : public VideoPlayer { bool Looping = false; bool ReaderEOF = false; bool PlaybackStarted = false; - av::Timestamp PreviousFrameTimestamp; - Clock::DoubleSeconds FrameTimer; - Clock::DoubleSeconds MaxFrameDuration; + double PreviousFrameTimestamp = 0.0; + double FrameTimer = 0.0; + double MaxFrameDuration = 0.0; bool NoAudio = false; int FrameCount = 0; }; From 4d22bb9566073ec162114e2ef521a34105fe93d3 Mon Sep 17 00:00:00 2001 From: Dextinfire <> Date: Sat, 21 Dec 2024 03:40:58 -0800 Subject: [PATCH 3/5] extern c --- src/video/ffmpegplayer.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/video/ffmpegplayer.cpp b/src/video/ffmpegplayer.cpp index c4bd7d92..b37bcff7 100644 --- a/src/video/ffmpegplayer.cpp +++ b/src/video/ffmpegplayer.cpp @@ -7,15 +7,16 @@ #include #include #include -#include -#include #include - #include #include +extern "C" { +#include +#include +} + #include -#include #include #include #include From bf615a1b887cf4ac2a3e4060ddcb4f19793b1276 Mon Sep 17 00:00:00 2001 From: Dextinfire <> Date: Sat, 21 Dec 2024 03:50:28 -0800 Subject: [PATCH 4/5] Fix build error --- src/video/ffmpegplayer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/video/ffmpegplayer.cpp b/src/video/ffmpegplayer.cpp index b37bcff7..6c2f20af 100644 --- a/src/video/ffmpegplayer.cpp +++ b/src/video/ffmpegplayer.cpp @@ -225,12 +225,12 @@ bool FFmpegPlayer::QueuesHaveEnoughPackets() { return VideoStream->PacketQueue.size(); }(); - const size_t audioQueueSize = [this]() { + const size_t audioQueueSize = [this]() -> size_t { if (AudioStream) { std::lock_guard audioPacketLock(AudioStream->PacketLock); return AudioStream->PacketQueue.size(); } - return 26ull; + return 26; }(); return (videoQueueSize > 25 && audioQueueSize > 25); } From 410ff88ac70b83ec8454d3ce20055a208524e6ea Mon Sep 17 00:00:00 2001 From: Dextinfire <> Date: Sat, 21 Dec 2024 04:12:58 -0800 Subject: [PATCH 5/5] Missing includes --- src/video/clock.cpp | 1 + src/video/clock.h | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/video/clock.cpp b/src/video/clock.cpp index 67fa4577..dd503878 100644 --- a/src/video/clock.cpp +++ b/src/video/clock.cpp @@ -1,4 +1,5 @@ #include "clock.h" +#include "../impacto.h" extern "C" { #include diff --git a/src/video/clock.h b/src/video/clock.h index 9db53366..86d51389 100644 --- a/src/video/clock.h +++ b/src/video/clock.h @@ -1,8 +1,4 @@ #pragma once - -#include -#include - namespace Impacto::Video { class Clock {