Merge pull request #95 from CommitteeOfZero/ffmpeg-audio-sync

src/audio/openal/ffmpegaudioplayer.cpp

@@ -153,10 +153,10 @@ void FFmpegAudioPlayer::Process() {
     int samplePosition = BufferStartPositions[currentlyPlayingBuffer] + offset;
     int sampleRate = Player->AudioStream->CodecContext.sampleRate();
     samplePosition = std::min(samplePosition, Player->AudioStream->Duration);
-    ImpLogSlow(LL_Trace, LC_Video, "samplePosition: %f\n",
-               samplePosition / sampleRate);
-
-    AudioClock.Set(av::Timestamp(samplePosition, av::Rational(sampleRate)), 0);
+    auto audioTime = av::Timestamp(samplePosition, av::Rational(1, sampleRate));
+    double audioS = audioTime.seconds();
+    ImpLogSlow(LL_Trace, LC_Video, "samplePosition: %f\n", audioS);
+    AudioClock.Set(audioS, 0);
 
     FillAudioBuffers();
     ALint sourceState;

src/audio/openal/ffmpegaudioplayer.h

@@ -30,7 +30,7 @@ class FFmpegAudioPlayer : public Audio::FFmpegAudioPlayer {
   uint8_t HostBuffer[AudioBufferSize];
   int FirstFreeBuffer = 0;
   int FreeBufferCount = AudioBufferCount;
-  int BufferStartPositions[AudioBufferCount];
+  int BufferStartPositions[AudioBufferCount] = {};
   bool First = true;
 };
 

src/video/clock.cpp

@@ -1,42 +1,42 @@
 #include "clock.h"
+#include "../impacto.h"
+
+extern "C" {
+#include <libavutil/time.h>
+}
 
 namespace Impacto::Video {
 
-Clock::Clock() {
-  Speed = 1.0;
-  Paused = true;
-  Pts = MonotonicClock::duration::min();
-  LastUpdated = MonotonicClock::duration::min();
-  PtsDrift = MonotonicClock::duration::min();
-  Serial = -1;
-}
+Clock::Clock()
+    : Pts(NAN),
+      PtsDrift(NAN),
+      LastUpdated(av_gettime_relative() / 1000000.0),
+      Speed(1.0),
+      Serial(-1),
+      Paused(false) {}
 
-void Clock::SyncTo(Clock const& target) {
-  DoubleSeconds clock = Get();
-  DoubleSeconds targetClock = target.Get();
-  if (targetClock == MonotonicClock::duration::min() &&
-      (clock != MonotonicClock::duration::min() ||
-       std::chrono::abs(clock - targetClock) > DoubleSeconds(10.0))) {
-    Set(targetClock, target.Serial);
-    Paused = false;
-  }
+void Clock::SyncTo(Clock* target) {
+  double clock = Get();
+  double targetClock = target->Get();
+  if (!isnan(targetClock) && (isnan(clock) || fabs(clock - targetClock) > 10.0))
+    Set(targetClock, target->Serial);
 }
 
-void Clock::Set(av::Timestamp const& pts, int serial) {
-  Paused = false;
-  DoubleSeconds time = MonotonicClock::now().time_since_epoch();
-  Pts = DoubleSeconds{pts.seconds()};
+void Clock::Set(double pts, int serial) {
+  double time = av_gettime_relative() / 1000000.0;
+  Pts = pts;
   LastUpdated = time;
   PtsDrift = Pts - time;
   Serial = serial;
 }
 
-Clock::DoubleSeconds Clock::Get() const {
-  if (Paused || Pts == MonotonicClock::duration::min()) {
+double Clock::Get() {
+  if (Paused) {
     return Pts;
+  } else {
+    double time = av_gettime_relative() / 1000000.0;
+    return PtsDrift + time - (time - LastUpdated) * (1.0 - Speed);
   }
-  DoubleSeconds time = MonotonicClock::now().time_since_epoch();
-  return PtsDrift + time - (time - LastUpdated) * (1.0 - Speed);
 }
 
 }  // namespace Impacto::Video

src/video/clock.h

@@ -1,26 +1,18 @@
 #pragma once
-
-#include <chrono>
-#include <timestamp.h>
-
 namespace Impacto::Video {
 
 class Clock {
  public:
-  using MonotonicClock = std::chrono::steady_clock;
-  using DoubleSeconds = std::chrono::duration<double>;
-  DoubleSeconds Pts;
-  // duration between monotonic clock and frame timestamp
-  DoubleSeconds PtsDrift;
-  DoubleSeconds LastUpdated;
+  double Pts;
+  double PtsDrift;
+  double LastUpdated;
   double Speed;
   int Serial;
   bool Paused;
 
   Clock();
-  void SyncTo(Clock const& target);
-  void Set(av::Timestamp const& pts, int serial);
-  DoubleSeconds Get() const;
+  void SyncTo(Clock* target);
+  void Set(double pts, int serial);
+  double Get();
 };
-
 }  // namespace Impacto::Video

src/video/ffmpegplayer.cpp

@@ -8,12 +8,15 @@
 #include <formatcontext.h>
 #include <codeccontext.h>
 #include <timestamp.h>
-
 #include <packet.h>
 #include <rational.h>
 
+extern "C" {
+#include <libavutil/avutil.h>
+#include <libavutil/time.h>
+}
+
 #include <algorithm>
-#include <chrono>
 #include <cstdint>
 #include <mutex>
 #include <optional>
@@ -127,7 +130,6 @@ template void FFmpegPlayer::OpenCodec(
 void FFmpegPlayer::Play(Io::Stream* stream, bool looping, bool alpha) {
   // Don't do anything if we don't have the video system
   if (!IsInit) return;
-
   if (stream == nullptr) {
     ImpLog(LL_Error, LC_Video,
            "Stream was a nullptr! This means the caller is buggy. Backing "
@@ -199,7 +201,7 @@ void FFmpegPlayer::Play(Io::Stream* stream, bool looping, bool alpha) {
 
   VideoClock = Clock();
   MasterClock = &VideoClock;
-  MaxFrameDuration = Clock::DoubleSeconds{
+  MaxFrameDuration = {
       FormatContext.inputFormat().flags() & AVFMT_TS_DISCONT ? 10.0 : 3600.0};
 
   // Danger zone
@@ -218,17 +220,18 @@ void FFmpegPlayer::Play(Io::Stream* stream, bool looping, bool alpha) {
 }
 
 bool FFmpegPlayer::QueuesHaveEnoughPackets() {
-  size_t videoQueueSize = [this]() {
+  const size_t videoQueueSize = [this]() {
     std::lock_guard videoPacketLock(VideoStream->PacketLock);
     return VideoStream->PacketQueue.size();
   }();
 
-  size_t audioQueueSize = 0;
-  if (AudioStream) {
-    std::lock_guard audioPacketLock(AudioStream->PacketLock);
-    audioQueueSize = AudioStream->PacketQueue.size();
-  }
-
+  const size_t audioQueueSize = [this]() -> size_t {
+    if (AudioStream) {
+      std::lock_guard audioPacketLock(AudioStream->PacketLock);
+      return AudioStream->PacketQueue.size();
+    }
+    return 26;
+  }();
   return (videoQueueSize > 25 && audioQueueSize > 25);
 }
 
@@ -252,8 +255,8 @@ void FFmpegPlayer::HandleSeekRequest() {
     AudioStream->FlushFrameQueue();
   }
 
-  FrameTimer = FrameTimer.min();
-  PreviousFrameTimestamp = {};
+  FrameTimer = 0;
+  PreviousFrameTimestamp = -1;
 }
 
 void FFmpegPlayer::Read() {
@@ -304,7 +307,6 @@ void FFmpegPlayer::Read() {
 
 template <AVMediaType MediaType>
 void FFmpegPlayer::Decode() {
-  int ret = 0;
   std::mutex waitMutex;
   FFmpegStream<MediaType>* stream;
   if constexpr (MediaType == AVMEDIA_TYPE_VIDEO)
@@ -450,71 +452,74 @@ void FFmpegPlayer::Seek(int64_t pos) {
 void FFmpegPlayer::Update(float dt) {
   if (IsPlaying) {
     if (AudioStream) AudioPlayer->Process();
-    AVFrameItem<AVMEDIA_TYPE_VIDEO> frame;
+    double duration{};
+    double time;
     {
-      std::unique_lock frameLock(VideoStream->FrameLock);
-      if (VideoStream->FrameQueue.empty()) {
+      std::unique_lock frameLock{VideoStream->FrameLock};
+      size_t frameQueueSize = VideoStream->FrameQueue.size();
+      if (frameQueueSize == 0) {
+        frameLock.unlock();
         VideoStream->DecodeCond.notify_one();
         return;
       }
+      AVFrameItem<AVMEDIA_TYPE_VIDEO> const& frame =
+          VideoStream->FrameQueue.front();
+
       ImpLogSlow(LL_Trace, LC_Video, "VideoStream->FrameQueue.size() = %d\n",
-                 VideoStream->FrameQueue.size());
-      frame = VideoStream->FrameQueue.front();
+                 frameQueueSize);
       SetFlag(SF_MOVIE_DRAWWAIT, false);
-    }
 
-    if (frame.Serial == INT32_MIN) {
-      if (Looping) {
-        Seek(0);
-      } else {
-        Stop();
+      if (frame.Serial == INT32_MIN) {
+        if (Looping) {
+          Seek(0);
+        } else {
+          Stop();
+        }
+        return;
+      }
+      time = av_gettime_relative() / 1000000.0;
+      if (!FrameTimer) {
+        FrameTimer = time;
+      }
+      if (PreviousFrameTimestamp != -1) {
+        auto inverseFrameRate = av::Rational(1, 30);
+        size_t frameNum = av_rescale_q(frame.Timestamp.timestamp(),
+                                       frame.Timestamp.timebase().getValue(),
+                                       inverseFrameRate.getValue());
+        // This isn't the place for it but I can't think of
+        // anything right now
+        ScrWork[SW_MOVIEFRAME] = (int)frameNum;
+        duration = (frame.Timestamp.seconds() - PreviousFrameTimestamp);
       }
-      return;
-    }
-    if (FrameTimer == FrameTimer.zero()) {
-      FrameTimer = Clock::MonotonicClock::now().time_since_epoch();
-    }
-    auto time = Clock::MonotonicClock::now().time_since_epoch();
-    std::chrono::duration<double> duration{};
-    if (PreviousFrameTimestamp.isValid()) {
-      auto inverseFrameRate = av::Rational(1, 30);
-      int frameNum = av_rescale_q(frame.Timestamp.timestamp(),
-                                  frame.Timestamp.timebase().getValue(),
-                                  inverseFrameRate.getValue());
-      // This isn't the place for it but I can't think of
-      // anything right now
-      ScrWork[SW_MOVIEFRAME] = frameNum;
-      duration = std::chrono::duration<double>(
-          (frame.Timestamp - PreviousFrameTimestamp).seconds());
     }
 
     if (AudioStream) {
       duration = GetTargetDelay(duration);
     } else {
-      duration = Clock::DoubleSeconds(
-          (double)VideoStream->stream.averageFrameRate().getDenominator() /
-          VideoStream->stream.averageFrameRate().getNumerator());
+      duration =
+          ((double)VideoStream->stream.averageFrameRate().getDenominator() /
+           VideoStream->stream.averageFrameRate().getNumerator());
     }
 
     if (time < FrameTimer + duration) {
       return;
     }
 
-    FrameTimer += std::chrono::duration_cast<std::chrono::seconds>(duration);
-    if (duration.count() > 0 &&
-        (time - FrameTimer) > std::chrono::duration<double>(0.1)) {
+    VideoStream->FrameLock.lock();
+    AVFrameItem<AVMEDIA_TYPE_VIDEO> frame =
+        std::move(VideoStream->FrameQueue.front());
+    VideoStream->FrameQueue.pop();
+    VideoStream->FrameLock.unlock();
+    FrameTimer += duration;
+    if (duration > 0 && (time - FrameTimer) > 0.1) {
       FrameTimer = time;
     }
     PreviousFrameTimestamp = frame.Timestamp;
 
-    VideoClock.Set(frame.Timestamp, frame.Serial);
+    VideoClock.Set(frame.Timestamp.seconds(), frame.Serial);
     VideoTexture->Submit(frame.Frame.data(0), frame.Frame.data(1),
                          frame.Frame.data(2));
     PlaybackStarted = true;
-    {
-      std::lock_guard frameLock(VideoStream->FrameLock);
-      VideoStream->FrameQueue.pop();
-    }
     VideoStream->DecodeCond.notify_one();
   }
 }
@@ -531,15 +536,13 @@ void FFmpegPlayer::Render(float videoAlpha) {
   }
 }
 
-Clock::DoubleSeconds FFmpegPlayer::GetTargetDelay(
-    Clock::DoubleSeconds duration) {
-  auto diff = (VideoClock.Get() - MasterClock->Get());
-  auto sync_threshold = std::max(Clock::DoubleSeconds(0.04),
-                                 std::min(Clock::DoubleSeconds(0.1), duration));
-  if (!isnan(diff.count()) && std::chrono::abs(diff) < MaxFrameDuration) {
+double FFmpegPlayer::GetTargetDelay(double duration) {
+  double diff = VideoClock.Get() - MasterClock->Get();
+  double sync_threshold = std::max(0.04, std::min(0.1, duration));
+  if (!isnan(diff) && fabs(diff) < MaxFrameDuration) {
     if (diff <= -sync_threshold)
-      duration = std::max(Clock::DoubleSeconds(0.0), duration + diff);
-    else if (diff >= sync_threshold && duration > Clock::DoubleSeconds(0.1))
+      duration = std::max(0.0, duration + diff);
+    else if (diff >= sync_threshold && duration > 0.1)
       duration = duration + diff;
     else if (diff >= sync_threshold)
       duration = 2 * duration;

src/video/ffmpegplayer.h

@@ -57,8 +57,7 @@ class FFmpegPlayer : public VideoPlayer {
 
  private:
   void FillAudioBuffers();
-  std::chrono::duration<double> GetTargetDelay(
-      std::chrono::duration<double> duration);
+  double GetTargetDelay(double duration);
   bool QueuesHaveEnoughPackets();
 
   void HandleSeekRequest();
@@ -91,9 +90,9 @@ class FFmpegPlayer : public VideoPlayer {
   bool Looping = false;
   bool ReaderEOF = false;
   bool PlaybackStarted = false;
-  av::Timestamp PreviousFrameTimestamp;
-  Clock::DoubleSeconds FrameTimer;
-  Clock::DoubleSeconds MaxFrameDuration;
+  double PreviousFrameTimestamp = 0.0;
+  double FrameTimer = 0.0;
+  double MaxFrameDuration = 0.0;
   bool NoAudio = false;
   int FrameCount = 0;
 };