Engine: implemented video timing based on FPS

Engine/media/video/flic_player.cpp

@@ -48,7 +48,7 @@ HError FlicPlayer::OpenImpl(std::unique_ptr<Common::Stream> data_stream,
 
     _frameDepth = 8;
     _frameSize = Size(fliwidth, fliheight);
-    _frameRate = 1000 / fli_speed;
+    _frameRate = 1000.f / fli_speed;
     return HError::None();
 }
 

Engine/media/video/video_core.cpp

@@ -114,7 +114,8 @@ int video_core_slot_init(std::unique_ptr<AGS::Common::Stream> in,
     auto player = create_video_player(ext_hint);
     if (!player)
         return -1;
-    if (!player->Open(std::move(in), name, params.Flags, params.TargetSize, params.TargetColorDepth))
+    if (!player->Open(std::move(in), name,
+            params.Flags, params.TargetSize, params.TargetColorDepth, params.FPS))
         return -1;
     return video_core_slot_init(std::move(player));
 }
@@ -252,7 +253,7 @@ static void video_core_entry()
 
         video_core_entry_poll();
 
-        g_vcore.poll_cv.wait_for(lk, std::chrono::milliseconds(16));
+        g_vcore.poll_cv.wait_for(lk, std::chrono::milliseconds(8));
     }
 }
 #endif

Engine/media/video/video_core.h

@@ -42,7 +42,7 @@ struct VideoInitParams
     AGS::Engine::VideoFlags Flags = AGS::Engine::kVideo_None;
     Size TargetSize;
     int TargetColorDepth = 0;
-    int FPS = 0;
+    float FPS = 0.f;
 };
 
 // Video slot controls: slots are abstract holders for a playback.

Engine/media/video/videoplayer.cpp

@@ -38,7 +38,7 @@ HError VideoPlayer::Open(std::unique_ptr<Common::Stream> data_stream,
 
 HError VideoPlayer::Open(std::unique_ptr<Common::Stream> data_stream,
     const String &name, int flags,
-    const Size &target_sz, int target_depth)
+    const Size &target_sz, int target_depth, float target_fps)
 {
     // We request a target depth from decoder, but it may ignore our request,
     // so we have to check actual "native" frame's depth after
@@ -48,6 +48,7 @@ HError VideoPlayer::Open(std::unique_ptr<Common::Stream> data_stream,
 
     _name = name;
     _flags = flags;
+    _targetFPS = target_fps > 0.f ? target_fps : _frameRate;
     // Start the audio stream
     if (HasAudio())
     {
@@ -63,7 +64,8 @@ HError VideoPlayer::Open(std::unique_ptr<Common::Stream> data_stream,
         SetTargetFrame(target_sz);
     }
 
-    _frameTime = 1000 / _frameRate;
+    // TODO: actually support dynamic FPS, need to adjust audio speed
+    _frameTime = 1000.f / _targetFPS;
     return HError::None();
 }
 
@@ -128,13 +130,17 @@ void VideoPlayer::Play()
 
     switch (_playState)
     {
-    case PlayStatePaused: Resume(); /* fall-through */
-    case PlayStateInitial: _playState = PlayStatePlaying; break;
-    default: break; // TODO: support rewind/replay from stop/finished state?
+    case PlayStatePaused:
+        ResumeImpl();
+        /* fallthrough */
+    case PlayStateInitial:
+        if (_audioOut)
+            _audioOut->Play();
+        _playState = PlayStatePlaying;
+        break;
+    default:
+        break; // TODO: support rewind/replay from stop/finished state?
     }
-
-    if (_audioOut)
-        _audioOut->Play();
 }
 
 void VideoPlayer::Pause()
@@ -144,15 +150,7 @@ void VideoPlayer::Pause()
     if (_audioOut)
         _audioOut->Pause();
     _playState = PlayStatePaused;
-}
-
-void VideoPlayer::Resume()
-{
-    if (_playState != PlayStatePaused) return;
-
-    if (_audioOut)
-        _audioOut->Resume();
-    _playState = PlayStatePlaying;
+    _pauseTime = AGS_Clock::now();
 }
 
 void VideoPlayer::Seek(float pos_ms)
@@ -162,16 +160,37 @@ void VideoPlayer::Seek(float pos_ms)
 
 void VideoPlayer::SetSpeed(float speed)
 {
-    // FIXME: rewrite frame timing, otherwise it will break when changing speed
-    // need something like a "stage time" reference set when speed changed last time?
-    // or recalculate "timestamp"
-    // since we also have pause/resume problem, we might have "virtual time"
-    /*
+    // Update our virtual "start time" to keep the proper frame timing
+    AGS_Clock::time_point now{};
+    AGS_Clock::duration play_dur{};
+    switch (_playState)
+    {
+    case PlaybackState::PlayStatePlaying:
+        now = AGS_Clock::now();
+        play_dur = now - _firstFrameTime;
+        break;
+    case PlaybackState::PlayStatePaused:
+        now = _pauseTime;
+        play_dur = now - _firstFrameTime;
+        break;
+    default:
+        break;
+    }
+
+    auto old_frametime = _frameTime;
     _targetFPS = _frameRate * speed;
     _frameTime = 1000.f / _targetFPS;
+
+    // Adjust our virtual timestamps by the difference between
+    // previous play duration, and new duration calculated from the new speed.
+    float ft_rel = _frameTime / old_frametime;
+    AGS_Clock::duration virtual_play_dur =
+        AGS_Clock::duration((int64_t)(play_dur.count() * ft_rel));
+    _firstFrameTime = now - virtual_play_dur;
+
+    // Adjust the audio speed separately
     if (_audioOut)
         _audioOut->SetSpeed(speed);
-    */
 }
 
 void VideoPlayer::SetVolume(float volume)
@@ -218,6 +237,13 @@ bool VideoPlayer::Poll()
     return true;
 }
 
+void VideoPlayer::ResumeImpl()
+{
+    // Update our virtual "start time" to keep the proper frame timing
+    auto pause_dur = AGS_Clock::now() - _pauseTime;
+    _firstFrameTime += pause_dur;
+}
+
 void VideoPlayer::BufferVideo()
 {
     if (_videoFrameQueue.size() >= _videoQueueMax)
@@ -274,10 +300,14 @@ void VideoPlayer::BufferAudio()
 
 bool VideoPlayer::ProcessVideo()
 {
+    const auto now = AGS_Clock::now();
+    const auto duration =
+        std::chrono::duration_cast<std::chrono::milliseconds>(now - _firstFrameTime).count();
     // If has got a ready video frame, then test whether it's time to drop it
     if (_videoReadyFrame)
     {
-        if (true /* test timestamp! */)
+        // TODO: get frame's timestamp if available from decoder?
+        if (duration >= _framesPlayed * _frameTime)
         {
             _videoFramePool.push(std::move(_videoReadyFrame));
         }
@@ -287,10 +317,20 @@ bool VideoPlayer::ProcessVideo()
     // a new one from queue
     if (!_videoReadyFrame && _videoFrameQueue.size() > 0)
     {
-        if (true /* test timestamp! */)
+        // TODO: get frame's timestamp if available from decoder?
+        if (_framesPlayed == 0u ||
+            duration >= _framesPlayed * _frameTime)
         {
             _videoReadyFrame = std::move(_videoFrameQueue.front());
             _videoFrameQueue.pop_front();
+
+            // First frame: save timestamp
+            if (_framesPlayed == 0u)
+            {
+                _firstFrameTime = now;
+            }
+
+            _framesPlayed++;
         }
     }
 

Engine/media/video/videoplayer.h

@@ -26,6 +26,7 @@
 #include <deque>
 #include <memory>
 #include <stack>
+#include "ac/timer.h"
 #include "gfx/bitmap.h"
 #include "media/audio/audiodefines.h"
 #include "media/audio/openalsource.h"
@@ -59,7 +60,7 @@ class VideoPlayer
         const String &name, int flags);
     Common::HError Open(std::unique_ptr<Common::Stream> data_stream,
         const String &name, int flags,
-        const Size &target_sz, int target_depth);
+        const Size &target_sz, int target_depth = 0, float target_fps = 0.f);
     // Tells if the videoplayer object is valid and ready to render
     virtual bool IsValid() { return false; }
     bool HasVideo() const { return (_flags & kVideo_EnableVideo) != 0; }
@@ -79,7 +80,7 @@ class VideoPlayer
     const Size &GetFrameSize() const { return _frameSize; }
     const int GetFrameDepth() const { return _frameDepth; }
     // Get suggested video framerate (frames per second)
-    uint32_t GetFramerate() const { return _frameRate; }
+    float GetFramerate() const { return _frameRate; }
     // Tells if video playback is looping
     bool IsLooping() const { return (_flags & kVideo_Loop) != 0; }
     // Get current playback state
@@ -124,12 +125,11 @@ class VideoPlayer
     // Native video frame's format
     int _frameDepth = 0; // bits per pixel
     Size _frameSize{};
-    uint32_t _frameRate = 0u;
+    float _frameRate = 0.f;
 
 private:
-    // Resumes after pausing
-    void Resume();
-
+    // Resume after pause
+    void ResumeImpl();
     // Read and queue video frames
     void BufferVideo();
     // Read and queue audio frames
@@ -147,17 +147,24 @@ class VideoPlayer
     // Output video frame's color depth and size
     Size _targetSize;
     int _targetDepth = 0;
-    size_t _videoQueueMax = 5u;
-    size_t _audioQueueMax = 0u; // we don't have a real queue atm
+    float _targetFPS = 0.f;
+    uint32_t _videoQueueMax = 5u;
+    uint32_t _audioQueueMax = 0u; // we don't have a real queue atm
     // Playback state
-    uint32_t _frameTime = 0u; // frame duration in ms
     PlaybackState _playState = PlayStateInitial;
+    uint32_t _framesPlayed = 0u;
+    // Stage timestamps, used to calculate the next frame timing;
+    // note that these are "virtual time", and are adjusted whenever playback
+    // is paused and resumed, or playback speed changes.
+    AGS_Clock::time_point _firstFrameTime; // time when the first frame was ready
+    AGS_Clock::time_point _pauseTime; // time when the playback was paused
     // Audio
     // Audio queue (single frame for now, because output buffers too)
     SoundBuffer _audioFrame{};
     // Audio output object
     std::unique_ptr<OpenAlSource> _audioOut;
     // Video
+    float _frameTime = 0.f; // frame duration in ms
     // Helper buffer for retrieving video frames of different size/depth;
     // should match "native" video frame size and color depth
     std::unique_ptr<Common::Bitmap> _vframeBuf;