From 7bc3587bf50178a23f62ed9fbf42b4564a48d950 Mon Sep 17 00:00:00 2001 From: Ivan Mogilko Date: Wed, 28 Feb 2024 15:24:37 +0300 Subject: [PATCH] Engine: implement video Looping, Rewind and NextFrame --- Editor/AGS.Editor/Resources/agsdefns.sh | 8 +-- Engine/ac/video_script.cpp | 23 ++++++--- Engine/media/video/theora_player.cpp | 36 +++++++++++-- Engine/media/video/theora_player.h | 6 +++ Engine/media/video/video.cpp | 62 ++++++++++++++++++----- Engine/media/video/video.h | 5 ++ Engine/media/video/video_core.cpp | 23 ++++++--- Engine/media/video/video_core.h | 12 ++--- Engine/media/video/videoplayer.cpp | 67 +++++++++++++++++++++---- Engine/media/video/videoplayer.h | 17 +++++-- 10 files changed, 206 insertions(+), 53 deletions(-) diff --git a/Editor/AGS.Editor/Resources/agsdefns.sh b/Editor/AGS.Editor/Resources/agsdefns.sh index 2faa6e4b5f8..e10786803d4 100644 --- a/Editor/AGS.Editor/Resources/agsdefns.sh +++ b/Editor/AGS.Editor/Resources/agsdefns.sh @@ -2674,10 +2674,10 @@ builtin managed struct VideoPlayer { import void Pause(); /// Advances video by 1 frame, may be called when the video is paused. import void NextFrame(); - /// Changes playback to continue from the specified frame. - import void SeekFrame(int frame); - /// Changes playback to continue from the specified position in milliseconds. - import void SeekMs(int position); + /// Changes playback to continue from the specified frame; returns new position or -1 on error. + import int SeekFrame(int frame); + /// Changes playback to continue from the specified position in milliseconds; returns new position or -1 on error. + import int SeekMs(int position); /// Stops the video completely. import void Stop(); diff --git a/Engine/ac/video_script.cpp b/Engine/ac/video_script.cpp index b14281dc88e..e92884c64b4 100644 --- a/Engine/ac/video_script.cpp +++ b/Engine/ac/video_script.cpp @@ -66,17 +66,26 @@ void VideoPlayer_Pause(ScriptVideoPlayer *sc_video) void VideoPlayer_NextFrame(ScriptVideoPlayer *sc_video) { - // TODO + if (sc_video->GetID() < 0) + return; + VideoControl *video_ctrl = get_video_control(sc_video->GetID()); + video_ctrl->NextFrame(); } -void VideoPlayer_SeekFrame(ScriptVideoPlayer *sc_video, int frame) +int VideoPlayer_SeekFrame(ScriptVideoPlayer *sc_video, int frame) { - // TODO + if (sc_video->GetID() < 0) + return 0; + VideoControl *video_ctrl = get_video_control(sc_video->GetID()); + return video_ctrl->SeekFrame(frame); } -void VideoPlayer_SeekMs(ScriptVideoPlayer *sc_video, int pos) +int VideoPlayer_SeekMs(ScriptVideoPlayer *sc_video, int pos) { - // TODO + if (sc_video->GetID() < 0) + return 0; + VideoControl *video_ctrl = get_video_control(sc_video->GetID()); + return video_ctrl->SeekMs(pos); } void VideoPlayer_Stop(ScriptVideoPlayer *sc_video) @@ -215,12 +224,12 @@ RuntimeScriptValue Sc_VideoPlayer_NextFrame(void *self, const RuntimeScriptValue RuntimeScriptValue Sc_VideoPlayer_SeekFrame(void *self, const RuntimeScriptValue *params, int32_t param_count) { - API_OBJCALL_VOID_PINT(ScriptVideoPlayer, VideoPlayer_SeekFrame); + API_OBJCALL_INT_PINT(ScriptVideoPlayer, VideoPlayer_SeekFrame); } RuntimeScriptValue Sc_VideoPlayer_SeekMs(void *self, const RuntimeScriptValue *params, int32_t param_count) { - API_OBJCALL_VOID_PINT(ScriptVideoPlayer, VideoPlayer_SeekMs); + API_OBJCALL_INT_PINT(ScriptVideoPlayer, VideoPlayer_SeekMs); } RuntimeScriptValue Sc_VideoPlayer_Stop(void *self, const RuntimeScriptValue *params, int32_t param_count) diff --git a/Engine/media/video/theora_player.cpp b/Engine/media/video/theora_player.cpp index 06cd1adf382..5cd2211eadb 100644 --- a/Engine/media/video/theora_player.cpp +++ b/Engine/media/video/theora_player.cpp @@ -51,9 +51,27 @@ void apeg_stream_skip(int bytes, void *ptr) } // -HError TheoraPlayer::OpenImpl(std::unique_ptr data_stream, +HError TheoraPlayer::OpenImpl(std::unique_ptr data_stream, const String &name, int &flags, int target_depth) { + // Init APEG + HError err = OpenAPEGStream(data_stream.get(), name, flags, target_depth); + if (!err) + return err; + + _name = name; + _dataStream = std::move(data_stream); + return HError::None(); +} + +HError TheoraPlayer::OpenAPEGStream(Stream *data_stream, const Common::String &name, int flags, int target_depth) +{ + if (_apegStream) + { + apeg_close_stream(_apegStream); + _apegStream = nullptr; + } + // NOTE: following settings affect only next apeg_open_stream* or // apeg_reset_stream. apeg_set_stream_reader(apeg_stream_init, apeg_stream_read, apeg_stream_skip); @@ -63,7 +81,7 @@ HError TheoraPlayer::OpenImpl(std::unique_ptr data_stream, apeg_disable_length_detection(TRUE); apeg_ignore_audio((flags & kVideo_EnableAudio) == 0); - APEG_STREAM* apeg_stream = apeg_open_stream_ex(data_stream.get()); + APEG_STREAM* apeg_stream = apeg_open_stream_ex(data_stream); if (!apeg_stream) { return new Error(String::FromFormat("Failed to open theora video '%s'; could be an invalid or unsupported format", name.GetCStr())); @@ -75,9 +93,8 @@ HError TheoraPlayer::OpenImpl(std::unique_ptr data_stream, } _apegStream = apeg_stream; - - // Init APEG - _dataStream = std::move(data_stream); + _usedFlags = flags; + _usedDepth = target_depth; _frameDepth = target_depth; _frameRate = _apegStream->frame_rate; _frameSize = Size(video_w, video_h); @@ -109,6 +126,15 @@ void TheoraPlayer::CloseImpl() _apegStream = nullptr; } +bool TheoraPlayer::RewindImpl() +{ + if (apeg_reset_stream(_apegStream) != APEG_OK) + { + OpenAPEGStream(_dataStream.get(), _name, _usedFlags, _usedDepth); + } + return _apegStream != nullptr; +} + bool TheoraPlayer::NextVideoFrame(Bitmap *dst) { assert(_apegStream); diff --git a/Engine/media/video/theora_player.h b/Engine/media/video/theora_player.h index db007e81671..bde8c36152b 100644 --- a/Engine/media/video/theora_player.h +++ b/Engine/media/video/theora_player.h @@ -38,12 +38,18 @@ class TheoraPlayer : public VideoPlayer Common::HError OpenImpl(std::unique_ptr data_stream, const String &name, int &flags, int target_depth) override; void CloseImpl() override; + bool RewindImpl() override; // Retrieves next video frame, implementation-specific bool NextVideoFrame(Common::Bitmap *dst) override; // Retrieves next audio frame, implementation-specific SoundBuffer NextAudioFrame() override; + Common::HError OpenAPEGStream(Stream *data_stream, const String &name, int flags, int target_depth); + + String _name; std::unique_ptr _dataStream; + int _usedFlags = 0; + int _usedDepth = 0; APEG_STREAM *_apegStream = nullptr; // Optional wrapper around original buffer frame (in case we want to extract a portion of it) std::unique_ptr _theoraFullFrame; diff --git a/Engine/media/video/video.cpp b/Engine/media/video/video.cpp index 2b7e3fedfa0..77531f6d012 100644 --- a/Engine/media/video/video.cpp +++ b/Engine/media/video/video.cpp @@ -343,6 +343,54 @@ void VideoControl::Pause() _state = video_core_slot_pause(_videoID); } +bool VideoControl::NextFrame() +{ + if (!IsReady()) + return false; + video_core_slot_pause(_videoID); + if (!video_core_slot_next_frame(_videoID)) + return false; + _state = video_core_slot_get_play_state(_videoID, &_posMs); + _frameIndex; // FIXME + TryAcquireFrame(); + return true; +} + +uint32_t VideoControl::SeekFrame(uint32_t frame) +{ + if (!IsReady()) + return -1; + video_core_slot_pause(_videoID); + bool res = video_core_slot_seek(_videoID, 0.f, frame); + video_core_slot_get_play_state(_videoID, &_posMs, &_frameIndex); + return res ? _frameIndex : UINT32_MAX; +} + +float VideoControl::SeekMs(float pos_ms) +{ + if (!IsReady()) + return -1.f; + video_core_slot_pause(_videoID); + bool res = video_core_slot_seek(_videoID, pos_ms); + video_core_slot_get_play_state(_videoID, &_posMs, &_frameIndex); + return res ? _posMs : -1.f; +} + +void VideoControl::TryAcquireFrame() +{ + auto new_frame = video_core_slot_acquire_vframe(_videoID); + if (new_frame) + { + // FIXME: this is ugly to use different levels of sprite interface here, + // expand dynamic_sprite group of functions instead! + auto old_sprite = spriteset.RemoveSprite(_spriteID); + spriteset.SetSprite(_spriteID, std::move(new_frame), SPF_DYNAMICALLOC); + game_sprite_updated(_spriteID, false); + // Give old frame back to video + video_core_slot_release_vframe(_videoID, std::move(old_sprite)); + } +} + bool VideoControl::Update() { if (!IsReady()) @@ -369,7 +417,7 @@ bool VideoControl::Update() auto speed_f = _speed; if (speed_f <= 0.0) { speed_f = 1.0f; } - video_core_slot_configure(_videoID, vol_f, speed_f); + video_core_slot_configure(_videoID, vol_f, speed_f, _looping); _paramsChanged = false; } @@ -390,17 +438,7 @@ bool VideoControl::Update() return false; // Try get new video frame - auto new_frame = video_core_slot_acquire_vframe(_videoID); - if (new_frame) - { - // FIXME: this is ugly to use different levels of sprite interface here, - // expand dynamic_sprite group of functions instead! - auto old_sprite = spriteset.RemoveSprite(_spriteID); - spriteset.SetSprite(_spriteID, std::move(new_frame), SPF_DYNAMICALLOC); - game_sprite_updated(_spriteID, false); - // Give old frame back to video - video_core_slot_release_vframe(_videoID, std::move(old_sprite)); - } + TryAcquireFrame(); return true; } diff --git a/Engine/media/video/video.h b/Engine/media/video/video.h index 0a068e3afd4..f0792647767 100644 --- a/Engine/media/video/video.h +++ b/Engine/media/video/video.h @@ -118,6 +118,9 @@ class VideoControl bool Play(); void Pause(); + bool NextFrame(); + uint32_t SeekFrame(uint32_t frame); + float SeekMs(float pos_ms); // Synchronize VideoControl with the video playback subsystem; // - start scheduled playback; @@ -127,6 +130,8 @@ class VideoControl bool Update(); private: + void TryAcquireFrame(); + const int _videoID; const int _spriteID; int _scriptHandle = -1; diff --git a/Engine/media/video/video_core.cpp b/Engine/media/video/video_core.cpp index 58e1651e4a1..d27801c47af 100644 --- a/Engine/media/video/video_core.cpp +++ b/Engine/media/video/video_core.cpp @@ -150,16 +150,24 @@ void video_core_slot_stop(int slot_handle) g_vcore.poll_cv.notify_all(); } -void video_core_slot_seek_ms(int slot_handle, float pos_ms) +bool video_core_slot_seek(int slot_handle, float pos_ms, uint32_t frame) { std::lock_guard lk(g_vcore.poll_mutex_m); - g_vcore.slots_[slot_handle]->Seek(pos_ms); + bool res; + if (frame != UINT32_MAX) + res = g_vcore.slots_[slot_handle]->SeekFrame(pos_ms); + else + res = g_vcore.slots_[slot_handle]->Seek(pos_ms); g_vcore.poll_cv.notify_all(); + return res; } -void video_core_slot_next_frame(int slot_handle) +bool video_core_slot_next_frame(int slot_handle) { - // TODO + std::lock_guard lk(g_vcore.poll_mutex_m); + bool res = g_vcore.slots_[slot_handle]->NextFrame(); + g_vcore.poll_cv.notify_all(); + return res; } std::unique_ptr video_core_slot_acquire_vframe(int slot_handle) @@ -184,12 +192,13 @@ void video_core_slot_release_vframe(int slot_handle, std::unique_ptr lk(g_vcore.poll_mutex_m); auto &player = g_vcore.slots_[slot_handle]; player->SetVolume(volume * GlobalGainScaling); player->SetSpeed(speed); + player->SetLooping(loop); g_vcore.poll_cv.notify_all(); } @@ -213,12 +222,14 @@ float video_core_slot_get_duration(int slot_handle) return dur; } -PlaybackState video_core_slot_get_play_state(int slot_handle, float *pos_ms) +PlaybackState video_core_slot_get_play_state(int slot_handle, float *pos_ms, uint32_t *frame_pos) { std::lock_guard lk(g_vcore.poll_mutex_m); auto state = g_vcore.slots_[slot_handle]->GetPlayState(); if (pos_ms) *pos_ms = g_vcore.slots_[slot_handle]->GetPositionMs(); + if (frame_pos) + *frame_pos = g_vcore.slots_[slot_handle]->GetFrameIndex(); g_vcore.poll_cv.notify_all(); return state; } diff --git a/Engine/media/video/video_core.h b/Engine/media/video/video_core.h index 68f02309a8d..71ddc134ae3 100644 --- a/Engine/media/video/video_core.h +++ b/Engine/media/video/video_core.h @@ -58,11 +58,11 @@ PlaybackState video_core_slot_play(int slot_handle); PlaybackState video_core_slot_pause(int slot_handle); // Stop playback on a slot, disposes sound data, frees a slot void video_core_slot_stop(int slot_handle); -// Seek on a slot, new position in milliseconds -void video_core_slot_seek_ms(int slot_handle, float pos_ms); +// Seek on a slot, new position in milliseconds or in frames +bool video_core_slot_seek(int slot_handle, float pos_ms, uint32_t frame = UINT32_MAX); // Advanced a single video frame on a slot; -// this works even if slot is paused. -void video_core_slot_next_frame(int slot_handle); +// this pauses the video +bool video_core_slot_next_frame(int slot_handle); // Locks and returns last prepared video frame; // while locked this bitmap cannot be modified. @@ -80,10 +80,10 @@ void video_core_entry_poll(); // Video core config // Sets up single playback parameters // TODO: consider a struct for passing params instead, perhaps separate video and audio params -void video_core_slot_configure(int slot_handle, float volume, float speed); +void video_core_slot_configure(int slot_handle, float volume, float speed, bool loop); // Returns current playback state, optionally fills in position -PlaybackState video_core_slot_get_play_state(int slot_handle, float *pos_ms = nullptr); +PlaybackState video_core_slot_get_play_state(int slot_handle, float *pos_ms = nullptr, uint32_t *frame_pos = nullptr); // Returns video position in milliseconds float video_core_slot_get_pos_ms(int slot_handle); // Returns video duration in milliseconds diff --git a/Engine/media/video/videoplayer.cpp b/Engine/media/video/videoplayer.cpp index 2ecead05e2e..0cfc8a7ad35 100644 --- a/Engine/media/video/videoplayer.cpp +++ b/Engine/media/video/videoplayer.cpp @@ -145,7 +145,8 @@ void VideoPlayer::Play() void VideoPlayer::Pause() { - if (_playState != PlayStatePlaying) return; + if (_playState != PlayStatePlaying) + return; if (_audioOut) _audioOut->Pause(); @@ -153,9 +154,45 @@ void VideoPlayer::Pause() _pauseTime = AGS_Clock::now(); } -void VideoPlayer::Seek(float pos_ms) +float VideoPlayer::Seek(float pos_ms) +{ + if ((pos_ms == 0.f) && RewindImpl()) + { + return 0.f; + } + return -1.f; // TODO +} + +uint32_t VideoPlayer::SeekFrame(uint32_t frame) { - // TODO + if ((frame == 0) && RewindImpl()) + { + return 0u; + } + return UINT32_MAX; // TODO +} + +bool VideoPlayer::NextFrame() +{ + if (!IsPlaybackReady(_playState) || !HasVideo()) + return false; + if (_playState != PlaybackState::PlayStatePaused) + Pause(); + + if (!ProcessVideo(true)) + { + if (IsLooping() && RewindImpl()) + { + _framesPlayed = 0; + return ProcessVideo(true); + } + else + { + _playState = PlayStateFinished; + return false; + } + } + return true; } void VideoPlayer::SetSpeed(float speed) @@ -223,7 +260,7 @@ bool VideoPlayer::Poll() if (_playState != PlayStatePlaying) return false; - bool res_video = HasVideo() && ProcessVideo(); + bool res_video = HasVideo() && ProcessVideo(false); bool res_audio = HasAudio() && ProcessAudio(); // Stop if nothing is left to process, or if there was error @@ -231,8 +268,16 @@ bool VideoPlayer::Poll() return false; if (!res_video && !res_audio) { - _playState = PlayStateFinished; - return false; + if (IsLooping() && RewindImpl()) + { + _framesPlayed = 0; + return true; + } + else + { + _playState = PlayStateFinished; + return false; + } } return true; } @@ -242,6 +287,9 @@ void VideoPlayer::ResumeImpl() // Update our virtual "start time" to keep the proper frame timing auto pause_dur = AGS_Clock::now() - _pauseTime; _firstFrameTime += pause_dur; + + // TODO: Separate case of resuming after NextFrame, + // must Seek to frame, or audio will fall behind } void VideoPlayer::BufferVideo() @@ -298,7 +346,7 @@ void VideoPlayer::BufferAudio() _audioFrame = NextAudioFrame(); } -bool VideoPlayer::ProcessVideo() +bool VideoPlayer::ProcessVideo(bool force_next) { const auto now = AGS_Clock::now(); const auto duration = @@ -307,7 +355,8 @@ bool VideoPlayer::ProcessVideo() if (_videoReadyFrame) { // TODO: get frame's timestamp if available from decoder? - if (duration >= _framesPlayed * _frameTime) + if (force_next || + duration >= _framesPlayed * _frameTime) { _videoFramePool.push(std::move(_videoReadyFrame)); } @@ -318,7 +367,7 @@ bool VideoPlayer::ProcessVideo() if (!_videoReadyFrame && _videoFrameQueue.size() > 0) { // TODO: get frame's timestamp if available from decoder? - if (_framesPlayed == 0u || + if (force_next || _framesPlayed == 0u || duration >= _framesPlayed * _frameTime) { _videoReadyFrame = std::move(_videoFrameQueue.front()); diff --git a/Engine/media/video/videoplayer.h b/Engine/media/video/videoplayer.h index 16445ffb85d..1b9b20c6e32 100644 --- a/Engine/media/video/videoplayer.h +++ b/Engine/media/video/videoplayer.h @@ -73,14 +73,19 @@ class VideoPlayer void Pause(); // Stops the playback, releasing any resources void Stop(); - // Seek to the given time position - void Seek(float pos_ms); + // Seek to the given time position; returns new pos or -1 on error + float Seek(float pos_ms); + // Seek to the given frame; returns new pos or -1 (UINT32_MAX) on error + uint32_t SeekFrame(uint32_t frame); + // Steps one frame forward, returns whether was successful + bool NextFrame(); const String &GetName() const { return _name; } const Size &GetFrameSize() const { return _frameSize; } - const int GetFrameDepth() const { return _frameDepth; } + int GetFrameDepth() const { return _frameDepth; } // Get suggested video framerate (frames per second) float GetFramerate() const { return _frameRate; } + uint32_t GetFrameIndex() const { return _framesPlayed; /* CHECKME! */ } // Tells if video playback is looping bool IsLooping() const { return (_flags & kVideo_Loop) != 0; } // Get current playback state @@ -92,6 +97,7 @@ class VideoPlayer void SetSpeed(float speed); void SetVolume(float volume); + void SetLooping(bool loop) { _flags = (_flags & ~kVideo_Loop) | (kVideo_Loop * loop); } // Retrieve the currently prepared video frame std::unique_ptr GetReadyFrame(); @@ -109,6 +115,8 @@ class VideoPlayer { return new Common::Error("Internal error: operation not implemented"); }; // Closes the video, implementation-specific virtual void CloseImpl() {}; + // Rewind to the start + virtual bool RewindImpl() { return false; } // Retrieves next video frame, implementation-specific virtual bool NextVideoFrame(Common::Bitmap *dst) { return false; }; // Retrieves next audio frame, implementation-specific @@ -136,7 +144,7 @@ class VideoPlayer void BufferAudio(); // Process buffered video frame(s); // returns if should continue working - bool ProcessVideo(); + bool ProcessVideo(bool force_next); // Process buffered audio frame(s); // returns if should continue working bool ProcessAudio(); @@ -152,6 +160,7 @@ class VideoPlayer uint32_t _audioQueueMax = 0u; // we don't have a real queue atm // Playback state PlaybackState _playState = PlayStateInitial; + // Frames counter, increments with playback, resets on rewind or seek uint32_t _framesPlayed = 0u; // Stage timestamps, used to calculate the next frame timing; // note that these are "virtual time", and are adjusted whenever playback