From 8a0706974d80a3626656c2dc7242fa4f9e1e46de Mon Sep 17 00:00:00 2001 From: Ivan Mogilko Date: Sat, 9 Dec 2023 01:12:43 +0300 Subject: [PATCH 1/2] EXPERIMENT: IAGSAudioPlayer --- Engine/plugin/agsplugin.cpp | 58 ++++++++++++++++++++++++++++++- Engine/plugin/agsplugin.h | 69 ++++++++++++++++++++++++++++++++++--- 2 files changed, 121 insertions(+), 6 deletions(-) diff --git a/Engine/plugin/agsplugin.cpp b/Engine/plugin/agsplugin.cpp index 1a3584cecf4..74066267a87 100644 --- a/Engine/plugin/agsplugin.cpp +++ b/Engine/plugin/agsplugin.cpp @@ -50,6 +50,8 @@ #include "main/engine.h" #include "main/game_run.h" #include "media/audio/audio_system.h" +#include "media/audio/openalsource.h" +#include "media/audio/sdldecoder.h" #include "script/script.h" #include "script/script_runtime.h" #include "plugin/plugin_engine.h" @@ -97,7 +99,7 @@ extern RuntimeScriptValue GlobalReturnValue; // **************** PLUGIN IMPLEMENTATION **************** -const int PLUGIN_API_VERSION = 28; +const int PLUGIN_API_VERSION = 29; struct EnginePlugin { EnginePlugin() { @@ -827,6 +829,60 @@ ::IAGSStream *IAGSEngine::GetFileStreamByHandle(int32 fhandle) get_file_stream(fhandle, "IAGSEngine::GetFileStreamByHandle")); } +class AGSAudioPlayer : public IAGSAudioPlayer +{ +public: + AGSAudioPlayer(std::unique_ptr &&src) + : _audioOut(std::move(src)) + { + _audioOut->Play(); + } + virtual ~AGSAudioPlayer() = default; + + // Tells which version of the plugin API this object corresponds to; + // this lets users know which of the following methods are valid to use. + int GetVersion() override + { + return PLUGIN_API_VERSION; + } + void SetConfig(AGSAudioPlayConfig *config) override + { + _audioOut->SetVolume(config->Volume); + } + void Close() override + { + delete this; // darn... + } + void Pause() override + { + _audioOut->Pause(); + } + void Resume() override + { + _audioOut->Resume(); + } + size_t PutData(AGSAudioFrame *frame) override + { + SoundBuffer buf(frame->Data, frame->DataSize, frame->Timestamp); + size_t put_sz = _audioOut->PutData(buf); + _audioOut->Poll(); + return put_sz; + } + +private: + std::unique_ptr _audioOut; +}; + +IAGSAudioPlayer *IAGSEngine::OpenAudioPlayer(AGSAudioFormat *in_format, AGSAudioPlayConfig *config) +{ + SDL_AudioFormat sdl_fmt = in_format->Format; // todo convert + int chans = in_format->Channels; + int freq = in_format->Freq; + auto *player = new AGSAudioPlayer( + std::unique_ptr(new OpenAlSource(sdl_fmt, chans, freq))); + return player; // todo: should save somewhere for safe close on exit?? +} + // *********** General plugin implementation ********** void pl_stop_plugins() { diff --git a/Engine/plugin/agsplugin.h b/Engine/plugin/agsplugin.h index cffc62ca481..5d1a9b1d4b2 100644 --- a/Engine/plugin/agsplugin.h +++ b/Engine/plugin/agsplugin.h @@ -229,7 +229,7 @@ class IAGSScriptManagedObject { virtual int Serialize(void *address, char *buffer, int bufsize) = 0; protected: IAGSScriptManagedObject() {}; - ~IAGSScriptManagedObject() {}; + virtual ~IAGSScriptManagedObject() {}; }; class IAGSManagedObjectReader { @@ -237,7 +237,7 @@ class IAGSManagedObjectReader { virtual void Unserialize(int key, const char *serializedData, int dataSize) = 0; protected: IAGSManagedObjectReader() {}; - ~IAGSManagedObjectReader() {}; + virtual ~IAGSManagedObjectReader() {}; }; class IAGSFontRenderer { @@ -252,7 +252,7 @@ class IAGSFontRenderer { virtual void EnsureTextValidForFont(char *text, int fontNumber) = 0; protected: IAGSFontRenderer() = default; - ~IAGSFontRenderer() = default; + virtual ~IAGSFontRenderer() = default; }; class IAGSFontRenderer2 : public IAGSFontRenderer { @@ -263,7 +263,7 @@ class IAGSFontRenderer2 : public IAGSFontRenderer { virtual int GetLineSpacing(int fontNumber) = 0; protected: IAGSFontRenderer2() = default; - ~IAGSFontRenderer2() = default; + virtual ~IAGSFontRenderer2() = default; }; struct AGSRenderMatrixes { @@ -375,7 +375,61 @@ class IAGSStream { protected: IAGSStream() = default; - ~IAGSStream() = default; + virtual ~IAGSStream() = default; +}; + +#define AGS_AUDIOPLAY_FLD_VOLUME 0x0001 + +#define AGS_AUDIOFRAME_FLD_DATA 0x0001 +#define AGS_AUDIOFRAME_FLD_TIMESTAMP 0x0002 + +// use sdl2 format for test, but maybe switch to our own custom values just in case? +#define AGS_AUDIOFORMAT_F32 0x8120 + +struct AGSAudioFormat +{ + // Set which fields are valid, see AGS_AUDIOFORMAT_FLD_* flags + uint32_t Fields; + int Format; // s16, s32, f16, f32, etc + int Channels; + int Freq; +}; + +struct AGSAudioPlayConfig +{ + // Set which fields are valid, see AGS_AUDIOPLAY_FLD_* flags + uint32_t Fields; + float Volume; +}; + +struct AGSAudioFrame +{ + // Set which fields are valid, see AGS_AUDIOFRAME_FLD_* flags + uint32_t Fields; + void *Data; + size_t DataSize; + int64_t Timestamp; +}; + +// Something that plays the sound data +class IAGSAudioPlayer { +public: + // Tells which version of the plugin API this object corresponds to; + // this lets users know which of the following methods are valid to use. + virtual int GetVersion() = 0; + virtual void SetConfig(AGSAudioPlayConfig *config) = 0; + // Flushes and closes the AudioPlayer. + // After calling this the IAGSAudioPlayer pointer becomes INVALID. + virtual void Close() = 0; + virtual void Pause() = 0; + virtual void Resume() = 0; + // *copies* data on call, does not hold the frame (??? need different rule?) + // returns amount of data copied, or 0 if data cannot be accepted at the moment. + virtual size_t PutData(AGSAudioFrame *frame) = 0; + +protected: + IAGSAudioPlayer() = default; + ~IAGSAudioPlayer() = default; }; @@ -652,6 +706,11 @@ class IAGSEngine { // this stream should not be closed or disposed, doing so will lead to errors in the engine. // Returns null if handle is invalid. AGSIFUNC(IAGSStream*) GetFileStreamByHandle(int32 fhandle); + + // *** BELOW ARE INTERFACE VERSION 29 AND ABOVE ONLY + // open audio player, telling the format of the sound data that will be SENT into this player by user. + // optionally pass AGSAudioPlayConfig setting playback parameters (these may be reset later). + AGSIFUNC(IAGSAudioPlayer*) OpenAudioPlayer(AGSAudioFormat *in_format, AGSAudioPlayConfig *config); }; From c14efb3c0befa2a8d56f1604e3a3524132c33adf Mon Sep 17 00:00:00 2001 From: Ivan Mogilko Date: Sat, 9 Dec 2023 07:27:28 +0300 Subject: [PATCH 2/2] EXPERIMENT: up the OpenAlSource's MaxQueue Our setting is done wrong, because a) it's constants regardless of situation, b) it counts only number of buffers instead of total length (or size). First of all, buffers may be very small, and then if OpenAlSource is used by the system that does not care about it not being able to queue, then the playback will be skipping chunks. The low queue limit was necessary mostly to workaround effects lag, such as setting speed. This may be resolved differently. --- Engine/media/audio/audio_core.cpp | 4 +++- Engine/media/audio/openalsource.cpp | 7 ++++--- Engine/media/audio/openalsource.h | 13 ++++++++----- Engine/media/video/video.cpp | 5 +++-- Engine/plugin/agsplugin.cpp | 3 ++- 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/Engine/media/audio/audio_core.cpp b/Engine/media/audio/audio_core.cpp index fac3d91be01..9b9795cc108 100644 --- a/Engine/media/audio/audio_core.cpp +++ b/Engine/media/audio/audio_core.cpp @@ -77,7 +77,9 @@ AudioCoreSlot::AudioCoreSlot(int handle, std::unique_ptr decoder) : handle_(handle), _decoder(std::move(decoder)) { _source = std::make_unique( - _decoder->GetFormat(), _decoder->GetChannels(), _decoder->GetFreq()); + _decoder->GetFormat(), _decoder->GetChannels(), _decoder->GetFreq(), + 2 /* TODO: make this a constant, + and add comment explaining why we keep this queue limit low for now */); } void AudioCoreSlot::Init() diff --git a/Engine/media/audio/openalsource.cpp b/Engine/media/audio/openalsource.cpp index f14ecf1726f..5bb1b65bacf 100644 --- a/Engine/media/audio/openalsource.cpp +++ b/Engine/media/audio/openalsource.cpp @@ -95,12 +95,13 @@ static struct // OpenAlSource //----------------------------------------------------------------------------- -OpenAlSource::OpenAlSource(SDL_AudioFormat format, int channels, int freq) +OpenAlSource::OpenAlSource(SDL_AudioFormat format, int channels, int freq, uint32_t max_queue) { _inputFmt.format = format; _inputFmt.channels = static_cast(channels); _inputFmt.rate = freq; - _alFormat = OpenAlFormatFromSDLFormat(_inputFmt, _recvFmt);; + _alFormat = OpenAlFormatFromSDLFormat(_inputFmt, _recvFmt); + _maxQueue = max_queue; alGenSources(1, &_source); dump_al_errors(); _resampler.Setup(_inputFmt, _recvFmt); @@ -156,7 +157,7 @@ size_t OpenAlSource::PutData(const SoundBuffer &data) { Unqueue(); // If queue is full, bail out - if (_queued >= MaxQueue) { return 0u; } + if (_queued >= _maxQueue) { return 0u; } // Input buffer is empty? if (!data.Data || (data.Size == 0)) { return 0u; } // Check for free buffers, generate more if necessary diff --git a/Engine/media/audio/openalsource.h b/Engine/media/audio/openalsource.h index fdcb948c044..0ecb58f2ed6 100644 --- a/Engine/media/audio/openalsource.h +++ b/Engine/media/audio/openalsource.h @@ -33,12 +33,9 @@ namespace Engine class OpenAlSource { public: - // Max sound buffers to queue before/during processing - static const ALuint MaxQueue = 2; - // Initializes Al source for the given format; if there's no direct format equivalent // found, setups a resampler. - OpenAlSource(SDL_AudioFormat format, int channels, int freq); + OpenAlSource(SDL_AudioFormat format, int channels, int freq, uint32_t max_queue = 1024u); OpenAlSource(OpenAlSource&& src); ~OpenAlSource(); @@ -87,7 +84,13 @@ class OpenAlSource PlaybackState _playState = PlayStateInitial; float _speed = 1.f; // change in playback rate float _predictTs = 0.f; // next timestamp prediction - unsigned _queued = 0u; + // Max sound buffers to queue during processing. + // TODO: replace limit of buffer count with limit of memory and/or duration. + // TODO: add a comment explaining why do we have to set very small queue limit + // for regular audio slot play (to decrease custom effects lag), + // alternatively have this logic externally to OpenALSource, somewhere in AudioCoreSlot. + uint32_t _maxQueue = 1024u; + uint32_t _queued = 0u; // SDL resampler state, in case dynamic resampling in necessary SDLResampler _resampler; diff --git a/Engine/media/video/video.cpp b/Engine/media/video/video.cpp index 03b41db50a2..e28c35ead15 100644 --- a/Engine/media/video/video.cpp +++ b/Engine/media/video/video.cpp @@ -36,9 +36,10 @@ #include "gfx/ddb.h" #include "gfx/graphicsdriver.h" #include "main/game_run.h" -#include "util/stream.h" #include "media/audio/audio_system.h" #include "media/audio/openal.h" +#include "util/memory_compat.h" +#include "util/stream.h" using namespace AGS::Common; using namespace AGS::Engine; @@ -73,7 +74,7 @@ HError VideoPlayer::Open(const String &name, int flags) { if ((_audioFormat > 0) && (_audioChannels > 0) && (_audioFreq > 0)) { - _audioOut.reset(new OpenAlSource(_audioFormat, _audioChannels, _audioFreq)); + _audioOut = std::make_unique(_audioFormat, _audioChannels, _audioFreq); _audioOut->Play(); _wantAudio = true; } diff --git a/Engine/plugin/agsplugin.cpp b/Engine/plugin/agsplugin.cpp index 74066267a87..c93f06d1c4c 100644 --- a/Engine/plugin/agsplugin.cpp +++ b/Engine/plugin/agsplugin.cpp @@ -57,6 +57,7 @@ #include "plugin/plugin_engine.h" #include "plugin/plugin_builtin.h" #include "util/library.h" +#include "util/memory_compat.h" #include "util/string.h" #include "util/wgt2allg.h" @@ -879,7 +880,7 @@ IAGSAudioPlayer *IAGSEngine::OpenAudioPlayer(AGSAudioFormat *in_format, AGSAudio int chans = in_format->Channels; int freq = in_format->Freq; auto *player = new AGSAudioPlayer( - std::unique_ptr(new OpenAlSource(sdl_fmt, chans, freq))); + std::make_unique(sdl_fmt, chans, freq)); return player; // todo: should save somewhere for safe close on exit?? }