Merge pull request #595 from wheremyfoodat/aac

HLE DSP: Implement AAC audio decoding

.gitmodules

@@ -76,3 +76,6 @@
 [submodule "third_party/metal-cpp"]
 	path = third_party/metal-cpp
 	url = https://github.com/Panda3DS-emu/metal-cpp
+[submodule "third_party/fdk-aac"]
+	path = third_party/fdk-aac
+	url = https://github.com/Panda3DS-emu/fdk-aac/

CMakeLists.txt

@@ -225,6 +225,7 @@ else()
 endif()
 
 add_subdirectory(third_party/teakra EXCLUDE_FROM_ALL)
+add_subdirectory(third_party/fdk-aac)
 
 set(CAPSTONE_ARCHITECTURE_DEFAULT OFF)
 set(CAPSTONE_ARM_SUPPORT ON)
@@ -275,7 +276,7 @@ set(APPLET_SOURCE_FILES src/core/applets/applet.cpp src/core/applets/mii_selecto
                         src/core/applets/error_applet.cpp
 )
 set(AUDIO_SOURCE_FILES src/core/audio/dsp_core.cpp src/core/audio/null_core.cpp src/core/audio/teakra_core.cpp
-                       src/core/audio/miniaudio_device.cpp src/core/audio/hle_core.cpp
+                       src/core/audio/miniaudio_device.cpp src/core/audio/hle_core.cpp src/core/audio/aac_decoder.cpp
 )
 set(RENDERER_SW_SOURCE_FILES src/core/renderer_sw/renderer_sw.cpp)
 
@@ -314,7 +315,7 @@ set(HEADER_FILES include/emulator.hpp include/helpers.hpp include/termcolor.hpp
                  include/audio/miniaudio_device.hpp include/ring_buffer.hpp include/bitfield.hpp include/audio/dsp_shared_mem.hpp
                  include/audio/hle_core.hpp include/capstone.hpp include/audio/aac.hpp include/PICA/pica_frag_config.hpp
                  include/PICA/pica_frag_uniforms.hpp include/PICA/shader_gen_types.hpp include/PICA/shader_decompiler.hpp
-                 include/sdl_sensors.hpp include/renderdoc.hpp
+                 include/sdl_sensors.hpp include/renderdoc.hpp include/audio/aac_decoder.hpp
 )
 
 cmrc_add_resource_library(
@@ -478,7 +479,7 @@ set(ALL_SOURCES ${SOURCE_FILES} ${FS_SOURCE_FILES} ${CRYPTO_SOURCE_FILES} ${KERN
     ${AUDIO_SOURCE_FILES} ${HEADER_FILES} ${FRONTEND_HEADER_FILES})
 target_sources(AlberCore PRIVATE ${ALL_SOURCES})
 
-target_link_libraries(AlberCore PRIVATE dynarmic cryptopp glad resources_console_fonts teakra)
+target_link_libraries(AlberCore PRIVATE dynarmic cryptopp glad resources_console_fonts teakra fdk-aac)
 target_link_libraries(AlberCore PUBLIC glad capstone)
 
 if(ENABLE_DISCORD_RPC AND NOT ANDROID)

include/audio/aac.hpp

@@ -54,6 +54,13 @@ namespace Audio::AAC {
 		u32_le sampleCount;
 	};
 
+	struct DecodeRequest {
+		u32_le address;        // Address of input AAC stream
+		u32_le size;           // Size of input AAC stream
+		u32_le destAddrLeft;   // Output address for left channel samples
+		u32_le destAddrRight;  // Output address for right channel samples
+	};
+
 	struct Message {
 		u16_le mode = Mode::None;  // Encode or decode AAC?
 		u16_le command = Command::Init;
@@ -62,7 +69,9 @@ namespace Audio::AAC {
 		// Info on the AAC request
 		union {
 			std::array<u8, 24> commandData{};
+
 			DecodeResponse decodeResponse;
+			DecodeRequest decodeRequest;
 		};
 	};
 

include/audio/aac_decoder.hpp

@@ -0,0 +1,24 @@
+#pragma once
+#include <functional>
+
+#include "audio/aac.hpp"
+#include "helpers.hpp"
+
+struct AAC_DECODER_INSTANCE;
+
+namespace Audio::AAC {
+	class Decoder {
+		using DecoderHandle = AAC_DECODER_INSTANCE*;
+		using PaddrCallback = std::function<u8*(u32)>;
+
+		DecoderHandle decoderHandle = nullptr;
+
+		bool isInitialized() { return decoderHandle != nullptr; }
+		void initialize();
+
+	  public:
+		// Decode function. Takes in a reference to the AAC response & request, and a callback for paddr -> pointer conversions
+		void decode(AAC::Message& response, const AAC::Message& request, PaddrCallback paddrCallback);
+		~Decoder();
+	};
+}  // namespace Audio::AAC

include/audio/hle_core.hpp

@@ -2,10 +2,12 @@
 #include <array>
 #include <cassert>
 #include <deque>
+#include <memory>
 #include <queue>
 #include <vector>
 
 #include "audio/aac.hpp"
+#include "audio/aac_decoder.hpp"
 #include "audio/dsp_core.hpp"
 #include "audio/dsp_shared_mem.hpp"
 #include "memory.hpp"
@@ -33,8 +35,8 @@ namespace Audio {
 			SampleFormat format;
 			SourceType sourceType;
 
-			bool fromQueue = false;        // Is this buffer from the buffer queue or an embedded buffer?
-			bool hasPlayedOnce = false;    // Has the buffer been played at least once before?
+			bool fromQueue = false;      // Is this buffer from the buffer queue or an embedded buffer?
+			bool hasPlayedOnce = false;  // Has the buffer been played at least once before?
 
 			bool operator<(const Buffer& other) const {
 				// Lower ID = Higher priority
@@ -129,6 +131,8 @@ namespace Audio {
 		std::array<Source, Audio::HLE::sourceCount> sources;  // DSP voices
 		Audio::HLE::DspMemory dspRam;
 
+		std::unique_ptr<Audio::AAC::Decoder> aacDecoder;
+
 		void resetAudioPipe();
 		bool loaded = false;  // Have we loaded a component?
 

src/core/audio/aac_decoder.cpp

@@ -0,0 +1,139 @@
+#include "audio/aac_decoder.hpp"
+
+#include <aacdecoder_lib.h>
+
+#include <vector>
+using namespace Audio;
+
+void AAC::Decoder::decode(AAC::Message& response, const AAC::Message& request, AAC::Decoder::PaddrCallback paddrCallback) {
+	// Copy the command and mode fields of the request to the response
+	response.command = request.command;
+	response.mode = request.mode;
+	response.decodeResponse.size = request.decodeRequest.size;
+
+	// Write a dummy response at first. We'll be overwriting it later if decoding goes well
+	response.resultCode = AAC::ResultCode::Success;
+	response.decodeResponse.channelCount = 2;
+	response.decodeResponse.sampleCount = 1024;
+	response.decodeResponse.sampleRate = AAC::SampleRate::Rate48000;
+
+	if (!isInitialized()) {
+		initialize();
+
+		// AAC decoder failed to initialize, return dummy data and return without decoding
+		if (!isInitialized()) {
+			Helpers::warn("Failed to initialize AAC decoder");
+			return;
+		}
+	}
+
+	u8* input = paddrCallback(request.decodeRequest.address);
+	const u8* inputEnd = paddrCallback(request.decodeRequest.address + request.decodeRequest.size);
+	u8* outputLeft = paddrCallback(request.decodeRequest.destAddrLeft);
+	u8* outputRight = nullptr;
+
+	if (input == nullptr || inputEnd == nullptr || outputLeft == nullptr) {
+		Helpers::warn("Invalid pointers passed to AAC decoder");
+		return;
+	}
+
+	u32 bytesValid = request.decodeRequest.size;
+	u32 bufferSize = request.decodeRequest.size;
+
+	// Each frame is 2048 samples with 2 channels
+	static constexpr usize frameSize = 2048 * 2;
+	std::array<s16, frameSize> frame;
+	std::array<std::vector<s16>, 2> audioStreams;
+
+	bool queriedStreamInfo = false;
+
+	while (bytesValid != 0) {
+		if (aacDecoder_Fill(decoderHandle, &input, &bufferSize, &bytesValid) != AAC_DEC_OK) {
+			Helpers::warn("Failed to fill AAC decoder with samples");
+			return;
+		}
+
+		auto decodeResult = aacDecoder_DecodeFrame(decoderHandle, frame.data(), frameSize, 0);
+
+		if (decodeResult == AAC_DEC_TRANSPORT_SYNC_ERROR) {
+			// https://android.googlesource.com/platform/external/aac/+/2ddc922/libAACdec/include/aacdecoder_lib.h#362
+			// According to the above, if we get a sync error, we're not meant to stop decoding, but rather just continue feeding data
+		} else if (decodeResult == AAC_DEC_OK) {
+			auto getSampleRate = [](u32 rate) {
+				switch (rate) {
+					case 8000: return AAC::SampleRate::Rate8000;
+					case 11025: return AAC::SampleRate::Rate11025;
+					case 12000: return AAC::SampleRate::Rate12000;
+					case 16000: return AAC::SampleRate::Rate16000;
+					case 22050: return AAC::SampleRate::Rate22050;
+					case 24000: return AAC::SampleRate::Rate24000;
+					case 32000: return AAC::SampleRate::Rate32000;
+					case 44100: return AAC::SampleRate::Rate44100;
+					case 48000:
+					default: return AAC::SampleRate::Rate48000;
+				}
+			};
+
+			auto info = aacDecoder_GetStreamInfo(decoderHandle);
+			response.decodeResponse.sampleCount = info->frameSize;
+			response.decodeResponse.channelCount = info->numChannels;
+			response.decodeResponse.sampleRate = getSampleRate(info->sampleRate);
+
+			int channels = info->numChannels;
+			// Reserve space in our output stream vectors so push_back doesn't do allocations
+			for (int i = 0; i < channels; i++) {
+				audioStreams[i].reserve(audioStreams[i].size() + info->frameSize);
+			}
+
+			// Fetch output pointer for right output channel if we've got > 1 channel
+			if (channels > 1 && outputRight == nullptr) {
+				outputRight = paddrCallback(request.decodeRequest.destAddrRight);
+				// If the right output channel doesn't point to a proper padddr, return
+				if (outputRight == nullptr) {
+					Helpers::warn("Right AAC output channel doesn't point to valid physical address");
+					return;
+				}
+			}
+
+			for (int sample = 0; sample < info->frameSize; sample++) {
+				for (int stream = 0; stream < channels; stream++) {
+					audioStreams[stream].push_back(frame[(sample * channels) + stream]);
+				}
+			}
+		} else {
+			Helpers::warn("Failed to decode AAC frame");
+			return;
+		}
+	}
+
+	for (int i = 0; i < 2; i++) {
+		auto& stream = audioStreams[i];
+		u8* pointer = (i == 0) ? outputLeft : outputRight;
+
+		if (!stream.empty() && pointer != nullptr) {
+			std::memcpy(pointer, stream.data(), stream.size() * sizeof(s16));
+		}
+	}
+}
+
+void AAC::Decoder::initialize() {
+	decoderHandle = aacDecoder_Open(TRANSPORT_TYPE::TT_MP4_ADTS, 1);
+
+	if (decoderHandle == nullptr) [[unlikely]] {
+		return;
+	}
+
+	// Cap output channel count to 2
+	if (aacDecoder_SetParam(decoderHandle, AAC_PCM_MAX_OUTPUT_CHANNELS, 2) != AAC_DEC_OK) [[unlikely]] {
+		aacDecoder_Close(decoderHandle);
+		decoderHandle = nullptr;
+		return;
+	}
+}
+
+AAC::Decoder::~Decoder() {
+	if (isInitialized()) {
+		aacDecoder_Close(decoderHandle);
+		decoderHandle = nullptr;
+	}
+}

src/core/audio/hle_core.cpp

@@ -6,6 +6,7 @@
 #include <thread>
 #include <utility>
 
+#include "audio/aac_decoder.hpp"
 #include "services/dsp.hpp"
 
 namespace Audio {
@@ -23,6 +24,8 @@ namespace Audio {
 		for (int i = 0; i < sources.size(); i++) {
 			sources[i].index = i;
 		}
+
+		aacDecoder.reset(new Audio::AAC::Decoder());
 	}
 
 	void HLE_DSP::resetAudioPipe() {
@@ -584,7 +587,6 @@ namespace Audio {
 		switch (request.command) {
 			case AAC::Command::EncodeDecode:
 				// Dummy response to stop games from hanging
-				// TODO: Fix this when implementing AAC
 				response.resultCode = AAC::ResultCode::Success;
 				response.decodeResponse.channelCount = 2;
 				response.decodeResponse.sampleCount = 1024;
@@ -593,6 +595,10 @@ namespace Audio {
 
 				response.command = request.command;
 				response.mode = request.mode;
+
+				// We've already got an AAC decoder but it's currently disabled until mixing & output is properly implemented
+				// TODO: Uncomment this when the time comes
+				// aacDecoder->decode(response, request, [this](u32 paddr) { return getPointerPhys<u8>(paddr); });
 				break;
 
 			case AAC::Command::Init: