diff --git a/include/memory.hpp b/include/memory.hpp
index a3f4efbe9..6e25f1f8c 100644
--- a/include/memory.hpp
+++ b/include/memory.hpp
@@ -152,8 +152,11 @@ class Memory {
 	static constexpr size_t FASTMEM_FCRAM_OFFSET = 0;                                    // Offset of FCRAM in the fastmem arena
 	static constexpr size_t FASTMEM_DSP_RAM_OFFSET = FASTMEM_FCRAM_OFFSET + FCRAM_SIZE;  // Offset of DSP RAM
 
-#ifdef PANDA3DS_HARDWARE_FASTMEM
-	std::unique_ptr<Common::HostMemory> arena;
+	static constexpr size_t FASTMEM_BACKING_SIZE = FCRAM_SIZE + DSP_RAM_SIZE;
+	// Total size of the virtual address space we will occupy (4GB)
+	static constexpr size_t FASTMEM_VIRTUAL_SIZE = 4_GB;
+
+	Common::HostMemory* arena;
 
 	void addFastmemView(u32 guestVaddr, size_t arenaOffset, size_t size, bool w, bool x = false) {
 		if (useFastmem) {
@@ -169,10 +172,6 @@ class Memory {
 			arena->Map(guestVaddr, arenaOffset, size, perms, false);
 		}
 	}
-#else
-	void resetFastmemViews() {}
-	void addFastmemView(u32 guestVaddr, size_t arenaOffset, size_t size, bool r, bool w, bool x = false) {}
-#endif
 
 	std::bitset<FCRAM_PAGE_COUNT> usedFCRAMPages;
 	std::optional<u32> findPaddr(u32 size);
@@ -334,11 +333,6 @@ class Memory {
 	Regions getConsoleRegion();
 	void copySharedFont(u8* ptr, u32 vaddr);
 
-#ifdef PANDA3DS_HARDWARE_FASTMEM
 	bool isFastmemEnabled() { return useFastmem; }
 	u8* getFastmemArenaBase() { return arena->VirtualBasePointer(); }
-#else
-	bool isFastmemEnabled() { return false; }
-	u8* getFastmemArenaBase() { return nullptr; }
-#endif
 };
diff --git a/src/core/memory.cpp b/src/core/memory.cpp
index 5675196ad..7f9e825c5 100644
--- a/src/core/memory.cpp
+++ b/src/core/memory.cpp
@@ -15,35 +15,16 @@ CMRC_DECLARE(ConsoleFonts);
 using namespace KernelMemoryTypes;
 
 Memory::Memory(const EmulatorConfig& config) : config(config) {
-	fcram = new uint8_t[FCRAM_SIZE]();
+	arena = new Common::HostMemory(FASTMEM_BACKING_SIZE, FASTMEM_VIRTUAL_SIZE);
 
 	readTable.resize(totalPageCount, 0);
 	writeTable.resize(totalPageCount, 0);
 	memoryInfo.reserve(32);  // Pre-allocate some room for memory allocation info to avoid dynamic allocs
 
-#ifdef PANDA3DS_HARDWARE_FASTMEM
-	u8* arenaFcram = nullptr;
-	u8* arenaDSPRam = nullptr;
+	fcram = arena->BackingBasePointer() + FASTMEM_FCRAM_OFFSET;
+	// arenaDSPRam = arena->BackingBasePointer() + FASTMEM_DSP_RAM_OFFSET;
 
-	constexpr size_t BACKING_SIZE = FCRAM_SIZE + DSP_RAM_SIZE;
-	constexpr size_t VIRTUAL_SIZE = 4_GB;  // Total size of the virtual address space we will occupy (4GB)
-
-	try {
-		arena.reset(new Common::HostMemory(BACKING_SIZE, VIRTUAL_SIZE));
-		arenaFcram = arena->BackingBasePointer() + FASTMEM_FCRAM_OFFSET;
-		// arenaDSPRam = arena->VirtualBasePointer() + FASTMEM_DSP_RAM_OFFSET;
-
-		useFastmem = true;
-		delete[] fcram;
-
-		fcram = arenaFcram;
-	} catch (...) {
-		useFastmem = false;
-	}
-#else
-	useFastmem = false;
-	fastmemArenaBase = nullptr;
-#endif
+	useFastmem = arena->VirtualBasePointer() != nullptr;
 }
 
 void Memory::reset() {
@@ -98,7 +79,9 @@ void Memory::reset() {
 		readTable[i + initialPage] = pointer;
 		writeTable[i + initialPage] = pointer;
 	}
-	// addFastmemView(VirtualAddrs::DSPMemStart, FASTMEM_DSP_RAM_OFFSET, DSP_RAM_SIZE, true, false);  // Allocate RW mapping for DSP RAM
+
+	// Allocate RW mapping for DSP RAM
+	// addFastmemView(VirtualAddrs::DSPMemStart, FASTMEM_DSP_RAM_OFFSET, DSP_RAM_SIZE, true, false);
 
 	// Later adjusted based on ROM header when possible
 	region = Regions::USA;
@@ -514,6 +497,10 @@ void Memory::mirrorMapping(u32 destAddress, u32 sourceAddress, u32 size) {
 
 	const u32 pageCount = size / pageSize;  // How many pages we need to mirror
 	for (u32 i = 0; i < pageCount; i++) {
+		if (useFastmem) {
+			Helpers::panic("Unimplemented: Mirror mapping with fastmem enabled");
+		}
+
 		// Redo the shift here to "properly" handle wrapping around the address space instead of reading OoB
 		const u32 sourcePage = sourceAddress / pageSize;
 		const u32 destPage = destAddress / pageSize;
diff --git a/third_party/host_memory/host_memory.cpp b/third_party/host_memory/host_memory.cpp
index 4d9e9d097..dda43549e 100644
--- a/third_party/host_memory/host_memory.cpp
+++ b/third_party/host_memory/host_memory.cpp
@@ -59,7 +59,7 @@ namespace Common {
 	constexpr size_t PageAlignment = 0x1000;
 	constexpr size_t HugePageSize = 0x200000;
 
-#ifdef _WIN32
+#if defined(_WIN32) && defined(PANDA3DS_HARDWARE_FASTMEM)
 
 // Manually imported for MinGW compatibility
 #ifndef MEM_RESERVE_PLACEHOLDER
@@ -365,7 +365,7 @@ namespace Common {
 		std::unordered_map<size_t, size_t> placeholder_host_pointers;  ///< Placeholder backing offset
 	};
 
-#elif defined(__linux__) || defined(__FreeBSD__)  // ^^^ Windows ^^^ vvv Linux vvv
+#elif (defined(__linux__) || defined(__FreeBSD__)) && defined(PANDA3DS_HARDWARE_FASTMEM)  // ^^^ Windows ^^^ vvv Linux vvv
 
 #ifdef __ANDROID__
 #define ASHMEM_DEVICE "/dev/ashmem"