Win32: Switch to a UNICODE build. This took quite a bit of fixing.

Common/Common.vcxproj

@@ -27,22 +27,22 @@
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
-    <CharacterSet>MultiByte</CharacterSet>
+    <CharacterSet>Unicode</CharacterSet>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <UseDebugLibraries>true</UseDebugLibraries>
-    <CharacterSet>MultiByte</CharacterSet>
+    <CharacterSet>Unicode</CharacterSet>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
-    <CharacterSet>MultiByte</CharacterSet>
+    <CharacterSet>Unicode</CharacterSet>
   </PropertyGroup>
   <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'" Label="Configuration">
     <ConfigurationType>StaticLibrary</ConfigurationType>
     <UseDebugLibraries>false</UseDebugLibraries>
-    <CharacterSet>MultiByte</CharacterSet>
+    <CharacterSet>Unicode</CharacterSet>
   </PropertyGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
   <ImportGroup Label="ExtensionSettings">

Common/ConsoleListener.cpp

@@ -28,6 +28,7 @@
 #endif
 
 #include "thread/threadutil.h"
+#include "util/text/utf8.h"
 #include "Common.h"
 #include "LogManager.h" // Common
 #include "ConsoleListener.h" // Common
@@ -89,7 +90,9 @@ void ConsoleListener::Open(bool Hidden, int Width, int Height, const char *Title
 		// Save the window handle that AllocConsole() created
 		hConsole = GetStdHandle(STD_OUTPUT_HANDLE);
 		// Set the console window title
-		SetConsoleTitle(Title);
+		SetConsoleTitle(ConvertUTF8ToWString(Title).c_str());
+		SetConsoleCP(CP_UTF8);
+
 		// Set letter space
 		LetterSpace(Width, LOG_MAX_DISPLAY_LINES);
 		//MoveWindow(GetConsoleWindow(), 200,200, 800,800, true);
@@ -415,6 +418,7 @@ void ConsoleListener::WriteToConsole(LogTypes::LOG_LEVELS Level, const char *Tex
 	*/
 	DWORD cCharsWritten;
 	WORD Color;
+	static wchar_t tempBuf[2048];
 
 	switch (Level)
 	{
@@ -441,12 +445,16 @@ void ConsoleListener::WriteToConsole(LogTypes::LOG_LEVELS Level, const char *Tex
 	{
 		// First 10 chars white
 		SetConsoleTextAttribute(hConsole, FOREGROUND_RED | FOREGROUND_GREEN | FOREGROUND_BLUE | FOREGROUND_INTENSITY);
-		WriteConsole(hConsole, Text, 10, &cCharsWritten, NULL);
+		int wlen = MultiByteToWideChar(CP_UTF8, 0, Text, (int)Len, NULL, NULL);
+		MultiByteToWideChar(CP_UTF8, 0, Text, (int)Len, tempBuf, wlen);
+		WriteConsole(hConsole, tempBuf, 10, &cCharsWritten, NULL);
 		Text += 10;
 		Len -= 10;
 	}
 	SetConsoleTextAttribute(hConsole, Color);
-	WriteConsole(hConsole, Text, (DWORD)Len, &cCharsWritten, NULL);
+	int wlen = MultiByteToWideChar(CP_UTF8, 0, Text, (int)Len, NULL, NULL);
+	MultiByteToWideChar(CP_UTF8, 0, Text, (int)Len, tempBuf, wlen);
+	WriteConsole(hConsole, tempBuf, (DWORD)Len, &cCharsWritten, NULL);
 }
 #endif
 
@@ -476,7 +484,7 @@ void ConsoleListener::PixelSpace(int Left, int Top, int Width, int Height, bool
 
 	static const int MAX_BYTES = 1024 * 16;
 
-	std::vector<std::array<CHAR, MAX_BYTES>> Str;
+	std::vector<std::array<wchar_t, MAX_BYTES>> Str;
 	std::vector<std::array<WORD, MAX_BYTES>> Attr;
 
 	// ReadConsoleOutputAttribute seems to have a limit at this level

Common/ExtendedTrace.cpp

@@ -17,6 +17,8 @@
 #include "CommonWindows.h"
 #include <stdio.h>
 #include "ExtendedTrace.h"
+
+#include <string>
 using namespace std;
 
 #include <tchar.h>
@@ -320,7 +322,7 @@ void StackTrace( HANDLE hThread, LPCTSTR lpszMessage, FILE *file )
 
 		GetFunctionInfoFromAddresses( (ULONG)callStack.AddrPC.Offset, (ULONG)callStack.AddrFrame.Offset, symInfo );
 		GetSourceInfoFromAddress( (ULONG)callStack.AddrPC.Offset, srcInfo );
-		etfprint(file, string("     ") + srcInfo + string(" : ") + symInfo + string("\n"));
+		etfprint(file, wstring(L"     ") + srcInfo + L" : " + symInfo + L"\n");
 
 		for( ULONG index = 0; ; index++ ) 
 		{
@@ -343,30 +345,14 @@ void StackTrace( HANDLE hThread, LPCTSTR lpszMessage, FILE *file )
 
 			GetFunctionInfoFromAddresses( (ULONG)callStack.AddrPC.Offset, (ULONG)callStack.AddrFrame.Offset, symInfo );
 			GetSourceInfoFromAddress( (UINT)callStack.AddrPC.Offset, srcInfo );
-			etfprint(file, string("     ") + srcInfo + string(" : ") + symInfo + string("\n"));
+			etfprint(file, wstring(L"     ") + srcInfo + L" : " + symInfo + L"\n");
 
 		}
 
 		if ( hThread != GetCurrentThread() )
 			ResumeThread( hThread );
 }
 
-#ifndef UNICODE
-
-void StackTrace( HANDLE hThread,  wchar_t const*lpszMessage, FILE *file, DWORD eip, DWORD esp, DWORD ebp )
-{
-	// TODO: remove when Common builds as unicode
-	size_t origsize = wcslen(lpszMessage) + 1;
-	const size_t newsize = 100;
-	size_t convertedChars = 0;
-	char nstring[newsize];
-	wcstombs_s(&convertedChars, nstring, origsize, lpszMessage, _TRUNCATE);
-
-	StackTrace(hThread, nstring, file, eip, esp, ebp );
-}
-
-#endif
-
 void StackTrace( HANDLE hThread, LPCTSTR lpszMessage, FILE *file, DWORD eip, DWORD esp, DWORD ebp )
 {
 	STACKFRAME     callStack;
@@ -397,7 +383,7 @@ void StackTrace( HANDLE hThread, LPCTSTR lpszMessage, FILE *file, DWORD eip, DWO
 
 		GetFunctionInfoFromAddresses( (ULONG)callStack.AddrPC.Offset, (ULONG)callStack.AddrFrame.Offset, symInfo );
 		GetSourceInfoFromAddress( (UINT)callStack.AddrPC.Offset, srcInfo );
-		etfprint(file, string("     ") + srcInfo + string(" : ") + symInfo + string("\n"));
+		etfprint(file, wstring(L"     ") + srcInfo + L" : " + symInfo + L"\n");
 
 		for( ULONG index = 0; ; index++ ) 
 		{
@@ -420,8 +406,7 @@ void StackTrace( HANDLE hThread, LPCTSTR lpszMessage, FILE *file, DWORD eip, DWO
 
 			GetFunctionInfoFromAddresses( (ULONG)callStack.AddrPC.Offset, (ULONG)callStack.AddrFrame.Offset, symInfo );
 			GetSourceInfoFromAddress( (UINT)callStack.AddrPC.Offset, srcInfo );
-			etfprint(file, string("     ") + srcInfo + string(" : ") + symInfo + string("\n"));
-
+			etfprint(file, wstring(L"     ") + srcInfo + L" : " + symInfo + L"\n");
 		}
 
 		if ( hThread != GetCurrentThread() )
@@ -443,4 +428,9 @@ void etfprint(FILE *file, const std::string &text) {
 	fwrite(text.data(), 1, len, file);
 }
 
+void etfprint(FILE *file, const std::wstring &text) {
+	size_t len = text.length();
+	fwrite(text.data(), sizeof(wchar_t), len, file);
+}
+
 #endif //WIN32

Common/ExtendedTrace.h

@@ -39,6 +39,7 @@ void StackTrace( HANDLE hThread, wchar_t const* lpszMessage, FILE *file, DWORD e
 // functions by Masken
 void etfprintf(FILE *file, const char *format, ...);
 void etfprint(FILE *file, const std::string &text);
+void etfprint(FILE *file, const std::wstring &text);
 #define UEFBUFSIZE 2048
 extern char g_uefbuf[UEFBUFSIZE];
 

Common/FileSearch.cpp

@@ -32,6 +32,7 @@
 #include "FileSearch.h"
 
 #include "StringUtils.h"
+#include "util/text/utf8.h"
 
 
 CFileSearch::CFileSearch(const CFileSearch::XStringVector& _rSearchStrings, const CFileSearch::XStringVector& _rDirectories)
@@ -53,7 +54,8 @@ void CFileSearch::FindFiles(const std::string& _searchString, const std::string&
 	BuildCompleteFilename(GCMSearchPath, _strPath, _searchString);
 #ifdef _WIN32
 	WIN32_FIND_DATA findData;
-	HANDLE FindFirst = FindFirstFile(GCMSearchPath.c_str(), &findData);
+	std::wstring searchPathW = ConvertUTF8ToWString(GCMSearchPath);
+	HANDLE FindFirst = FindFirstFile(searchPathW.c_str(), &findData);
 
 	if (FindFirst != INVALID_HANDLE_VALUE)
 	{
@@ -64,7 +66,7 @@ void CFileSearch::FindFiles(const std::string& _searchString, const std::string&
 			if (findData.cFileName[0] != '.')
 			{
 				std::string strFilename;
-				BuildCompleteFilename(strFilename, _strPath, findData.cFileName);
+				BuildCompleteFilename(strFilename, _strPath, ConvertWStringToUTF8(findData.cFileName));
 				m_FileNames.push_back(strFilename);
 			}
 

Common/FileUtil.cpp

@@ -41,6 +41,8 @@
 #include <CoreFoundation/CFBundle.h>
 #endif
 
+#include "util/text/utf8.h"
+
 #include <fstream>
 #include <sys/stat.h>
 
@@ -154,7 +156,7 @@ bool Delete(const std::string &filename)
 	}
 
 #ifdef _WIN32
-	if (!DeleteFile(filename.c_str()))
+	if (!DeleteFile(ConvertUTF8ToWString(filename).c_str()))
 	{
 		WARN_LOG(COMMON, "Delete: DeleteFile failed on %s: %s", 
 				 filename.c_str(), GetLastErrorMsg());
@@ -176,7 +178,7 @@ bool CreateDir(const std::string &path)
 {
 	INFO_LOG(COMMON, "CreateDir: directory %s", path.c_str());
 #ifdef _WIN32
-	if (::CreateDirectory(path.c_str(), NULL))
+	if (::CreateDirectory(ConvertUTF8ToWString(path).c_str(), NULL))
 		return true;
 	DWORD error = GetLastError();
 	if (error == ERROR_ALREADY_EXISTS)
@@ -267,7 +269,7 @@ bool DeleteDir(const std::string &filename)
 	}
 
 #ifdef _WIN32
-	if (::RemoveDirectory(filename.c_str()))
+	if (::RemoveDirectory(ConvertUTF8ToWString(filename).c_str()))
 		return true;
 #else
 	if (rmdir(filename.c_str()) == 0)
@@ -296,7 +298,7 @@ bool Copy(const std::string &srcFilename, const std::string &destFilename)
 	INFO_LOG(COMMON, "Copy: %s --> %s", 
 			srcFilename.c_str(), destFilename.c_str());
 #ifdef _WIN32
-	if (CopyFile(srcFilename.c_str(), destFilename.c_str(), FALSE))
+	if (CopyFile(ConvertUTF8ToWString(srcFilename).c_str(), ConvertUTF8ToWString(destFilename).c_str(), FALSE))
 		return true;
 
 	ERROR_LOG(COMMON, "Copy: failed %s --> %s: %s", 
@@ -461,88 +463,14 @@ bool CreateEmptyFile(const std::string &filename)
 	return true;
 }
 
-
-// Scans the directory tree gets, starting from _Directory and adds the
-// results into parentEntry. Returns the number of files+directories found
-u32 ScanDirectoryTree(const std::string &directory, FSTEntry& parentEntry)
-{
-	INFO_LOG(COMMON, "ScanDirectoryTree: directory %s", directory.c_str());
-	// How many files + directories we found
-	u32 foundEntries = 0;
-#ifdef _WIN32
-	// Find the first file in the directory.
-	WIN32_FIND_DATA ffd;
-
-	HANDLE hFind = FindFirstFile((directory + "\\*").c_str(), &ffd);
-	if (hFind == INVALID_HANDLE_VALUE)
-	{
-		FindClose(hFind);
-		return foundEntries;
-	}
-	// windows loop
-	do
-	{
-		FSTEntry entry;
-		const std::string virtualName(ffd.cFileName);
-#else
-	struct dirent_large { struct dirent entry; char padding[FILENAME_MAX+1]; };
-	struct dirent_large diren;
-	struct dirent *result = NULL;
-
-	DIR *dirp = opendir(directory.c_str());
-	if (!dirp)
-		return 0;
-
-	// non windows loop
-	while (!readdir_r(dirp, (dirent*)&diren, &result) && result)
-	{
-		FSTEntry entry;
-		const std::string virtualName(result->d_name);
-#endif
-		// check for "." and ".."
-		if (((virtualName[0] == '.') && (virtualName[1] == '\0')) ||
-				((virtualName[0] == '.') && (virtualName[1] == '.') && 
-				 (virtualName[2] == '\0')))
-			continue;
-		entry.virtualName = virtualName;
-		entry.physicalName = directory;
-		entry.physicalName += DIR_SEP + entry.virtualName;
-
-		if (IsDirectory(entry.physicalName.c_str()))
-		{
-			entry.isDirectory = true;
-			// is a directory, lets go inside
-			entry.size = ScanDirectoryTree(entry.physicalName, entry);
-			foundEntries += (u32)entry.size;
-		}
-		else
-		{ // is a file 
-			entry.isDirectory = false;
-			entry.size = GetSize(entry.physicalName.c_str());
-		}
-		++foundEntries;
-		// Push into the tree
-		parentEntry.children.push_back(entry);		
-#ifdef _WIN32 
-	} while (FindNextFile(hFind, &ffd) != 0);
-	FindClose(hFind);
-#else
-	}
-	closedir(dirp);
-#endif
-	// Return number of entries found.
-	return foundEntries;
-}
-
-
 // Deletes the given directory and anything under it. Returns true on success.
 bool DeleteDirRecursively(const std::string &directory)
 {
 	INFO_LOG(COMMON, "DeleteDirRecursively: %s", directory.c_str());
 #ifdef _WIN32
 	// Find the first file in the directory.
 	WIN32_FIND_DATA ffd;
-	HANDLE hFind = FindFirstFile((directory + "\\*").c_str(), &ffd);
+	HANDLE hFind = FindFirstFile(ConvertUTF8ToWString(directory + "\\*").c_str(), &ffd);
 
 	if (hFind == INVALID_HANDLE_VALUE)
 	{
@@ -553,7 +481,7 @@ bool DeleteDirRecursively(const std::string &directory)
 	// windows loop
 	do
 	{
-		const std::string virtualName = ffd.cFileName;
+		const std::string virtualName = ConvertWStringToUTF8(ffd.cFileName);
 #else
 	struct dirent dirent, *result = NULL;
 	DIR *dirp = opendir(directory.c_str());

Common/FileUtil.h

@@ -99,10 +99,6 @@ bool Copy(const std::string &srcFilename, const std::string &destFilename);
 // creates an empty file filename, returns true on success 
 bool CreateEmptyFile(const std::string &filename);
 
-// Scans the directory tree gets, starting from _Directory and adds the
-// results into parentEntry. Returns the number of files+directories found
-u32 ScanDirectoryTree(const std::string &directory, FSTEntry& parentEntry);
-
 // deletes the given directory and anything under it. Returns true on success.
 bool DeleteDirRecursively(const std::string &directory);
 

Common/Misc.cpp

@@ -35,7 +35,7 @@ const char* GetLastErrorMsg()
 #ifdef _WIN32
 	static __declspec(thread) char err_str[buff_size] = {};
 
-	FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, NULL, GetLastError(),
+	FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, GetLastError(),
 		MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
 		err_str, buff_size, NULL);
 #else

Common/MsgHandler.cpp

@@ -20,6 +20,7 @@
 
 #include "Common.h" // Local
 #include "StringUtils.h"
+#include "util/text/utf8.h"
 
 bool DefaultMsgHandler(const char* caption, const char* text, bool yes_no, int Style);
 static MsgAlertHandler msg_handler = DefaultMsgHandler;
@@ -106,15 +107,18 @@ bool MsgAlert(bool yes_no, int Style, const char* format, ...)
 bool DefaultMsgHandler(const char* caption, const char* text, bool yes_no, int Style)
 {
 #ifdef _WIN32
-    int STYLE = MB_ICONINFORMATION;
-    if (Style == QUESTION) STYLE = MB_ICONQUESTION;
-    if (Style == WARNING) STYLE = MB_ICONWARNING;
-
-    return IDYES == MessageBox(0, text, caption, STYLE | (yes_no ? MB_YESNO : MB_OK));
-
+	int STYLE = MB_ICONINFORMATION;
+	if (Style == QUESTION) STYLE = MB_ICONQUESTION;
+	if (Style == WARNING) STYLE = MB_ICONWARNING;
+
+	std::wstring wtext = ConvertUTF8ToWString(text);
+	std::wstring wcaption = ConvertUTF8ToWString(caption);
+
+	return IDYES == MessageBox(0, wtext.c_str(), wcaption.c_str(), STYLE | (yes_no ? MB_YESNO : MB_OK));
+
 #else
-    printf("%s\n", text);
-    return true;
+	printf("%s\n", text);
+	return true;
 #endif
 }