Skip to content

Commit

Permalink
Fix deadlock during desync by moving into render thread
Browse files Browse the repository at this point in the history
  • Loading branch information
@IonAgorria
IonAgorria committed May 24, 2024
1 parent a274c77 commit ca7cc43
Show file tree
Hide file tree
Showing 7 changed files with 92 additions and 73 deletions.
20 changes: 12 additions & 8 deletions Source/Network/CommonEvents.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -490,25 +490,29 @@ struct netCommand4C_DisplayDistrincAreas : public netCommandGeneral {
unsigned char* pDAData_;
};

struct DesyncNotify {
int desync_amount = 0;
std::string gameID = {};
};

///Sent from server when client is detected to be desynced
struct netCommand4C_DesyncNotify : public netCommandGeneral {
public:
int desync_amount;
std::string gameID;
DesyncNotify data = {};

netCommand4C_DesyncNotify(const std::string& gameID_) : netCommandGeneral(NETCOM_4C_ID_DESYNC_NOTIFY) {
desync_amount = 0;
gameID = gameID_;
data.desync_amount = 0;
data.gameID = gameID_;
}

netCommand4C_DesyncNotify(XBuffer& in) : netCommandGeneral(NETCOM_4C_ID_DESYNC_NOTIFY) {
in > desync_amount;
in > StringInWrapper(gameID);
in > data.desync_amount;
in > StringInWrapper(data.gameID);
}

void Write(XBuffer& out) const override {
out < desync_amount;
out < StringOutWrapper(gameID);
out < data.desync_amount;
out < StringOutWrapper(data.gameID);
};
};

Expand Down
58 changes: 3 additions & 55 deletions Source/Network/P2P_interface1Th.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -416,66 +416,14 @@ void PNetCenter::HandlerInputNetCommand()
fprintf(stderr, "NETCOM_4C_ID_DESYNC_NOTIFY\n");


if (nc.desync_amount > PNC_DESYNC_RESTORE_ATTEMPTS) {
if (nc.data.desync_amount > PNC_DESYNC_RESTORE_ATTEMPTS) {
ExecuteInterfaceCommand(PNC_INTERFACE_COMMAND_DESYNC);
} else {
} else {
if (!isHost()) {
ExecuteInternalCommand(PNC_COMMAND__DESYNC, false);
}

std::string crash_dir = CRASH_DIR;
terminate_with_char(crash_dir, PATH_SEP);
crash_dir += "desync_" + nc.gameID + "_" + std::to_string(m_localNETID) + PATH_SEP;
create_directories(crash_dir);

//Write net log
XBuffer netlog(2048, true);
netlog < currentVersion < "\r\n";
netlog < "ArchFlags: " <= computeArchFlags();
netlog < " HostNETID: " <= m_hostNETID;
netlog < " LocalNETID: " <= m_localNETID;
netlog < " Amount: " <= nc.desync_amount;
netlog < "\r\n";
universe()->writeLogList2Buffer(netlog);
XStream f(crash_dir + "netlog.txt", XS_OUT);
f.write(netlog.address(), netlog.tell());
f.close();
universe()->clearLogList();

//Attempt to save state
gameShell->savePrm().manualData.clearSoundTracks(); //Avoid host overriding client soundtracks
std::unique_ptr<MissionDescription> md = std::make_unique<MissionDescription>();
gameShell->universalSave((crash_dir + "save").c_str(), true, md.get());
md->PrintInfo();

//Attempt to save reel
universe()->savePlayReel((crash_dir + "reel").c_str());

fprintf(stderr, "%d Error network synchronization, dumped at: %s\n", clocki(), crash_dir.c_str());

std::unique_ptr<LocalizedText> text = std::make_unique<LocalizedText>(
qdTextDB::instance().getText("Interface.Menu.Messages.Multiplayer.Nonsinchronization"),
getLocale()
);
text->text += " " + std::to_string(nc.desync_amount);
ExecuteInterfaceCommand(
PNC_INTERFACE_COMMAND_INFO_MESSAGE,
std::move(text)
);

//Do not send binary and script data to host except host itself
//Also trim some data in partial mode
if (m_localNETID != m_hostNETID || nc.desync_amount < PNC_DESYNC_RESTORE_MODE_FULL) {
md->binaryData.alloc(0);
md->scriptsData.alloc(0);
}

md->setSaveName("");

//Send the ack
netCommand4H_DesyncAcknowledge ack(std::move(md));
std::swap(ack.netlog, netlog);
SendEventSync(&ack);
gameShell->MultiplayerGameDesyncNotify(nc.data);
}
break;
}
Expand Down
6 changes: 3 additions & 3 deletions Source/Network/P2P_interface2Th.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -626,7 +626,7 @@ void PNetCenter::LLogicQuant()
client->desync_state = PNC_DESYNC_RESTORE_FAILED;
std::string gameID = std::to_string(time(nullptr)) + "_failed";
netCommand4C_DesyncNotify ev_notify = netCommand4C_DesyncNotify(gameID);
ev_notify.desync_amount = client->desync_amount;
ev_notify.data.desync_amount = client->desync_amount;
SendEvent(ev_notify, client->netidPlayer);
fprintf(stderr, "Failed to recover network synchronization with 0x%" PRIX64 " after %d times\n", client->netidPlayer, client->desync_amount);
} else {
Expand Down Expand Up @@ -882,9 +882,9 @@ end_while_01:;
netCommand4C_DesyncNotify ev_notify = netCommand4C_DesyncNotify(gameID);
for (auto& client : to_notify) {
if (client->netidPlayer == m_hostNETID) {
ev_notify.desync_amount = highest_amount;
ev_notify.data.desync_amount = highest_amount;
} else {
ev_notify.desync_amount = client->desync_amount;
ev_notify.data.desync_amount = client->desync_amount;
}
SendEvent(ev_notify, client->netidPlayer);

Expand Down
1 change: 0 additions & 1 deletion Source/Network/P2P_interface2Th_Host.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -466,7 +466,6 @@ void PNetCenter::HostReceiveQuant()
*/
{
fprintf(stderr, "Invalid netCommand %d to host from non-client NETID 0x%" PRIX64 "\n", currentCMD, netid);
xassert(0);
in_HostBuf.ignoreNetCommand();
}

Expand Down
2 changes: 1 addition & 1 deletion Source/UserInterface/GameShell.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2667,7 +2667,7 @@ void GameShell::updateResolution(bool change_depth, bool change_size, bool chang
}
}

void GameShell::serverMessage(LocalizedText* text) {
void GameShell::serverMessage(const LocalizedText* text) {
_shellIconManager.showHintChat(text, 5000);
}

Expand Down
11 changes: 6 additions & 5 deletions Source/UserInterface/GameShell.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,10 +44,6 @@ class GameShell
void startCmdline(const CommandLineData& data);
void switchToInitialMenu();

void MultiplayerGameStarting();
void MultiplayerGameStart(const MissionDescription& mission);
void MultiplayerGameRestore(const MissionDescription& mission);

bool universalSave(const char* name, bool userSave, MissionDescription* missionOutput = nullptr);
SavePrm& savePrm() { return savePrm_; }
const SaveManualData& manualData() { return savePrm_.manualData; }
Expand Down Expand Up @@ -260,7 +256,7 @@ class GameShell
void showConnectFailedInGame(const std::string& playerList);
void hideConnectFailedInGame(bool connectionRestored = true);

void serverMessage(LocalizedText* message);
void serverMessage(const LocalizedText* message);

enum GeneralErrorType {
GENERAL_CONNECTION_FAILED,
Expand All @@ -275,6 +271,11 @@ class GameShell
void updateLatencyInfo(const NetLatencyInfo& info, const MissionDescription* md);

void addStringToChatWindow(bool clanOnly, const std::string& newString, const std::string& locale);

void MultiplayerGameStarting();
void MultiplayerGameStart(const MissionDescription& mission);
void MultiplayerGameRestore(const MissionDescription& mission);
void MultiplayerGameDesyncNotify(DesyncNotify& nc);

//-----end of network function----

Expand Down
67 changes: 67 additions & 0 deletions Source/UserInterface/Menu/MultiplayerCommon.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -162,6 +162,73 @@ void GameShell::MultiplayerGameRestore(const MissionDescription& mission) {
_shellIconManager.AddDynamicHandler(SwitchMultiplayerToRestoreQuant, CBCODE_QUANT);
}

DesyncNotify lastDesyncNotify;

int multiplayerGameDesyncNotifyQuant(float,float) {
MTAutoSingleThread auto_single;
PNetCenter* net = gameShell->getNetClient();
if (lastDesyncNotify.gameID.empty()) {
return 0;
}

LocalizedText text(
qdTextDB::instance().getText("Interface.Menu.Messages.Multiplayer.Nonsinchronization"),
getLocale()
);
text.text += " " + std::to_string(lastDesyncNotify.desync_amount);
gameShell->serverMessage(&text);

std::string crash_dir = CRASH_DIR;
terminate_with_char(crash_dir, PATH_SEP);
crash_dir += "desync_" + lastDesyncNotify.gameID + "_" + std::to_string(net->m_localNETID) + PATH_SEP;
create_directories(crash_dir);

//Write net log
XBuffer netlog(2048, true);
netlog < currentVersion < "\r\n";
netlog < "ArchFlags: " <= computeArchFlags();
netlog < " HostNETID: " <= net->m_hostNETID;
netlog < " LocalNETID: " <= net->m_localNETID;
netlog < " Amount: " <= lastDesyncNotify.desync_amount;
netlog < "\r\n";
universe()->writeLogList2Buffer(netlog);
XStream f(crash_dir + "netlog.txt", XS_OUT);
f.write(netlog.address(), netlog.tell());
f.close();
universe()->clearLogList();

//Attempt to save state
gameShell->savePrm().manualData.clearSoundTracks(); //Avoid host overriding client soundtracks
std::unique_ptr<MissionDescription> md = std::make_unique<MissionDescription>();
gameShell->universalSave((crash_dir + "save").c_str(), true, md.get());
md->PrintInfo();

//Attempt to save reel
universe()->savePlayReel((crash_dir + "reel").c_str());

fprintf(stderr, "%d Error network synchronization, dumped at: %s\n", clocki(), crash_dir.c_str());

//Do not send binary and script data to host except host itself
//Also trim some data in partial mode
if (net->m_localNETID != net->m_hostNETID || lastDesyncNotify.desync_amount < PNC_DESYNC_RESTORE_MODE_FULL) {
md->binaryData.alloc(0);
md->scriptsData.alloc(0);
}

md->setSaveName("");

//Send the ack
netCommand4H_DesyncAcknowledge ack(std::move(md));
std::swap(ack.netlog, netlog);
net->SendEventSync(&ack);
return 0;
}

void GameShell::MultiplayerGameDesyncNotify(DesyncNotify& nc) {
lastDesyncNotify = nc;
_shellIconManager.AddDynamicHandler(multiplayerGameDesyncNotifyQuant, CBCODE_QUANT);
}

//////////show/hide message ingame////////////////////

std::string messageBoxText;
Expand Down

0 comments on commit ca7cc43

Please sign in to comment.