feat: health check

internxt · Oct 17, 2023 · bd5f34e · bd5f34e
1 parent 2fbca82
commit bd5f34e
Show file tree

Hide file tree

Showing 2 changed files with 55 additions and 0 deletions.
diff --git a/src/main/background-processes/sync-engine.ts b/src/main/background-processes/sync-engine.ts
@@ -2,10 +2,56 @@ import { BrowserWindow, ipcMain } from 'electron';
 import path from 'path';
 import Logger from 'electron-log';
 import eventBus from '../event-bus';
+import nodeSchedule from 'node-schedule';
 
 let worker: BrowserWindow | null = null;
 let workerIsRunning = false;
 let startingWorker = false;
+let healthCheckSchedule: nodeSchedule.Job | null = null;
+
+async function healthCheck() {
+  const responsePromise = new Promise<void>((resolve, reject) => {
+    ipcMain.once('SYNC_ENGINE:PONG', () => {
+      resolve();
+    });
+
+    const millisecondsToWait = 2_000;
+
+    setTimeout(() => {
+      reject(
+        new Error(
+          `Health check failed after ${millisecondsToWait} milliseconds`
+        )
+      );
+    }, millisecondsToWait);
+  });
+
+  worker?.webContents.send('SYNC_ENGINE:PING');
+
+  await responsePromise;
+}
+
+function scheduleHeathCheck() {
+  if (healthCheckSchedule) {
+    healthCheckSchedule.cancel(false);
+  }
+
+  const relaunchOnFail = () =>
+    healthCheck()
+      .then(() => {
+        // Logger.debug('Health check succeeded');
+      })
+      .catch(() => {
+        Logger.warn('Health check failed, relaunching the worker');
+        workerIsRunning = false;
+        worker?.destroy();
+        spawnSyncEngineWorker();
+      });
+
+  healthCheckSchedule = nodeSchedule.scheduleJob('*/30 * * * * *', async () => {
+    await relaunchOnFail();
+  });
+}
 
 function spawnSyncEngineWorker() {
   if (startingWorker) {
@@ -35,6 +81,7 @@ function spawnSyncEngineWorker() {
     )
     .then(() => {
       Logger.info('[MAIN] Sync engine worker loaded');
+      scheduleHeathCheck();
     })
     .catch((err) => {
       Logger.error('[MAIN] Error loading sync engine worker', err);
@@ -66,6 +113,8 @@ function spawnSyncEngineWorker() {
 export async function stopSyncEngineWatcher() {
   Logger.info('[MAIN] STOPPING SYNC ENGINE WORKER...');
 
+  healthCheckSchedule?.cancel(false);
+
   if (!workerIsRunning) {
     Logger.info('[MAIN] WORKER WAS NOT RUNNING');
     worker?.destroy();
@@ -112,6 +161,8 @@ export async function stopSyncEngineWatcher() {
 async function stopAndClearSyncEngineWatcher() {
   Logger.info('[MAIN] STOPPING AND CLEAR SYNC ENGINE WORKER...');
 
+  healthCheckSchedule?.cancel(false);
+
   if (!workerIsRunning) {
     Logger.info('[MAIN] WORKER WAS NOT RUNNING');
     worker?.destroy();

diff --git a/src/workers/sync-engine/index.ts b/src/workers/sync-engine/index.ts
@@ -68,6 +68,10 @@ async function setUp() {
     }
   });
 
+  ipcRenderer.on('SYNC_ENGINE:PING', (event) => {
+    event.sender.send('SYNC_ENGINE:PONG');
+  });
+
   await bindings.start(
     packageJson.version,
     '{E9D7EB38-B229-5DC5-9396-017C449D59CD}'