better app close and processes killing

electron/install.js

@@ -14,7 +14,7 @@ const homedir = os.homedir();
  * - use "" (nothing as a suffix) for latest release candidate, for example "0.1.0rc26"
  * - use "alpha" for alpha release, for example "0.1.0rc26-alpha"
  */
-const OlasMiddlewareVersion = '0.1.0rc127';
+const OlasMiddlewareVersion = '0.1.0rc139';
 
 const path = require('path');
 const { app } = require('electron');

electron/main.js

@@ -76,6 +76,7 @@ function showNotification(title, body) {
 async function beforeQuit() {
   if (operateDaemonPid) {
     try {
+      await fetch(`http://localhost:${appConfig.ports.prod.operate}/stop_all_services`);
       await killProcesses(operateDaemonPid);
     } catch (e) {
       logger.electron(e);

operate/cli.py

@@ -207,6 +207,10 @@ def cancel_funding_job(service: str) -> None:
 
     def pause_all_services_on_startup() -> None:
         logger.info("Stopping services on startup...")
+        pause_all_services()
+        logger.info("Stopping services on startup done.")
+
+    def pause_all_services():
         service_hashes = [i["hash"] for i in operate.service_manager().json]
 
         for service in service_hashes:
@@ -220,7 +224,14 @@ def pause_all_services_on_startup() -> None:
             logger.info(f"Cancelling funding job for {service}")
             cancel_funding_job(service=service)
             health_checker.stop_for_service(service=service)
-        logger.info("Stopping services on startup done.")
+
+    def pause_all_services_on_exit() -> None:
+        logger.info("Stopping services on exit...")
+        pause_all_services()
+        logger.info("Stopping services on exit done.")
+
+    signal.signal(signal.SIGINT, pause_all_services_on_exit)
+    signal.signal(signal.SIGTERM, pause_all_services_on_exit)
 
     # on backend app started we assume there are now started agents, so we force to pause all
     pause_all_services_on_startup()
@@ -268,6 +279,13 @@ async def _kill_server(request: Request) -> JSONResponse:
         """Kill backend server from inside."""
         os.kill(os.getpid(), signal.SIGINT)
 
+    @app.get(f"/stop_all_services")
+    async def _kill_server(request: Request) -> JSONResponse:
+        """Kill backend server from inside."""
+        logger.info("Stopping services on demand...")
+        pause_all_services()
+        logger.info("Stopping services on demand done.")
+
     @app.get("/api")
     @with_retries
     async def _get_api(request: Request) -> JSONResponse:

operate/services/deployment_runner.py

@@ -78,6 +78,8 @@ def kill_process(pid: int) -> None:
     children = list(reversed(current_process.children(recursive=True)))
     for child in children:
         _kill_process(child.pid)
+        _kill_process(child.pid)
+    _kill_process(pid)
     _kill_process(pid)
 
 

operate/services/service.py

@@ -25,6 +25,8 @@
 import shutil
 import subprocess  # nosec
 import sys
+from time import sleep
+from traceback import print_exc
 import typing as t
 from copy import copy, deepcopy
 from dataclasses import dataclass
@@ -504,7 +506,14 @@ def _build_host(self, force: bool = True, chain_id: str = "100") -> None:
 
         if build.exists() and force:
             stop_host_deployment(build_dir=build)
-            shutil.rmtree(build)
+            try:
+                sleep(3)
+                shutil.rmtree(build)
+            except:
+                # sleep and try again. exception if fails
+                print_exc()
+                sleep(3)
+                shutil.rmtree(build)
 
         service = Service.load(path=self.path)
         if service.helper.config.number_of_agents > 1:

operate/services/utils/tendermint.py

@@ -187,6 +187,14 @@ def __init__(
         self.logger = logger or logging.getLogger()
         self.log_file = os.environ.get("LOG_FILE", DEFAULT_TENDERMINT_LOG_FILE)
         self.write_to_log = write_to_log
+
+    def _set_on_stop_handler(self):
+        signal.signal(signal.SIGINT, self._stop_signal_handler)
+        signal.signal(signal.SIGTERM, self._stop_signal_handler)
+
+    def _stop_signal_handler(self):
+        self.logger.info("Handling stop signal!")
+        self.stop()
 
     def _build_init_command(self) -> List[str]:
         """Build the 'init' command."""
@@ -258,6 +266,7 @@ def _start_monitoring_thread(self) -> None:
 
     def start(self, debug: bool = False) -> None:
         """Start a Tendermint node process."""
+        self._set_on_stop_handler()
         self._start_tm_process(debug)
         self._start_monitoring_thread()
 
@@ -310,8 +319,9 @@ def _stop_monitoring_thread(self) -> None:
 
     def stop(self) -> None:
         """Stop a Tendermint node process."""
-        self._stop_tm_process()
         self._stop_monitoring_thread()
+        self._stop_tm_process()
+
 
     @staticmethod
     def _write_to_console(line: str) -> None:

package.json

@@ -58,5 +58,5 @@
     "download-binaries": "sh download_binaries.sh",
     "build:pearl": "sh build_pearl.sh"
   },
-  "version": "0.1.0-rc127"
+  "version": "0.1.0-rc139"
 }

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "olas-operate-middleware"
-version = "0.1.0-rc127"
+version = "0.1.0-rc139"
 description = ""
 authors = ["David Vilela <[email protected]>", "Viraj Patel <[email protected]>"]
 readme = "README.md"