From 21906de9504c51fab0d152a72be173976377e9be Mon Sep 17 00:00:00 2001 From: "Yuri (solarw) Turchenkov" Date: Wed, 18 Sep 2024 12:58:28 +0300 Subject: [PATCH] better app close and processes killing --- electron/install.js | 2 +- electron/main.js | 1 + operate/cli.py | 20 +++++++++++++++++++- operate/services/deployment_runner.py | 2 ++ operate/services/service.py | 11 ++++++++++- operate/services/utils/tendermint.py | 12 +++++++++++- package.json | 2 +- pyproject.toml | 2 +- 8 files changed, 46 insertions(+), 6 deletions(-) diff --git a/electron/install.js b/electron/install.js index a117a78b3..634b2d533 100644 --- a/electron/install.js +++ b/electron/install.js @@ -14,7 +14,7 @@ const homedir = os.homedir(); * - use "" (nothing as a suffix) for latest release candidate, for example "0.1.0rc26" * - use "alpha" for alpha release, for example "0.1.0rc26-alpha" */ -const OlasMiddlewareVersion = '0.1.0rc127'; +const OlasMiddlewareVersion = '0.1.0rc139'; const path = require('path'); const { app } = require('electron'); diff --git a/electron/main.js b/electron/main.js index 5216987e2..a6d848fd1 100644 --- a/electron/main.js +++ b/electron/main.js @@ -76,6 +76,7 @@ function showNotification(title, body) { async function beforeQuit() { if (operateDaemonPid) { try { + await fetch(`http://localhost:${appConfig.ports.prod.operate}/stop_all_services`); await killProcesses(operateDaemonPid); } catch (e) { logger.electron(e); diff --git a/operate/cli.py b/operate/cli.py index 2c491ad3b..449fe203e 100644 --- a/operate/cli.py +++ b/operate/cli.py @@ -207,6 +207,10 @@ def cancel_funding_job(service: str) -> None: def pause_all_services_on_startup() -> None: logger.info("Stopping services on startup...") + pause_all_services() + logger.info("Stopping services on startup done.") + + def pause_all_services(): service_hashes = [i["hash"] for i in operate.service_manager().json] for service in service_hashes: @@ -220,7 +224,14 @@ def pause_all_services_on_startup() -> None: logger.info(f"Cancelling funding job for {service}") cancel_funding_job(service=service) health_checker.stop_for_service(service=service) - logger.info("Stopping services on startup done.") + + def pause_all_services_on_exit() -> None: + logger.info("Stopping services on exit...") + pause_all_services() + logger.info("Stopping services on exit done.") + + signal.signal(signal.SIGINT, pause_all_services_on_exit) + signal.signal(signal.SIGTERM, pause_all_services_on_exit) # on backend app started we assume there are now started agents, so we force to pause all pause_all_services_on_startup() @@ -268,6 +279,13 @@ async def _kill_server(request: Request) -> JSONResponse: """Kill backend server from inside.""" os.kill(os.getpid(), signal.SIGINT) + @app.get(f"/stop_all_services") + async def _kill_server(request: Request) -> JSONResponse: + """Kill backend server from inside.""" + logger.info("Stopping services on demand...") + pause_all_services() + logger.info("Stopping services on demand done.") + @app.get("/api") @with_retries async def _get_api(request: Request) -> JSONResponse: diff --git a/operate/services/deployment_runner.py b/operate/services/deployment_runner.py index b53f9b12e..ebde6527b 100644 --- a/operate/services/deployment_runner.py +++ b/operate/services/deployment_runner.py @@ -78,6 +78,8 @@ def kill_process(pid: int) -> None: children = list(reversed(current_process.children(recursive=True))) for child in children: _kill_process(child.pid) + _kill_process(child.pid) + _kill_process(pid) _kill_process(pid) diff --git a/operate/services/service.py b/operate/services/service.py index ab2d26315..8295962af 100644 --- a/operate/services/service.py +++ b/operate/services/service.py @@ -25,6 +25,8 @@ import shutil import subprocess # nosec import sys +from time import sleep +from traceback import print_exc import typing as t from copy import copy, deepcopy from dataclasses import dataclass @@ -504,7 +506,14 @@ def _build_host(self, force: bool = True, chain_id: str = "100") -> None: if build.exists() and force: stop_host_deployment(build_dir=build) - shutil.rmtree(build) + try: + sleep(3) + shutil.rmtree(build) + except: + # sleep and try again. exception if fails + print_exc() + sleep(3) + shutil.rmtree(build) service = Service.load(path=self.path) if service.helper.config.number_of_agents > 1: diff --git a/operate/services/utils/tendermint.py b/operate/services/utils/tendermint.py index 9fa90d439..fadcc6b1d 100644 --- a/operate/services/utils/tendermint.py +++ b/operate/services/utils/tendermint.py @@ -187,6 +187,14 @@ def __init__( self.logger = logger or logging.getLogger() self.log_file = os.environ.get("LOG_FILE", DEFAULT_TENDERMINT_LOG_FILE) self.write_to_log = write_to_log + + def _set_on_stop_handler(self): + signal.signal(signal.SIGINT, self._stop_signal_handler) + signal.signal(signal.SIGTERM, self._stop_signal_handler) + + def _stop_signal_handler(self): + self.logger.info("Handling stop signal!") + self.stop() def _build_init_command(self) -> List[str]: """Build the 'init' command.""" @@ -258,6 +266,7 @@ def _start_monitoring_thread(self) -> None: def start(self, debug: bool = False) -> None: """Start a Tendermint node process.""" + self._set_on_stop_handler() self._start_tm_process(debug) self._start_monitoring_thread() @@ -310,8 +319,9 @@ def _stop_monitoring_thread(self) -> None: def stop(self) -> None: """Stop a Tendermint node process.""" - self._stop_tm_process() self._stop_monitoring_thread() + self._stop_tm_process() + @staticmethod def _write_to_console(line: str) -> None: diff --git a/package.json b/package.json index 923160d70..4ba12db74 100644 --- a/package.json +++ b/package.json @@ -58,5 +58,5 @@ "download-binaries": "sh download_binaries.sh", "build:pearl": "sh build_pearl.sh" }, - "version": "0.1.0-rc127" + "version": "0.1.0-rc139" } \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index aa2516ec9..8238403b2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "olas-operate-middleware" -version = "0.1.0-rc127" +version = "0.1.0-rc139" description = "" authors = ["David Vilela ", "Viraj Patel "] readme = "README.md"