From bf114b889ac675829e95f4b61ad426b7b0cd7c7c Mon Sep 17 00:00:00 2001 From: bbrzyski Date: Thu, 21 Nov 2024 10:27:39 +0100 Subject: [PATCH] Add command for starting the KPM server and client Signed-off-by: bbrzyski --- CHANGELOG.md | 1 + docs/source/getting_started.md | 12 +- topwrap/cli.py | 206 ++++++++++++++++++++++++++++----- topwrap/kpm_topwrap_client.py | 8 +- 4 files changed, 188 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ef71c64..9ef9f11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added "IP Cores", "Externals", and "Constants" layers to the GUI which you can hide/show from the settings - Nox session for downloading and packaging FuseSoc libraries - Support Python 3.13 and dropped support for Python 3.8 +- All in one command `topwrap gui` for building, starting the KPM server and connecting the client to it. ### Changed diff --git a/docs/source/getting_started.md b/docs/source/getting_started.md index de73a6a..89176ef 100644 --- a/docs/source/getting_started.md +++ b/docs/source/getting_started.md @@ -42,17 +42,11 @@ Topwrap will generate two files `gen_simple_core_1.yaml` and `gen_simple_core_2. Generated ip core yamls can be loaded into GUI. -1. Build and run gui server +Run the GUI client: ```bash -topwrap kpm_build_server && topwrap kpm_run_server & +topwrap gui gen_simple_core_1.yaml gen_simple_core_2.yaml ``` - -2. Run gui client with the generated ip core yamls -```bash -topwrap kpm_client gen_simple_core_1.yaml gen_simple_core_2.yaml -``` - -Now when you connect to [http://127.0.0.1:5000](http://127.0.0.1:5000) there should be kpm gui. +It should build and start server, connect the client to it and open the browser with GUI. Loaded ip cores can be found under IPcore section: diff --git a/topwrap/cli.py b/topwrap/cli.py index db3a0a9..6591f44 100644 --- a/topwrap/cli.py +++ b/topwrap/cli.py @@ -2,11 +2,16 @@ # SPDX-License-Identifier: Apache-2.0 import asyncio +import concurrent.futures import json import logging +import os import subprocess +import sys +import threading +import webbrowser from pathlib import Path -from typing import Optional, Tuple +from typing import Any, Optional, Tuple import click @@ -162,15 +167,80 @@ def parse_main( logging.info(f"VHDL Module '{vhdl_mod.module_name}'" f"saved in file '{yaml_path}'") -DEFAULT_WORKSPACE_DIR = Path("build", "workspace") -DEFAULT_BACKEND_DIR = Path("build", "backend") -DEFAULT_FRONTEND_DIR = Path("build", "frontend") +DEFAULT_SERVER_BASE_DIR = ( + Path(os.environ.get("XDG_CACHE_HOME", "~/.local/cache")).expanduser() / "topwrap/kpm_build" +) +DEFAULT_WORKSPACE_DIR = DEFAULT_SERVER_BASE_DIR / "workspace" +DEFAULT_BACKEND_DIR = DEFAULT_SERVER_BASE_DIR / "backend" +DEFAULT_FRONTEND_DIR = DEFAULT_SERVER_BASE_DIR / "frontend" DEFAULT_SERVER_ADDR = "127.0.0.1" DEFAULT_SERVER_PORT = 9000 DEFAULT_BACKEND_ADDR = "127.0.0.1" DEFAULT_BACKEND_PORT = 5000 +class KPM: + @staticmethod + def build_server(**params_dict: Any): + args = ["pipeline_manager", "build", "server-app"] + for k, v in params_dict.items(): + Path(v).mkdir(exist_ok=True, parents=True) + args += [f"--{k}".replace("_", "-"), f"{v}"] + subprocess.check_call(args) + + @staticmethod + def run_server( + server_ready_event: Optional[threading.Event] = None, + server_init_failed: Optional[threading.Event] = None, + show_kpm_logs: bool = True, + **params_dict: Any, + ): + args = ["pipeline_manager", "run"] + for k, v in params_dict.items(): + args += [f"--{k}".replace("_", "-"), f"{v}"] + + server_process = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + server_ready_string = "Uvicorn running on" + while server_process.poll() is None and server_process.stdout is not None: + server_logs = server_process.stdout.readline().decode("utf-8") + if server_ready_event is not None and server_ready_string in server_logs: + server_ready_event.set() + if show_kpm_logs: + sys.stdout.write(server_logs) + else: + logging.warning("KPM server has been terminated") + if server_ready_event is not None and not server_ready_event.isSet(): + if server_init_failed is not None: + server_init_failed.set() + # Remove the server ready event block + server_ready_event.set() + logging.warning( + "Make sure that there isn't any instance of pipeline manager running in the background" + ) + + @staticmethod + def run_client( + host: str, + port: int, + log_level: str, + design: Optional[Path], + yamlfiles: Tuple[Path, ...], + build_dir: Path, + client_ready_event: Optional[threading.Event] = None, + ): + configure_log_level(log_level) + logging.info("Starting kenning pipeline manager client") + config_user_repo = UserRepo() + config_user_repo.load_repositories_from_paths(config.get_repositories_paths()) + extended_yamlfiles = config_user_repo.get_core_designs() + extended_yamlfiles += yamlfiles + asyncio.run( + kpm_run_client( + RPCparams(host, port, extended_yamlfiles, build_dir, design), client_ready_event + ) + ) + + @main.command("kpm_client", help="Run a client app, that connects to a running KPM server") @click.option("--host", "-h", default=DEFAULT_SERVER_ADDR, help="KPM server address") @click.option("--port", "-p", default=DEFAULT_SERVER_PORT, help="KPM server listening port") @@ -197,18 +267,7 @@ def kpm_client_main( yamlfiles: Tuple[Path, ...], build_dir: Path, ): - configure_log_level(log_level) - - logging.info("Starting kenning pipeline manager client") - config_user_repo = UserRepo() - config_user_repo.load_repositories_from_paths(config.get_repositories_paths()) - extended_yamlfiles = config_user_repo.get_core_designs() - extended_yamlfiles += yamlfiles - - loop = asyncio.get_event_loop() - loop.run_until_complete( - kpm_run_client(RPCparams(host, port, extended_yamlfiles, build_dir, design)) - ) + KPM.run_client(host, port, log_level, design, yamlfiles, build_dir) @main.command("kpm_build_server", help="Build KPM server") @@ -225,13 +284,8 @@ def kpm_client_main( help="Directory where the built frontend should be stored", ) @click.pass_context -def kpm_build_server(ctx: click.Context, workspace_directory: Path, output_directory: Path): - workspace_directory.mkdir(exist_ok=True, parents=True) - output_directory.mkdir(exist_ok=True, parents=True) - args = ["pipeline_manager", "build", "server-app"] - for k, v in ctx.params.items(): - args += [f"--{k}".replace("_", "-"), f"{v}"] - subprocess.check_call(args) +def kpm_build_server_ctx(ctx: click.Context, **_): + KPM.build_server(**ctx.params) @main.command("kpm_run_server", help="Run a KPM server") @@ -259,12 +313,108 @@ def kpm_build_server(ctx: click.Context, workspace_directory: Path, output_direc default=DEFAULT_BACKEND_PORT, help="The port of the backend of Pipeline Manager", ) +@click.option("--verbosity", default="INFO", help="Verbosity level for KPM server logs") @click.pass_context -def kpm_run_server(ctx: click.Context, **_): - args = ["pipeline_manager", "run"] - for k, v in ctx.params.items(): - args += [f"--{k}".replace("_", "-"), f"{v}"] - subprocess.check_call(args) +def kpm_run_server_ctx(ctx: click.Context, **_): + KPM.run_server(**ctx.params) + + +@main.command("gui", help="Start GUI") +@click.option( + "--server-host", + default=DEFAULT_SERVER_ADDR, + help="The address of the Pipeline Manager TCP Server", +) +@click.option( + "--server-port", default=DEFAULT_SERVER_PORT, help="The port of the Pipeline Manager TCP Server" +) +@click.option( + "--backend-host", + default=DEFAULT_BACKEND_ADDR, + help="The address of the backend of Pipeline Manager", +) +@click.option( + "--backend-port", + default=DEFAULT_BACKEND_PORT, + help="The port of the backend of Pipeline Manager", +) +@click.option( + "--design", + "-d", + type=click_r_file, + help="Specify design file to load initially", +) +@click.option( + "--frontend-directory", + type=click_opt_rw_dir, + default=DEFAULT_FRONTEND_DIR, + help="Location of the built frontend", +) +@click.option( + "--workspace-directory", + type=click_opt_rw_dir, + default=DEFAULT_WORKSPACE_DIR, + help="Directory where the frontend sources should be stored", +) +@click.option("--log-level", default="INFO", help="Log level") +@click.argument("yamlfiles", type=click_r_file, nargs=-1) +def topwrap_gui( + design: Optional[Path], + log_level: str, + yamlfiles: Tuple[Path, ...], + frontend_directory: Path, + workspace_directory: Path, + server_host: str, + server_port: int, + backend_host: str, + backend_port: int, +): + configure_log_level(log_level) + logging.info("Checking if server is built") + if not frontend_directory.exists() or not workspace_directory.exists(): + logging.info("Server build is incomplete, building now") + KPM.build_server( + workspace_directory=workspace_directory, output_directory=frontend_directory + ) + else: + logging.info("Server build found") + + with concurrent.futures.ThreadPoolExecutor() as executor: + logging.info("Starting server") + server_ready_event = threading.Event() + server_init_failed = threading.Event() + + executor.submit( + KPM.run_server, + server_ready_event=server_ready_event, + show_kpm_logs=False, + server_init_failed=server_init_failed, + server_host=server_host, + server_port=server_port, + backend_host=backend_host, + backend_port=backend_port, + frontend_directory=frontend_directory, + ) + logging.info("Waiting for KPM server to initialize") + server_ready_event.wait() + if server_init_failed.isSet(): + logging.error("KPM server failed to initialize. Aborting") + return + logging.info("KPM server initialized") + client_ready_event = threading.Event() + executor.submit( + KPM.run_client, + design=design, + yamlfiles=yamlfiles, + host=server_host, + port=server_port, + log_level=log_level, + build_dir=Path("build"), + client_ready_event=client_ready_event, + ) + client_ready_event.wait() + logging.info("Opening browser with KPM GUI") + webbrowser.open(f"{backend_host}:{backend_port}") @main.command("specification", help="Generate KPM specification from IP core YAMLs") diff --git a/topwrap/kpm_topwrap_client.py b/topwrap/kpm_topwrap_client.py index 8a0df9c..f0444ed 100644 --- a/topwrap/kpm_topwrap_client.py +++ b/topwrap/kpm_topwrap_client.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 import logging +import threading from base64 import b64encode from datetime import datetime from pathlib import Path @@ -161,10 +162,13 @@ def _kpm_export_handler(dataflow: JsonType, yamlfiles: List[Path]) -> Tuple[str, return (design.to_yaml(), filename) -async def kpm_run_client(rpc_params: RPCparams): +async def kpm_run_client( + rpc_params: RPCparams, client_ready_event: Optional[threading.Event] = None +): client = CommunicationBackend(rpc_params.host, rpc_params.port) logging.debug("Initializing RPC client") await client.initialize_client(RPCMethods(rpc_params, client)) - + if client_ready_event is not None: + client_ready_event.set() logging.debug("starting json rpc client") await client.start_json_rpc_client()