From 7df601d9dee9707813c7f184386f27ce6afb63e1 Mon Sep 17 00:00:00 2001 From: jpkanter Date: Wed, 14 Jul 2021 00:03:21 +0200 Subject: [PATCH] added continue option to cli interface, tested it locally (and it works on my machineTM). Modiefied Spcht Project Exceptions a bit --- SpchtErrors.py | 46 ++++++++++++++----------------------- argparse.json | 6 +++++ local_tools.py | 62 +++++++++++++++++++++++++++++++++++--------------- main.py | 40 +++++++++++++++++++++++++++++--- 4 files changed, 104 insertions(+), 50 deletions(-) diff --git a/SpchtErrors.py b/SpchtErrors.py index 84a078d..a218842 100644 --- a/SpchtErrors.py +++ b/SpchtErrors.py @@ -29,46 +29,34 @@ """ -class WorkOrderInconsitencyError(TypeError): - """ - Work order makes no sense in a logical sense - """ - pass +class WorkOrderInconsitencyError(Exception): + def __repr__(self): + return "A change is inconsistent with the logic of a work order, like updating a status to a lower level than the previos one" -class WorkOrderError(TypeError): - """ - Generic error with the given work order - """ - pass +class WorkOrderError(Exception): + def __repr__(self): + return "Generic error with the given work order" -class WorkOrderTypeError(TypeError): - """ - For incorrect file types in work order parameters - """ - pass +class WorkOrderTypeError(Exception): + def __repr__(self): + return "For incorrect file types in work order parameters" -class ParameterError(KeyError): - """ - The given parameter lead to an outcome that did not work - """ - pass +class ParameterError(Exception): + def __repr__(self): + return "The given parameter lead to an outcome that did not work" -class OperationalError(TypeError): - """ - Something that stops the overall operation from proceeding - """ - pass +class OperationalError(Exception): + def __repr__(self): + return "Something that stops the overall operation from proceeding" class RequestError(ConnectionError): - """ - For requests that might fail for this or that reason within the bellows of the script - """ - pass + def __repr__(self): + return "For requests that might fail for this or that reason within the bellows of the script" class ParsingError(Exception): diff --git a/argparse.json b/argparse.json index c6d9eda..019b65d 100644 --- a/argparse.json +++ b/argparse.json @@ -242,5 +242,11 @@ "help": "Tries to load and validate the specified Spcht JSON File", "type": "str", "metavar": ["SPCHT FILE"] + }, + "ContinueOrder": + { + "help": "Continues a previously paused or interrupted work order, needs parameters", + "type": "str", + "metavar": ["WORK ORDER FILE"] } } diff --git a/local_tools.py b/local_tools.py index fc5afaf..073a607 100644 --- a/local_tools.py +++ b/local_tools.py @@ -48,6 +48,17 @@ # describes structure of the json response from solr Version 7.3.1 holding the ubl data +WORK_ORDER_STATUS = ("Freshly created", # * 0 + "fetch started", # * 1 + "fetch completed", # * 2 + "processing started", # * 3 + "processing completed", # * 4 + "intermediate process started", # * 5 + "intermediate process finished", # * 6 + "inserting started", # * 7 + "insert completed/finished", # * 8 + "fullfilled") # * 9 + def slice_header_json(data): STRUCTURE = { @@ -353,15 +364,18 @@ def UpdateWorkOrder(file_path: str, force=False, **kwargs: tuple or list) -> dic if old_value > update[len(update)-1]: raise SpchtErrors.WorkOrderInconsitencyError("New status higher than old one") if "insert" in kwargs: + protected_entries = ("file", "rdf_file") if isinstance(kwargs['insert'], tuple): kwargs['insert'] = [kwargs['insert']] for insert in kwargs['insert']: if len(insert) < 3: raise SpchtErrors.ParameterError("Not enough parameters") overwritten = AddNestedDictionaryKey(work_order, *insert) - # * sanity check - if overwritten and not force: + # * sanity check for certain fields + field_type = insert[len(insert)-1] + if overwritten and field_type in protected_entries and not force: raise SpchtErrors.WorkOrderInconsitencyError("Cannot overwrite any one file path") + # ? file entries are linked to somewhere, we dont want to overwrite those if "delete" in kwargs: if isinstance(kwargs['delete'], tuple): kwargs['delete'] = [kwargs['delete']] @@ -477,20 +491,12 @@ def CheckWorkOrder(work_order_file: str): :param str work_order_file: file path to a work order file :return: Nothing, only displays to console """ + global WORK_ORDER_STATUS print(work_order_file) work_order = load_from_json(work_order_file) if work_order is None: return False - main_status = ("Freshly created", # * 0 - "fetch started", # * 1 - "fetch completed", # * 2 - "processing started", # * 3 - "processing completed", # * 4 - "intermediate process started", # * 5 - "intermediate process finished", # * 6 - "inserting started", # * 7 - "insert completed/finished", # * 8 - "fullfilled") # * 9 + # ? surely this could have been a dictionary but it isn't time_infos = ("processing", "insert", "deletion", "solr") try: @@ -542,7 +548,7 @@ def CheckWorkOrder(work_order_file: str): if work_order['file_list'][key]['status'] == 7: counts['un_insert'] += 1 print("+++++++++++++++++++WORK ORDER INFO++++++++++++++++++") - print(f"Current status: {main_status[work_order['meta']['status']]}") + print(f"Current status: {WORK_ORDER_STATUS[work_order['meta']['status']]}") if counts['un_processing'] > 0: print(f"Unfinished processing: {counts['un_processing']}") if counts['un_insert'] > 0: @@ -655,7 +661,12 @@ def UseWorkOrder(work_order_file, deep_check = False, repair_mode = False, **kwa print(f"{msg}{boiler_print}") return 1 if work_order['meta']['status'] == 3: # processing started - SoftResetWorkOrder(work_order_file) + print(f"Pickuping the order in an 'inbetween' status - {WORK_ORDER_STATUS[status]}") + if not SoftResetWorkOrder(work_order_file): + msg = f"Reseting work order to state {WORK_ORDER_STATUS[work_order['meta']['status']]-1} failed" + print(msg) + logger.error(f"UserWorkOrder > {msg}") + return 3 if work_order['meta']['status'] == 2 or work_order['meta']['status'] == 3: # fetching completed # ! checks expected = ("work_order_file", "spcht_descriptor", "graph") @@ -674,7 +685,12 @@ def UseWorkOrder(work_order_file, deep_check = False, repair_mode = False, **kwa logger.info(f"Turtle Files created, commencing to next step") return 4 if work_order['meta']['status'] == 5: # intermediate processing started - SoftResetWorkOrder(work_order_file) + print(f"Pickuping the order in an 'inbetween' status - {WORK_ORDER_STATUS[work_order['meta']['status']]}") + if not SoftResetWorkOrder(work_order_file): + msg = f"Reseting work order to state {WORK_ORDER_STATUS[work_order['meta']['status']] - 1} failed" + print(msg) + logger.error(f"UserWorkOrder > {msg}") + return 3 if work_order['meta']['status'] == 4 or work_order['meta']['status'] == 5: # processing done if work_order['meta']['type'] == "insert": UpdateWorkOrder(work_order_file, update=("meta", "status", 6)) @@ -719,7 +735,12 @@ def UseWorkOrder(work_order_file, deep_check = False, repair_mode = False, **kwa else: logger.error(f"Unknown method '{work_order['meta']['method']}' in work order file {work_order_file}") if work_order['meta']['status'] == 7: # inserting started - SoftResetWorkOrder(work_order_file) + print(f"Pickuping the order in an 'inbetween' status - {WORK_ORDER_STATUS[work_order['meta']['status']]}") + if not SoftResetWorkOrder(work_order_file): + msg = f"Reseting work order to state {WORK_ORDER_STATUS[work_order['meta']['status']] - 1} failed" + print(msg) + logger.error(f"UserWorkOrder > {msg}") + return 3 if work_order['meta']['status'] == 6 or work_order['meta']['status'] == 7: # intermediate processing done if work_order['meta']['method'] == "isql": # ! checks @@ -1507,6 +1528,8 @@ def HardResetWorkOrder(work_order_file: str, **kwargs): fnc = "HardResetWorkOrder" print(msg) logging.error(f"{fnc} > {msg}") + else: + print(f"Generic TypeError: {e}") return False except OSError: print(f"Generic OSError while reseting work order file") @@ -1566,7 +1589,7 @@ def SoftResetWorkOrder(work_order_file: str, **kwargs): else: logger.info("Cannot soft reset anything.") return True - UpdateWorkOrder(work_order_file, update=list_of_updates, delete=list_of_deletes) + UpdateWorkOrder(work_order_file, update=list_of_updates, delete=list_of_deletes, force=True) return True @@ -1576,8 +1599,11 @@ def SoftResetWorkOrder(work_order_file: str, **kwargs): fnc = "SoftResetWorkOrder" print(msg) logging.error(f"{fnc} > {msg}") + else: + print(f"Generic TypeError: {e}") return False - return False + except Exception as e: + print(e) def PurgeWorkOrder(work_order_file: str, **kwargs): diff --git a/main.py b/main.py index 631254a..6214b24 100755 --- a/main.py +++ b/main.py @@ -249,6 +249,35 @@ def load_config(file_path="config.json"): else: print(status) + if args.ContinueOrder: + print("Continuing of an interrupted/paused order") + try: + for i in range(0, 6): + res = local_tools.UseWorkOrder(args.ContinueOrder, **PARA) + old_res = -1 + if isinstance(res, int): + if i > 0: + old_res = res + if res == 9: + print("Operation finished successfully") + local_tools.CheckWorkOrder(args.ContinueOrder) + exit(0) + if old_res == res: + print("Operation seems to be stuck on the same status, something is broken. Advising investigation") + local_tools.CheckWorkOrder(args.ContinueOrder) + exit(2) + print(local_tools.WORK_ORDER_STATUS[res]) + elif isinstance(res, list): + print("Fulfillment of current Work order status needs further parameters:") + for avery in res: + print(f"\t{colored(avery, attrs=['bold'])} - {colored(arguments[avery]['help'], 'green')}") + break + else: + print("Some really weird things happened, procedure reported an unexpeted status", file=sys.stderr) + except KeyboardInterrupt: + print("Process was aborted by user, use --ContinueOrder WORK_ORDER_NAME to continue") + exit(0) + if args.FullOrder: # ? notice for needed parameters before creating work order dynamic_requirements = [] @@ -327,12 +356,17 @@ def load_config(file_path="config.json"): elif not isinstance(res, list) and not isinstance(res, int): print("Process encountered a critical, unexpected situation, aborting", file=sys.stderr) exit(0) - if res == 9 or old_res == res: + if res == 9: print("Operation finished successfully") - local_tools.CheckWorkOrder(work_order) + local_tools.CheckWorkOrder(args.ContinueOrder) exit(0) + if old_res == res: + print("Operation seems to be stuck on the same status, something is broken. Advising investigation") + local_tools.CheckWorkOrder(args.ContinueOrder) + exit(2) except KeyboardInterrupt: - print("Aborted, FILL TEXT HERE ALAN") + print("Process was aborted by user, use --ContinueOrder WORK_ORDER_NAME to continue") + exit(0) # ? Utility Things