From a23e68a61726eed4da24ecccaaad271a9abd5fd1 Mon Sep 17 00:00:00 2001 From: Hoang Tung Dinh Date: Tue, 10 Sep 2024 16:50:38 +0200 Subject: [PATCH 1/5] Add function to check if there is at least one issue from a set of rules (#30) Signed-off-by: hoangtungdinh <11166240+hoangtungdinh@users.noreply.github.com> --- qc_baselib/result.py | 11 +++++- tests/test_result.py | 83 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/qc_baselib/result.py b/qc_baselib/result.py index 84aa1d7..a5a0b6e 100644 --- a/qc_baselib/result.py +++ b/qc_baselib/result.py @@ -5,7 +5,7 @@ from copy import deepcopy from dataclasses import dataclass -from typing import Union, List, Dict, Any +from typing import Union, List, Set from lxml import etree from .models import IssueSeverity, result @@ -544,3 +544,12 @@ def get_issues_by_rule_uid(self, rule_uid: str) -> List[result.IssueType]: rule_issues.append(issue) return rule_issues + + def has_at_least_one_issue_from_rules(self, rule_uid_set: Set[str]) -> bool: + for bundle in self._report_results.checker_bundles: + for checker in bundle.checkers: + for issue in checker.issues: + if issue.rule_uid in rule_uid_set: + return True + + return False diff --git a/tests/test_result.py b/tests/test_result.py index abe15d5..e74ed56 100644 --- a/tests/test_result.py +++ b/tests/test_result.py @@ -645,3 +645,86 @@ def test_markdown_docs_output(): assert output_md_text == example_md_text os.remove(TEST_MARKDOWN_DOC_OUTPUT_PATH) + + +def test_has_at_least_one_issue_from_rules() -> None: + result_report = Result() + + result_report.register_checker_bundle( + name="TestBundle", + build_date="2024-05-31", + description="Example checker bundle", + version="0.0.1", + summary="Tested example checkers", + ) + + result_report.register_checker( + checker_bundle_name="TestBundle", + checker_id="TestChecker", + description="Test checker", + summary="Executed evaluation", + ) + + rule_uid_1 = result_report.register_rule( + checker_bundle_name="TestBundle", + checker_id="TestChecker", + emanating_entity="test.com", + standard="qc", + definition_setting="1.0.0", + rule_full_name="first.rule", + ) + + result_report.register_issue( + checker_bundle_name="TestBundle", + checker_id="TestChecker", + description="Issue found at odr", + level=IssueSeverity.INFORMATION, + rule_uid=rule_uid_1, + ) + + result_report.register_checker( + checker_bundle_name="TestBundle", + checker_id="TestChecker", + description="Test checker", + summary="Executed evaluation", + ) + + rule_uid_2 = result_report.register_rule( + checker_bundle_name="TestBundle", + checker_id="TestChecker", + emanating_entity="test.com", + standard="qc", + definition_setting="1.0.0", + rule_full_name="second.rule", + ) + + result_report.register_issue( + checker_bundle_name="TestBundle", + checker_id="TestChecker", + description="Issue found at odr", + level=IssueSeverity.INFORMATION, + rule_uid=rule_uid_2, + ) + + assert ( + result_report.has_at_least_one_issue_from_rules( + {"test.com:qc:1.0.0:third.rule", "test.com:qc:1.0.0:fourth.rule"} + ) + == False + ) + + assert ( + result_report.has_at_least_one_issue_from_rules( + {"test.com:qc:1.0.0:first.rule", "test.com:qc:1.0.0:fourth.rule"} + ) + == True + ) + + assert ( + result_report.has_at_least_one_issue_from_rules( + {"test.com:qc:1.0.0:second.rule", "test.com:qc:1.0.0:fourth.rule"} + ) + == True + ) + + assert result_report.has_at_least_one_issue_from_rules({}) == False From 3319f7f52eb006bee20ec05c674037aee2d9de59 Mon Sep 17 00:00:00 2001 From: Hoang Tung Dinh Date: Tue, 10 Sep 2024 18:09:31 +0200 Subject: [PATCH 2/5] Add warning message when more than one rules are registered to check (#31) Signed-off-by: hoangtungdinh <11166240+hoangtungdinh@users.noreply.github.com> --- qc_baselib/result.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/qc_baselib/result.py b/qc_baselib/result.py index a5a0b6e..eaf1a1f 100644 --- a/qc_baselib/result.py +++ b/qc_baselib/result.py @@ -8,6 +8,7 @@ from typing import Union, List, Set from lxml import etree from .models import IssueSeverity, result +import logging REPORT_OUTPUT_FORMAT = "xqar" DEFAULT_REPORT_VERSION = "0.0.1" @@ -281,6 +282,16 @@ def register_rule( checker = self._get_checker(bundle=bundle, checker_id=checker_id) checker.addressed_rule.append(rule) + number_of_addressed_rules = len(checker.addressed_rule) + if number_of_addressed_rules > 1: + logging.warning( + f"There are {number_of_addressed_rules} rules registered to the check" + f" {checker_id}. A check should address exactly one rule, unless there" + " is a strong reason not to. See the following document for more" + " information:" + " https://github.com/asam-ev/qc-framework/blob/main/doc/manual/checker_library.md#check-characteristics" + ) + return rule.rule_uid def register_issue( From 0a41ff6f23c5667ecff08685e8c257dd9d59f609 Mon Sep 17 00:00:00 2001 From: Hoang Tung Dinh Date: Tue, 10 Sep 2024 18:36:41 +0200 Subject: [PATCH 3/5] Make summary as optional when register checkers and checker bundles (#32) Signed-off-by: hoangtungdinh <11166240+hoangtungdinh@users.noreply.github.com> --- qc_baselib/result.py | 10 +++++++--- tests/test_result.py | 19 +++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/qc_baselib/result.py b/qc_baselib/result.py index eaf1a1f..9e8a672 100644 --- a/qc_baselib/result.py +++ b/qc_baselib/result.py @@ -47,7 +47,6 @@ class Result: build_date="2024-05-31", description="Example checker bundle", version="0.0.1", - summary="Tested example checkers", ) result.write_to_file("testResults.xqar") @@ -226,7 +225,12 @@ def _get_issue( return issue def register_checker_bundle( - self, build_date: str, description: str, name: str, version: str, summary: str + self, + build_date: str, + description: str, + name: str, + version: str, + summary: str = "", ) -> None: bundle = result.CheckerBundleType( build_date=build_date, @@ -246,7 +250,7 @@ def register_checker( checker_bundle_name: str, checker_id: str, description: str, - summary: str, + summary: str = "", ) -> None: checker = result.CheckerType( diff --git a/tests/test_result.py b/tests/test_result.py index e74ed56..2d2c7d2 100644 --- a/tests/test_result.py +++ b/tests/test_result.py @@ -728,3 +728,22 @@ def test_has_at_least_one_issue_from_rules() -> None: ) assert result_report.has_at_least_one_issue_from_rules({}) == False + + +def test_registration_without_summary() -> None: + result_report = Result() + + result_report.register_checker_bundle( + name="TestBundle", + build_date="2024-05-31", + description="Example checker bundle", + version="0.0.1", + ) + + result_report.register_checker( + checker_bundle_name="TestBundle", + checker_id="TestChecker", + description="Test checker", + ) + + assert True From 88c338777746616879828eb3f417c77ab3f1caff Mon Sep 17 00:00:00 2001 From: Hoang Tung Dinh Date: Wed, 11 Sep 2024 09:45:40 +0200 Subject: [PATCH 4/5] Add more functions to support precondition check and update markdown generator (#33) Signed-off-by: hoangtungdinh <11166240+hoangtungdinh@users.noreply.github.com> --- qc_baselib/result.py | 39 +++++- tests/data/result_markdown_docs.md | 2 - tests/test_result.py | 183 ++++++++++++++++++++++++++++- 3 files changed, 211 insertions(+), 13 deletions(-) diff --git a/qc_baselib/result.py b/qc_baselib/result.py index 9e8a672..ed0fe8b 100644 --- a/qc_baselib/result.py +++ b/qc_baselib/result.py @@ -7,7 +7,7 @@ from dataclasses import dataclass from typing import Union, List, Set from lxml import etree -from .models import IssueSeverity, result +from .models import IssueSeverity, StatusType, result import logging REPORT_OUTPUT_FORMAT = "xqar" @@ -119,13 +119,12 @@ def write_markdown_doc(self, markdown_file_path: str) -> None: bundle_text += f"# Checker bundle: {bundle.name}\n\n" bundle_text += f"* Build version: {bundle.version}\n" bundle_text += f"* Description: {bundle.description}\n" - bundle_text += f"* Summary: {bundle.summary}\n" bundle_text += "\n" bundle_text += f"## Parameters\n\n" param_text = "" for param in bundle.params: - param_text += f"* {param.name}: \n" + param_text += f"* {param.name} \n" if len(param_text) == 0: param_text += f"* None\n" @@ -139,7 +138,6 @@ def write_markdown_doc(self, markdown_file_path: str) -> None: checker_text += "\n" checker_text += f"### {checker.checker_id}\n\n" checker_text += f"* Description: {checker.description}\n" - checker_text += f"* Summary: {checker.summary}\n" checker_text += f"* Addressed rules:\n" rule_text = "" @@ -428,7 +426,7 @@ def add_domain_specific_info( issue.domain_specific_info.append(domain_specific_tag) def set_checker_status( - self, checker_bundle_name: str, checker_id: str, status: result.StatusType + self, checker_bundle_name: str, checker_id: str, status: StatusType ) -> None: bundle = self._get_checker_bundle(checker_bundle_name=checker_bundle_name) checker = self._get_checker(bundle=bundle, checker_id=checker_id) @@ -560,7 +558,7 @@ def get_issues_by_rule_uid(self, rule_uid: str) -> List[result.IssueType]: return rule_issues - def has_at_least_one_issue_from_rules(self, rule_uid_set: Set[str]) -> bool: + def has_issue_in_rules(self, rule_uid_set: Set[str]) -> bool: for bundle in self._report_results.checker_bundles: for checker in bundle.checkers: for issue in checker.issues: @@ -568,3 +566,32 @@ def has_at_least_one_issue_from_rules(self, rule_uid_set: Set[str]) -> bool: return True return False + + def has_issue_in_checkers(self, check_id_set: Set[str]) -> bool: + for bundle in self._report_results.checker_bundles: + for checker in bundle.checkers: + if checker.checker_id in check_id_set and len(checker.issues) > 0: + return True + + return False + + def all_checkers_completed_without_issue(self, check_id_set: Set[str]) -> bool: + checker_id_map = dict() + for checker_id in check_id_set: + checker_id_map[checker_id] = False + + for bundle in self._report_results.checker_bundles: + for checker in bundle.checkers: + if ( + checker.checker_id in check_id_set + and checker.status == StatusType.COMPLETED + and len(checker.issues) == 0 + ): + checker_id_map[checker.checker_id] = True + + result = True + + for _, checker_result in checker_id_map.items(): + result = result and checker_result + + return result diff --git a/tests/data/result_markdown_docs.md b/tests/data/result_markdown_docs.md index f9caaf8..83b8881 100644 --- a/tests/data/result_markdown_docs.md +++ b/tests/data/result_markdown_docs.md @@ -10,7 +10,6 @@ * Build version: 0.0.1 * Description: Example checker bundle -* Summary: Tested example checkers ## Parameters @@ -21,6 +20,5 @@ ### TestChecker * Description: Test checker -* Summary: Executed evaluation * Addressed rules: * test.com:qc:1.0.0:qwerty.qwerty diff --git a/tests/test_result.py b/tests/test_result.py index 2d2c7d2..5350131 100644 --- a/tests/test_result.py +++ b/tests/test_result.py @@ -647,7 +647,7 @@ def test_markdown_docs_output(): os.remove(TEST_MARKDOWN_DOC_OUTPUT_PATH) -def test_has_at_least_one_issue_from_rules() -> None: +def test_has_issue_in_rules() -> None: result_report = Result() result_report.register_checker_bundle( @@ -707,27 +707,200 @@ def test_has_at_least_one_issue_from_rules() -> None: ) assert ( - result_report.has_at_least_one_issue_from_rules( + result_report.has_issue_in_rules( {"test.com:qc:1.0.0:third.rule", "test.com:qc:1.0.0:fourth.rule"} ) == False ) assert ( - result_report.has_at_least_one_issue_from_rules( + result_report.has_issue_in_rules( {"test.com:qc:1.0.0:first.rule", "test.com:qc:1.0.0:fourth.rule"} ) == True ) assert ( - result_report.has_at_least_one_issue_from_rules( + result_report.has_issue_in_rules( {"test.com:qc:1.0.0:second.rule", "test.com:qc:1.0.0:fourth.rule"} ) == True ) - assert result_report.has_at_least_one_issue_from_rules({}) == False + assert result_report.has_issue_in_rules({}) == False + + +def test_has_issue_in_checkers() -> None: + result_report = Result() + + result_report.register_checker_bundle( + name="TestBundle", + build_date="2024-05-31", + description="Example checker bundle", + version="0.0.1", + summary="Tested example checkers", + ) + + result_report.register_checker( + checker_bundle_name="TestBundle", + checker_id="FirstChecker", + description="Test checker", + summary="Executed evaluation", + ) + + rule_uid_1 = result_report.register_rule( + checker_bundle_name="TestBundle", + checker_id="FirstChecker", + emanating_entity="test.com", + standard="qc", + definition_setting="1.0.0", + rule_full_name="first.rule", + ) + + result_report.register_issue( + checker_bundle_name="TestBundle", + checker_id="FirstChecker", + description="Issue found at odr", + level=IssueSeverity.INFORMATION, + rule_uid=rule_uid_1, + ) + + result_report.register_checker( + checker_bundle_name="TestBundle", + checker_id="SecondChecker", + description="Test checker", + summary="Executed evaluation", + ) + + result_report.register_rule( + checker_bundle_name="TestBundle", + checker_id="SecondChecker", + emanating_entity="test.com", + standard="qc", + definition_setting="1.0.0", + rule_full_name="second.rule", + ) + + assert ( + result_report.has_issue_in_checkers({"SecondChecker", "ThirdChecker"}) == False + ) + + assert ( + result_report.has_issue_in_checkers({"FirstChecker", "SecondChecker"}) == True + ) + + assert result_report.has_issue_in_checkers({"FirstChecker", "ThirdChecker"}) == True + + assert result_report.has_issue_in_checkers({}) == False + + +def test_all_checkers_completed_without_issue() -> None: + result_report = Result() + + result_report.register_checker_bundle( + name="TestBundle", + build_date="2024-05-31", + description="Example checker bundle", + version="0.0.1", + ) + + result_report.register_checker( + checker_bundle_name="TestBundle", + checker_id="FirstChecker", + description="", + ) + + result_report.register_checker( + checker_bundle_name="TestBundle", + checker_id="SecondChecker", + description="", + ) + + assert result_report.all_checkers_completed_without_issue({"ThirdChecker"}) == False + + assert ( + result_report.all_checkers_completed_without_issue( + {"FirstChecker", "SecondChecker"} + ) + == False + ) + + result_report.set_checker_status( + checker_bundle_name="TestBundle", + checker_id="FirstChecker", + status=StatusType.COMPLETED, + ) + + assert result_report.all_checkers_completed_without_issue({"FirstChecker"}) == True + assert ( + result_report.all_checkers_completed_without_issue( + {"FirstChecker", "SecondChecker"} + ) + == False + ) + + result_report.set_checker_status( + checker_bundle_name="TestBundle", + checker_id="FirstChecker", + status=StatusType.SKIPPED, + ) + + assert result_report.all_checkers_completed_without_issue({"FirstChecker"}) == False + + result_report.set_checker_status( + checker_bundle_name="TestBundle", + checker_id="FirstChecker", + status=StatusType.ERROR, + ) + + assert result_report.all_checkers_completed_without_issue({"FirstChecker"}) == False + + result_report.set_checker_status( + checker_bundle_name="TestBundle", + checker_id="FirstChecker", + status=StatusType.COMPLETED, + ) + + result_report.set_checker_status( + checker_bundle_name="TestBundle", + checker_id="SecondChecker", + status=StatusType.COMPLETED, + ) + + assert ( + result_report.all_checkers_completed_without_issue( + {"FirstChecker", "SecondChecker"} + ) + == True + ) + + rule_uid_1 = result_report.register_rule( + checker_bundle_name="TestBundle", + checker_id="FirstChecker", + emanating_entity="test.com", + standard="qc", + definition_setting="1.0.0", + rule_full_name="first.rule", + ) + + result_report.register_issue( + checker_bundle_name="TestBundle", + checker_id="FirstChecker", + description="Issue found at odr", + level=IssueSeverity.INFORMATION, + rule_uid=rule_uid_1, + ) + + assert ( + result_report.all_checkers_completed_without_issue( + {"FirstChecker", "SecondChecker"} + ) + == False + ) + + assert result_report.all_checkers_completed_without_issue({"FirstChecker"}) == False + assert result_report.all_checkers_completed_without_issue({"SecondChecker"}) == True + assert result_report.all_checkers_completed_without_issue({}) == True def test_registration_without_summary() -> None: From 6736b54028972bd3f99d13601e69ff6995ab3872 Mon Sep 17 00:00:00 2001 From: Hoang Tung Dinh Date: Wed, 11 Sep 2024 14:44:00 +0200 Subject: [PATCH 5/5] Add helper functions for querying checker status (#34) Signed-off-by: hoangtungdinh <11166240+hoangtungdinh@users.noreply.github.com> --- qc_baselib/result.py | 19 ++++++++ tests/test_result.py | 114 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) diff --git a/qc_baselib/result.py b/qc_baselib/result.py index ed0fe8b..714d0c2 100644 --- a/qc_baselib/result.py +++ b/qc_baselib/result.py @@ -595,3 +595,22 @@ def all_checkers_completed_without_issue(self, check_id_set: Set[str]) -> bool: result = result and checker_result return result + + def get_checker_status(self, checker_id: str) -> Union[None, StatusType]: + """ + Return None if the checker is not found. + """ + for bundle in self._report_results.checker_bundles: + for checker in bundle.checkers: + if checker.checker_id == checker_id: + return checker.status + + return None + + def all_checkers_completed(self) -> bool: + for bundle in self._report_results.checker_bundles: + for checker in bundle.checkers: + if checker.status != StatusType.COMPLETED: + return False + + return True diff --git a/tests/test_result.py b/tests/test_result.py index 5350131..b3a8f77 100644 --- a/tests/test_result.py +++ b/tests/test_result.py @@ -920,3 +920,117 @@ def test_registration_without_summary() -> None: ) assert True + + +def test_get_checker_status() -> None: + result_report = Result() + + result_report.register_checker_bundle( + name="TestBundle", + build_date="2024-05-31", + description="Example checker bundle", + version="0.0.1", + ) + + assert result_report.get_checker_status("TestChecker") == None + + result_report.register_checker( + checker_bundle_name="TestBundle", + checker_id="TestChecker", + description="", + ) + + assert result_report.get_checker_status("TestChecker") == None + + result_report.set_checker_status( + checker_bundle_name="TestBundle", + checker_id="TestChecker", + status=StatusType.COMPLETED, + ) + + assert result_report.get_checker_status("TestChecker") == StatusType.COMPLETED + + result_report.set_checker_status( + checker_bundle_name="TestBundle", + checker_id="TestChecker", + status=StatusType.SKIPPED, + ) + + assert result_report.get_checker_status("TestChecker") == StatusType.SKIPPED + + result_report.set_checker_status( + checker_bundle_name="TestBundle", + checker_id="TestChecker", + status=StatusType.ERROR, + ) + + assert result_report.get_checker_status("TestChecker") == StatusType.ERROR + + +def test_all_checkers_completed() -> None: + result_report = Result() + + result_report.register_checker_bundle( + name="TestBundle", + build_date="2024-05-31", + description="Example checker bundle", + version="0.0.1", + ) + + assert result_report.all_checkers_completed() == True + + result_report.register_checker( + checker_bundle_name="TestBundle", + checker_id="TestChecker", + description="", + ) + + assert result_report.all_checkers_completed() == False + + result_report.set_checker_status( + checker_bundle_name="TestBundle", + checker_id="TestChecker", + status=StatusType.COMPLETED, + ) + + assert result_report.all_checkers_completed() == True + + result_report.set_checker_status( + checker_bundle_name="TestBundle", + checker_id="TestChecker", + status=StatusType.SKIPPED, + ) + + assert result_report.all_checkers_completed() == False + + result_report.set_checker_status( + checker_bundle_name="TestBundle", + checker_id="TestChecker", + status=StatusType.ERROR, + ) + + assert result_report.all_checkers_completed() == False + + result_report.register_checker( + checker_bundle_name="TestBundle", + checker_id="SecondTestChecker", + description="", + ) + + assert result_report.all_checkers_completed() == False + + result_report.set_checker_status( + checker_bundle_name="TestBundle", + checker_id="TestChecker", + status=StatusType.COMPLETED, + ) + + assert result_report.all_checkers_completed() == False + + result_report.set_checker_status( + checker_bundle_name="TestBundle", + checker_id="SecondTestChecker", + status=StatusType.COMPLETED, + ) + + assert result_report.all_checkers_completed() == True