From 5bc9a362affec2e657d9815eb620ae2ca51845c0 Mon Sep 17 00:00:00 2001 From: Aleksei Apaseev Date: Fri, 29 Nov 2024 16:08:42 +0800 Subject: [PATCH] feat(junit-merger): add flag to preserve Python test cases and PYTHON_FUNC attribute --- pytest-embedded-idf/tests/test_idf.py | 95 +++++++++++++++++++++++ pytest-embedded/pytest_embedded/plugin.py | 11 ++- pytest-embedded/pytest_embedded/unity.py | 9 ++- 3 files changed, 112 insertions(+), 3 deletions(-) diff --git a/pytest-embedded-idf/tests/test_idf.py b/pytest-embedded-idf/tests/test_idf.py index ab274ee4..a51991f9 100644 --- a/pytest-embedded-idf/tests/test_idf.py +++ b/pytest-embedded-idf/tests/test_idf.py @@ -877,6 +877,11 @@ def test_unity_test_case_runner(unity_tester): assert junit_report[3].find('failure') is None +def test_unity_test_case_runner_new(unity_tester): + unity_tester.run_all_cases() + + + def test_erase_all_with_port_cache(testdir): testdir.makepyfile(r""" def test_erase_all_with_port_cache_case1(dut): @@ -897,3 +902,93 @@ def test_erase_all_with_port_cache_case2(dut): ) result.assert_outcomes(passed=2) + + +def test_no_preserve_python_tests(testdir): + testdir.makepyfile(r""" + def test_python_case(dut): + dut.run_all_single_board_cases(name=["normal_case1", "multiple_stages_test"]) + """) + + testdir.runpytest( + '-s', + '--embedded-services', 'esp,idf', + '--app-path', os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), + '--log-cli-level', 'DEBUG', + '--junitxml', 'report.xml', + ) + + junit_report = ET.parse('report.xml').getroot()[0] + + assert junit_report.attrib['tests'] == '2' + for testcase in junit_report.findall('testcase'): + assert testcase.attrib['PYTHON_FUNC'] == '0' + +def test_preserve_python_tests(testdir): + testdir.makepyfile(r""" + def test_python_case(dut): + dut.run_all_single_board_cases(name=["normal_case1", "multiple_stages_test"]) + """) + + testdir.runpytest( + '-s', + '--embedded-services', 'esp,idf', + '--app-path', os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), + '--log-cli-level', 'DEBUG', + '--junitxml', 'report.xml', + '--preserve-python-tests-in-report', + ) + + junit_report = ET.parse('report.xml').getroot()[0] + + assert junit_report.attrib['tests'] == '2' + assert junit_report[0].attrib['PYTHON_FUNC'] == '1' + for testcase in junit_report[1:]: + assert testcase.attrib['PYTHON_FUNC'] == '0' + + +def test_preserve_python_tests_with_failures(testdir): + testdir.makepyfile(r""" + def test_python_case(dut): + dut.run_all_single_board_cases(name=["normal_case1", "normal_case2"]) + """) + + testdir.runpytest( + '-s', + '--embedded-services', 'esp,idf', + '--app-path', os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), + '--log-cli-level', 'DEBUG', + '--junitxml', 'report.xml', + '--preserve-python-tests-in-report', + ) + + junit_report = ET.parse('report.xml').getroot()[0] + + assert junit_report.attrib['failures'] == '1' + assert junit_report[0].attrib['PYTHON_FUNC'] == '1' # Python test case is preserved + assert junit_report[1].attrib['PYTHON_FUNC'] == '0' # C test case + assert junit_report[1].find('failure') is None # normal_case1 passed + assert junit_report[2].attrib['PYTHON_FUNC'] == '0' + assert junit_report[2].find('failure') is not None # normal_case2 failed + + +def test_python_func_attribute(testdir): + testdir.makepyfile(r""" + def test_python_case(dut): + dut.run_all_single_board_cases(name=["normal_case1", "multiple_stages_test"]) + """) + + testdir.runpytest( + '-s', + '--embedded-services', 'esp,idf', + '--app-path', os.path.join(testdir.tmpdir, 'unit_test_app_esp32'), + '--log-cli-level', 'DEBUG', + '--junitxml', 'report.xml', + '--preserve-python-tests-in-report', + ) + + junit_report = ET.parse('report.xml').getroot()[0] + + assert junit_report[0].attrib['PYTHON_FUNC'] == '1' # Python test case + for testcase in junit_report[1:]: + assert testcase.attrib['PYTHON_FUNC'] == '0' # Other test cases diff --git a/pytest-embedded/pytest_embedded/plugin.py b/pytest-embedded/pytest_embedded/plugin.py index d5311677..c05544a7 100644 --- a/pytest-embedded/pytest_embedded/plugin.py +++ b/pytest-embedded/pytest_embedded/plugin.py @@ -142,6 +142,14 @@ def pytest_addoption(parser): base_group.addoption( '--logfile-extension', default='.log', help='set the extension format of the log files. (Default: ".log")' ) + base_group.addoption( + '--preserve-python-tests-in-report', + action='store_true', + default=False, + help='Preserve Python test cases in the JUnit report alongside other test cases. ' + 'By default, Python test cases are replaced by the more detailed test case results ' + 'from submodules (e.g., DUT test cases). Use this flag to retain both.', + ) serial_group = parser.getgroup('embedded-serial') serial_group.addoption('--port', help='serial port. (Env: "ESPPORT" if service "esp" specified, Default: "None")') @@ -1183,7 +1191,8 @@ def unity_tester(dut: t.Union['IdfDut', t.Tuple['IdfDut']]) -> t.Optional['CaseT def pytest_configure(config: Config) -> None: - config.stash[_junit_merger_key] = JunitMerger(config.option.xmlpath) + preserve_python_tests_in_report = config.getoption('preserve_python_tests_in_report', False) + config.stash[_junit_merger_key] = JunitMerger(config.option.xmlpath, preserve_python_tests_in_report) config.stash[_junit_report_path_key] = config.option.xmlpath config.stash[_pytest_embedded_key] = PytestEmbedded( diff --git a/pytest-embedded/pytest_embedded/unity.py b/pytest-embedded/pytest_embedded/unity.py index 98fa2af6..ed749d53 100644 --- a/pytest-embedded/pytest_embedded/unity.py +++ b/pytest-embedded/pytest_embedded/unity.py @@ -206,8 +206,9 @@ class JunitMerger: SUB_JUNIT_FILENAME = 'dut.xml' # multi-dut junit reports should be dut-[INDEX].xml - def __init__(self, main_junit: Optional[str]) -> None: + def __init__(self, main_junit: Optional[str], preserve_python_tests_in_report: Optional[str] = False) -> None: self.junit_path = main_junit + self.preserve_python_tests_in_report = preserve_python_tests_in_report self._junit = None @@ -291,9 +292,13 @@ def merge(self, junit_files: List[str]): raise ValueError(f'Could\'t find test case {test_case_name}, dumped into "debug.xml" for debugging') junit_case_is_fail = junit_case.find('failure') is not None - junit_parent.remove(junit_case) + + junit_case.attrib['PYTHON_FUNC'] = '1' + if not self.preserve_python_tests_in_report: + junit_parent.remove(junit_case) for case in merging_cases: + case.attrib['PYTHON_FUNC'] = '0' junit_parent.append(case) junit_parent.attrib['errors'] = self._int_add(