diff --git a/build/test-requirements.txt b/build/test-requirements.txt index c5c18a048f56..49e5fb4f75c3 100644 --- a/build/test-requirements.txt +++ b/build/test-requirements.txt @@ -31,3 +31,6 @@ django-stubs # for coverage coverage pytest-cov + +# for pytest-describe related tests +pytest-describe diff --git a/python_files/tests/pytestadapter/.data/pytest_describe_plugin/describe_only.py b/python_files/tests/pytestadapter/.data/pytest_describe_plugin/describe_only.py new file mode 100644 index 000000000000..0702c032684b --- /dev/null +++ b/python_files/tests/pytestadapter/.data/pytest_describe_plugin/describe_only.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +def describe_A(): + def test_1(): # test_marker--test_1 + pass + + def test_2(): # test_marker--test_2 + pass diff --git a/python_files/tests/pytestadapter/.data/pytest_describe_plugin/nested_describe.py b/python_files/tests/pytestadapter/.data/pytest_describe_plugin/nested_describe.py new file mode 100644 index 000000000000..5b9c13cc8d53 --- /dev/null +++ b/python_files/tests/pytestadapter/.data/pytest_describe_plugin/nested_describe.py @@ -0,0 +1,31 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import pytest + + +def describe_list(): + @pytest.fixture + def list(): + return [] + + def describe_append(): + def add_empty(list): # test_marker--add_empty + list.append("foo") + list.append("bar") + assert list == ["foo", "bar"] + + def remove_empty(list): # test_marker--remove_empty + try: + list.remove("foo") + except ValueError: + pass + + def describe_remove(): + @pytest.fixture + def list(): + return ["foo", "bar"] + + def removes(list): # test_marker--removes + list.remove("foo") + assert list == ["bar"] diff --git a/python_files/tests/pytestadapter/expected_discovery_test_output.py b/python_files/tests/pytestadapter/expected_discovery_test_output.py index 56b116e7dfd5..aa74a424ea2a 100644 --- a/python_files/tests/pytestadapter/expected_discovery_test_output.py +++ b/python_files/tests/pytestadapter/expected_discovery_test_output.py @@ -1394,3 +1394,186 @@ ], "id_": TEST_DATA_PATH_STR, } +# This is the expected output for the describe_only.py tests. +# └── describe_only.py +# └── describe_A +# └── test_1 +# └── test_2 + +describe_only_path = TEST_DATA_PATH / "pytest_describe_plugin" / "describe_only.py" +pytest_describe_plugin_path = TEST_DATA_PATH / "pytest_describe_plugin" + +expected_describe_only_output = { + "name": ".data", + "path": TEST_DATA_PATH_STR, + "type_": "folder", + "children": [ + { + "name": "pytest_describe_plugin", + "path": os.fspath(pytest_describe_plugin_path), + "type_": "folder", + "id_": os.fspath(pytest_describe_plugin_path), + "children": [ + { + "name": "describe_only.py", + "path": os.fspath(describe_only_path), + "type_": "file", + "id_": os.fspath(describe_only_path), + "children": [ + { + "name": "describe_A", + "path": os.fspath(describe_only_path), + "type_": "class", + "children": [ + { + "name": "test_1", + "path": os.fspath(describe_only_path), + "lineno": find_test_line_number( + "test_1", + describe_only_path, + ), + "type_": "test", + "id_": get_absolute_test_id( + "pytest_describe_plugin/describe_only.py::describe_A::test_1", + describe_only_path, + ), + "runID": get_absolute_test_id( + "pytest_describe_plugin/describe_only.py::describe_A::test_1", + describe_only_path, + ), + }, + { + "name": "test_2", + "path": os.fspath(describe_only_path), + "lineno": find_test_line_number( + "test_2", + describe_only_path, + ), + "type_": "test", + "id_": get_absolute_test_id( + "pytest_describe_plugin/describe_only.py::describe_A::test_2", + describe_only_path, + ), + "runID": get_absolute_test_id( + "pytest_describe_plugin/describe_only.py::describe_A::test_2", + describe_only_path, + ), + }, + ], + "id_": "pytest_describe_plugin/describe_only.py::describe_A", + } + ], + } + ], + } + ], + "id_": TEST_DATA_PATH_STR, +} +# This is the expected output for the nested_describe.py tests. +# └── nested_describe.py +# └── describe_list +# └── describe_append +# └── add_empty +# └── remove_empty +# └── describe_remove +# └── removes +nested_describe_path = TEST_DATA_PATH / "pytest_describe_plugin" / "nested_describe.py" +expected_nested_describe_output = { + "name": ".data", + "path": TEST_DATA_PATH_STR, + "type_": "folder", + "children": [ + { + "name": "pytest_describe_plugin", + "path": os.fspath(pytest_describe_plugin_path), + "type_": "folder", + "id_": os.fspath(pytest_describe_plugin_path), + "children": [ + { + "name": "nested_describe.py", + "path": os.fspath(nested_describe_path), + "type_": "file", + "id_": os.fspath(nested_describe_path), + "children": [ + { + "name": "describe_list", + "path": os.fspath(nested_describe_path), + "type_": "class", + "children": [ + { + "name": "describe_append", + "path": os.fspath(nested_describe_path), + "type_": "class", + "children": [ + { + "name": "add_empty", + "path": os.fspath(nested_describe_path), + "lineno": find_test_line_number( + "add_empty", + nested_describe_path, + ), + "type_": "test", + "id_": get_absolute_test_id( + "pytest_describe_plugin/nested_describe.py::describe_list::describe_append::add_empty", + nested_describe_path, + ), + "runID": get_absolute_test_id( + "pytest_describe_plugin/nested_describe.py::describe_list::describe_append::add_empty", + nested_describe_path, + ), + }, + { + "name": "remove_empty", + "path": os.fspath(nested_describe_path), + "lineno": find_test_line_number( + "remove_empty", + nested_describe_path, + ), + "type_": "test", + "id_": get_absolute_test_id( + "pytest_describe_plugin/nested_describe.py::describe_list::describe_append::remove_empty", + nested_describe_path, + ), + "runID": get_absolute_test_id( + "pytest_describe_plugin/nested_describe.py::describe_list::describe_append::remove_empty", + nested_describe_path, + ), + }, + ], + "id_": "pytest_describe_plugin/nested_describe.py::describe_list::describe_append", + }, + { + "name": "describe_remove", + "path": os.fspath(nested_describe_path), + "type_": "class", + "children": [ + { + "name": "removes", + "path": os.fspath(nested_describe_path), + "lineno": find_test_line_number( + "removes", + nested_describe_path, + ), + "type_": "test", + "id_": get_absolute_test_id( + "pytest_describe_plugin/nested_describe.py::describe_list::describe_remove::removes", + nested_describe_path, + ), + "runID": get_absolute_test_id( + "pytest_describe_plugin/nested_describe.py::describe_list::describe_remove::removes", + nested_describe_path, + ), + } + ], + "id_": "pytest_describe_plugin/nested_describe.py::describe_list::describe_remove", + }, + ], + "id_": "pytest_describe_plugin/nested_describe.py::describe_list", + } + ], + } + ], + } + ], + "id_": TEST_DATA_PATH_STR, +} diff --git a/python_files/tests/pytestadapter/expected_execution_test_output.py b/python_files/tests/pytestadapter/expected_execution_test_output.py index 521f72ab8439..8f378074343d 100644 --- a/python_files/tests/pytestadapter/expected_execution_test_output.py +++ b/python_files/tests/pytestadapter/expected_execution_test_output.py @@ -646,3 +646,91 @@ "subtest": None, } } + + +# This is the expected output for the pytest_describe_plugin/describe_only.py file. +# └── pytest_describe_plugin +# └── describe_only.py +# └── describe_A +# └── test_1: success +# └── test_2: success + +describe_only_expected_execution_output = { + get_absolute_test_id( + "pytest_describe_plugin/describe_only.py::describe_A::test_1", + TEST_DATA_PATH / "pytest_describe_plugin" / "describe_only.py", + ): { + "test": get_absolute_test_id( + "pytest_describe_plugin/describe_only.py::describe_A::test_1", + TEST_DATA_PATH / "pytest_describe_plugin" / "describe_only.py", + ), + "outcome": "success", + "message": None, + "traceback": None, + "subtest": None, + }, + get_absolute_test_id( + "pytest_describe_plugin/describe_only.py::describe_A::test_2", + TEST_DATA_PATH / "pytest_describe_plugin" / "describe_only.py", + ): { + "test": get_absolute_test_id( + "pytest_describe_plugin/describe_only.py::describe_A::test_2", + TEST_DATA_PATH / "pytest_describe_plugin" / "describe_only.py", + ), + "outcome": "success", + "message": None, + "traceback": None, + "subtest": None, + }, +} + +# This is the expected output for the pytest_describe_plugin/nested_describe.py file. +# └── pytest_describe_plugin +# └── nested_describe.py +# └── describe_list +# └── describe_append +# └── add_empty: success +# └── remove_empty: success +# └── describe_remove +# └── removes: success +nested_describe_expected_execution_output = { + get_absolute_test_id( + "pytest_describe_plugin/nested_describe.py::describe_list::describe_append::add_empty", + TEST_DATA_PATH / "pytest_describe_plugin" / "nested_describe.py", + ): { + "test": get_absolute_test_id( + "pytest_describe_plugin/nested_describe.py::describe_list::describe_append::add_empty", + TEST_DATA_PATH / "pytest_describe_plugin" / "nested_describe.py", + ), + "outcome": "success", + "message": None, + "traceback": None, + "subtest": None, + }, + get_absolute_test_id( + "pytest_describe_plugin/nested_describe.py::describe_list::describe_append::remove_empty", + TEST_DATA_PATH / "pytest_describe_plugin" / "nested_describe.py", + ): { + "test": get_absolute_test_id( + "pytest_describe_plugin/nested_describe.py::describe_list::describe_append::remove_empty", + TEST_DATA_PATH / "pytest_describe_plugin" / "nested_describe.py", + ), + "outcome": "success", + "message": None, + "traceback": None, + "subtest": None, + }, + get_absolute_test_id( + "pytest_describe_plugin/nested_describe.py::describe_list::describe_remove::removes", + TEST_DATA_PATH / "pytest_describe_plugin" / "nested_describe.py", + ): { + "test": get_absolute_test_id( + "pytest_describe_plugin/nested_describe.py::describe_list::describe_remove::removes", + TEST_DATA_PATH / "pytest_describe_plugin" / "nested_describe.py", + ), + "outcome": "success", + "message": None, + "traceback": None, + "subtest": None, + }, +} diff --git a/python_files/tests/pytestadapter/test_discovery.py b/python_files/tests/pytestadapter/test_discovery.py index c7752cf490ca..276753149410 100644 --- a/python_files/tests/pytestadapter/test_discovery.py +++ b/python_files/tests/pytestadapter/test_discovery.py @@ -161,6 +161,14 @@ def test_parameterized_error_collect(): "text_docstring.txt", expected_discovery_test_output.doctest_pytest_expected_output, ), + ( + "pytest_describe_plugin" + os.path.sep + "describe_only.py", + expected_discovery_test_output.expected_describe_only_output, + ), + ( + "pytest_describe_plugin" + os.path.sep + "nested_describe.py", + expected_discovery_test_output.expected_nested_describe_output, + ), ], ) def test_pytest_collect(file, expected_const): diff --git a/python_files/tests/pytestadapter/test_execution.py b/python_files/tests/pytestadapter/test_execution.py index 3ea8c685a9fe..245b13cf5d46 100644 --- a/python_files/tests/pytestadapter/test_execution.py +++ b/python_files/tests/pytestadapter/test_execution.py @@ -3,7 +3,6 @@ import json import os import pathlib -import shutil import sys from typing import Any, Dict, List @@ -66,80 +65,18 @@ def test_rootdir_specified(): assert actual_result_dict == expected_const -@pytest.mark.skipif( - sys.platform == "win32", - reason="See https://github.com/microsoft/vscode-python/issues/22965", -) -def test_syntax_error_execution(tmp_path): - """Test pytest execution on a file that has a syntax error. - - Copies the contents of a .txt file to a .py file in the temporary directory - to then run pytest execution on. - - The json should still be returned but the errors list should be present. - - Keyword arguments: - tmp_path -- pytest fixture that creates a temporary directory. - """ - # Saving some files as .txt to avoid that file displaying a syntax error for - # the extension as a whole. Instead, rename it before running this test - # in order to test the error handling. - file_path = TEST_DATA_PATH / "error_syntax_discovery.txt" - temp_dir = tmp_path / "temp_data" - temp_dir.mkdir() - p = temp_dir / "error_syntax_discovery.py" - shutil.copyfile(file_path, p) - actual = runner(["error_syntax_discover.py::test_function"]) - assert actual - actual_list: List[Dict[str, Dict[str, Any]]] = actual - - if actual_list is not None: - for actual_item in actual_list: - assert all(item in actual_item for item in ("status", "cwd", "error")) - assert actual_item.get("status") == "error" - assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) - error_content = actual_item.get("error") - if error_content is not None and isinstance( - error_content, (list, tuple, str) - ): # You can add other types if needed - assert len(error_content) == 1 - else: - pytest.fail(f"{error_content!r} is None or not a list, str, or tuple") - - -def test_bad_id_error_execution(): - """Test pytest discovery with a non-existent test_id. - - The json should still be returned but the errors list should be present. - """ - actual = runner(["not/a/real::test_id"]) - assert actual - actual_list: List[Dict[str, Dict[str, Any]]] = actual - if actual_list is not None: - for actual_item in actual_list: - assert all(item in actual_item for item in ("status", "cwd", "error")) - assert actual_item.get("status") == "error" - assert actual_item.get("cwd") == os.fspath(TEST_DATA_PATH) - error_content = actual_item.get("error") - if error_content is not None and isinstance( - error_content, (list, tuple, str) - ): # You can add other types if needed. - assert len(error_content) == 1 - else: - pytest.fail(f"{error_content!r} is None or not a list, str, or tuple") - - @pytest.mark.parametrize( ("test_ids", "expected_const"), [ - ( + pytest.param( [ "test_env_vars.py::test_clear_env", "test_env_vars.py::test_check_env", ], expected_execution_test_output.safe_clear_env_vars_expected_execution_output, + id="safe_clear_env_vars", ), - ( + pytest.param( [ "skip_tests.py::test_something", "skip_tests.py::test_another_thing", @@ -149,12 +86,14 @@ def test_bad_id_error_execution(): "skip_tests.py::TestClass::test_class_function_b", ], expected_execution_test_output.skip_tests_execution_expected_output, + id="skip_tests_execution", ), - ( + pytest.param( ["error_raise_exception.py::TestSomething::test_a"], expected_execution_test_output.error_raised_exception_execution_expected_output, + id="error_raised_exception", ), - ( + pytest.param( [ "unittest_folder/test_add.py::TestAddFunction::test_add_positive_numbers", "unittest_folder/test_add.py::TestAddFunction::test_add_negative_numbers", @@ -162,35 +101,40 @@ def test_bad_id_error_execution(): "unittest_folder/test_subtract.py::TestSubtractFunction::test_subtract_negative_numbers", ], expected_execution_test_output.uf_execution_expected_output, + id="unittest_multiple_files", ), - ( + pytest.param( [ "unittest_folder/test_add.py::TestAddFunction::test_add_positive_numbers", "unittest_folder/test_add.py::TestAddFunction::test_add_negative_numbers", ], expected_execution_test_output.uf_single_file_expected_output, + id="unittest_single_file", ), - ( + pytest.param( [ "unittest_folder/test_add.py::TestAddFunction::test_add_positive_numbers", ], expected_execution_test_output.uf_single_method_execution_expected_output, + id="unittest_single_method", ), - ( + pytest.param( [ "unittest_folder/test_add.py::TestAddFunction::test_add_positive_numbers", "unittest_folder/test_subtract.py::TestSubtractFunction::test_subtract_positive_numbers", ], expected_execution_test_output.uf_non_adjacent_tests_execution_expected_output, + id="unittest_non_adjacent_tests", ), - ( + pytest.param( [ "unittest_pytest_same_file.py::TestExample::test_true_unittest", "unittest_pytest_same_file.py::test_true_pytest", ], expected_execution_test_output.unit_pytest_same_file_execution_expected_output, + id="unittest_pytest_same_file", ), - ( + pytest.param( [ "dual_level_nested_folder/test_top_folder.py::test_top_function_t", "dual_level_nested_folder/test_top_folder.py::test_top_function_f", @@ -198,34 +142,57 @@ def test_bad_id_error_execution(): "dual_level_nested_folder/nested_folder_one/test_bottom_folder.py::test_bottom_function_f", ], expected_execution_test_output.dual_level_nested_folder_execution_expected_output, + id="dual_level_nested_folder", ), - ( - ["folder_a/folder_b/folder_a/test_nest.py::test_function"], ## + pytest.param( + ["folder_a/folder_b/folder_a/test_nest.py::test_function"], expected_execution_test_output.double_nested_folder_expected_execution_output, + id="double_nested_folder", ), - ( + pytest.param( [ - "parametrize_tests.py::TestClass::test_adding[3+5-8]", ## + "parametrize_tests.py::TestClass::test_adding[3+5-8]", "parametrize_tests.py::TestClass::test_adding[2+4-6]", "parametrize_tests.py::TestClass::test_adding[6+9-16]", ], expected_execution_test_output.parametrize_tests_expected_execution_output, + id="parametrize_tests", ), - ( + pytest.param( [ "parametrize_tests.py::TestClass::test_adding[3+5-8]", ], expected_execution_test_output.single_parametrize_tests_expected_execution_output, + id="single_parametrize_test", ), - ( + pytest.param( [ "text_docstring.txt::text_docstring.txt", ], expected_execution_test_output.doctest_pytest_expected_execution_output, + id="doctest_pytest", ), - ( + pytest.param( ["test_logging.py::test_logging2", "test_logging.py::test_logging"], expected_execution_test_output.logging_test_expected_execution_output, + id="logging_tests", + ), + pytest.param( + [ + "pytest_describe_plugin/describe_only.py::describe_A::test_1", + "pytest_describe_plugin/describe_only.py::describe_A::test_2", + ], + expected_execution_test_output.describe_only_expected_execution_output, + id="describe_only", + ), + pytest.param( + [ + "pytest_describe_plugin/nested_describe.py::describe_list::describe_append::add_empty", + "pytest_describe_plugin/nested_describe.py::describe_list::describe_append::remove_empty", + "pytest_describe_plugin/nested_describe.py::describe_list::describe_remove::removes", + ], + expected_execution_test_output.nested_describe_expected_execution_output, + id="nested_describe_plugin", ), ], ) @@ -233,22 +200,6 @@ def test_pytest_execution(test_ids, expected_const): """ Test that pytest discovery works as expected where run pytest is always successful, but the actual test results are both successes and failures. - 1: skip_tests_execution_expected_output: test run on a file with skipped tests. - 2. error_raised_exception_execution_expected_output: test run on a file that raises an exception. - 3. uf_execution_expected_output: unittest tests run on multiple files. - 4. uf_single_file_expected_output: test run on a single file. - 5. uf_single_method_execution_expected_output: test run on a single method in a file. - 6. uf_non_adjacent_tests_execution_expected_output: test run on unittests in two files with single selection in test explorer. - 7. unit_pytest_same_file_execution_expected_output: test run on a file with both unittest and pytest tests. - 8. dual_level_nested_folder_execution_expected_output: test run on a file with one test file - at the top level and one test file in a nested folder. - 9. double_nested_folder_expected_execution_output: test run on a double nested folder. - 10. parametrize_tests_expected_execution_output: test run on a parametrize test with 3 inputs. - 11. single_parametrize_tests_expected_execution_output: test run on single parametrize test. - 12. doctest_pytest_expected_execution_output: test run on doctest file. - 13. logging_test_expected_execution_output: test run on a file with logging. - - Keyword arguments: test_ids -- an array of test_ids to run. expected_const -- a dictionary of the expected output from running pytest discovery on the files. diff --git a/python_files/vscode_pytest/__init__.py b/python_files/vscode_pytest/__init__.py index 028839b13212..a867b9cfca95 100644 --- a/python_files/vscode_pytest/__init__.py +++ b/python_files/vscode_pytest/__init__.py @@ -4,6 +4,7 @@ from __future__ import annotations import atexit +import contextlib import json import os import pathlib @@ -29,6 +30,14 @@ from pluggy import Result +USES_PYTEST_DESCRIBE = False + +with contextlib.suppress(ImportError): + from pytest_describe.plugin import DescribeBlock + + USES_PYTEST_DESCRIBE = True + + class TestData(TypedDict): """A general class that all test objects inherit from.""" @@ -529,11 +538,15 @@ def build_test_tree(session: pytest.Session) -> TestNode: parent_test_case["children"].append(function_test_node) # If the parent is not a file, it is a class, add the function node as the test node to handle subsequent nesting. test_node = function_test_node - if isinstance(test_case.parent, pytest.Class): + if isinstance(test_case.parent, pytest.Class) or ( + USES_PYTEST_DESCRIBE and isinstance(test_case.parent, DescribeBlock) + ): case_iter = test_case.parent node_child_iter = test_node test_class_node: TestNode | None = None - while isinstance(case_iter, pytest.Class): + while isinstance(case_iter, pytest.Class) or ( + USES_PYTEST_DESCRIBE and isinstance(case_iter, DescribeBlock) + ): # While the given node is a class, create a class and nest the previous node as a child. try: test_class_node = class_nodes_dict[case_iter.nodeid] @@ -690,7 +703,7 @@ def create_session_node(session: pytest.Session) -> TestNode: } -def create_class_node(class_module: pytest.Class) -> TestNode: +def create_class_node(class_module: pytest.Class | DescribeBlock) -> TestNode: """Creates a class node from a pytest class object. Keyword arguments: