Skip to content

Commit

Permalink
Implementation of Test Coverage (microsoft#24118)
Browse files Browse the repository at this point in the history
  • Loading branch information
eleanorjboyd authored Sep 18, 2024
1 parent 717e518 commit 8268131
Show file tree
Hide file tree
Showing 30 changed files with 895 additions and 81 deletions.
5 changes: 5 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -49,3 +49,8 @@ dist/**
package.nls.*.json
l10n/
python-env-tools/**
# coverage files produced as test output
python_files/tests/*/.data/.coverage*
python_files/tests/*/.data/*/.coverage*
src/testTestingRootWkspc/coverageWorkspace/.coverage

2 changes: 2 additions & 0 deletions build/functional-test-requirements.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# List of requirements for functional tests
versioneer
numpy
pytest
pytest-cov
4 changes: 4 additions & 0 deletions build/test-requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -27,3 +27,7 @@ namedpipe; platform_system == "Windows"

# typing for Django files
django-stubs

# for coverage
coverage
pytest-cov
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
19 changes: 19 additions & 0 deletions python_files/tests/pytestadapter/.data/coverage_gen/reverse.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

def reverse_string(s):
if s is None or s == "":
return "Error: Input is None"
return s[::-1]

def reverse_sentence(sentence):
if sentence is None or sentence == "":
return "Error: Input is None"
words = sentence.split()
reversed_words = [reverse_string(word) for word in words]
return " ".join(reversed_words)

# Example usage
if __name__ == "__main__":
sample_string = "hello"
print(reverse_string(sample_string)) # Output: "olleh"
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

from .reverse import reverse_sentence, reverse_string


def test_reverse_sentence():
"""
Tests the reverse_sentence function to ensure it correctly reverses each word in a sentence.
Test cases:
- "hello world" should be reversed to "olleh dlrow"
- "Python is fun" should be reversed to "nohtyP si nuf"
- "a b c" should remain "a b c" as each character is a single word
"""
assert reverse_sentence("hello world") == "olleh dlrow"
assert reverse_sentence("Python is fun") == "nohtyP si nuf"
assert reverse_sentence("a b c") == "a b c"

def test_reverse_sentence_error():
assert reverse_sentence("") == "Error: Input is None"
assert reverse_sentence(None) == "Error: Input is None"


def test_reverse_string():
assert reverse_string("hello") == "olleh"
assert reverse_string("Python") == "nohtyP"
# this test specifically does not cover the error cases
44 changes: 44 additions & 0 deletions python_files/tests/pytestadapter/helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,26 @@ def runner_with_cwd(args: List[str], path: pathlib.Path) -> Optional[List[Dict[s
return runner_with_cwd_env(args, path, {})


def split_array_at_item(arr: List[str], item: str) -> Tuple[List[str], List[str]]:
"""
Splits an array into two subarrays at the specified item.
Args:
arr (List[str]): The array to be split.
item (str): The item at which to split the array.
Returns:
Tuple[List[str], List[str]]: A tuple containing two subarrays. The first subarray includes the item and all elements before it. The second subarray includes all elements after the item. If the item is not found, the first subarray is the original array and the second subarray is empty.
"""
if item in arr:
index = arr.index(item)
before = arr[: index + 1]
after = arr[index + 1 :]
return before, after
else:
return arr, []


def runner_with_cwd_env(
args: List[str], path: pathlib.Path, env_add: Dict[str, str]
) -> Optional[List[Dict[str, Any]]]:
Expand All @@ -217,10 +237,34 @@ def runner_with_cwd_env(
# If we are running Django, generate a unittest-specific pipe name.
process_args = [sys.executable, *args]
pipe_name = generate_random_pipe_name("unittest-discovery-test")
elif "_TEST_VAR_UNITTEST" in env_add:
before_args, after_ids = split_array_at_item(args, "*test*.py")
process_args = [sys.executable, *before_args]
pipe_name = generate_random_pipe_name("unittest-execution-test")
test_ids_pipe = os.fspath(
script_dir / "tests" / "unittestadapter" / ".data" / "coverage_ex" / "10943021.txt"
)
env_add.update({"RUN_TEST_IDS_PIPE": test_ids_pipe})
test_ids_arr = after_ids
with open(test_ids_pipe, "w") as f: # noqa: PTH123
f.write("\n".join(test_ids_arr))
else:
process_args = [sys.executable, "-m", "pytest", "-p", "vscode_pytest", "-s", *args]
pipe_name = generate_random_pipe_name("pytest-discovery-test")

if "COVERAGE_ENABLED" in env_add and "_TEST_VAR_UNITTEST" not in env_add:
process_args = [
sys.executable,
"-m",
"pytest",
"-p",
"vscode_pytest",
"--cov=.",
"--cov-branch",
"-s",
*args,
]

# Generate pipe name, pipe name specific per OS type.

# Windows design
Expand Down
50 changes: 50 additions & 0 deletions python_files/tests/pytestadapter/test_coverage.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import os
import pathlib
import sys

script_dir = pathlib.Path(__file__).parent.parent
sys.path.append(os.fspath(script_dir))

from .helpers import ( # noqa: E402
TEST_DATA_PATH,
runner_with_cwd_env,
)


def test_simple_pytest_coverage():
"""
Test coverage payload is correct for simple pytest example. Output of coverage run is below.
Name Stmts Miss Branch BrPart Cover
---------------------------------------------------
__init__.py 0 0 0 0 100%
reverse.py 13 3 8 2 76%
test_reverse.py 11 0 0 0 100%
---------------------------------------------------
TOTAL 24 3 8 2 84%
"""
args = []
env_add = {"COVERAGE_ENABLED": "True"}
cov_folder_path = TEST_DATA_PATH / "coverage_gen"
actual = runner_with_cwd_env(args, cov_folder_path, env_add)
assert actual
coverage = actual[-1]
assert coverage
results = coverage["result"]
assert results
assert len(results) == 3
focal_function_coverage = results.get(os.fspath(TEST_DATA_PATH / "coverage_gen" / "reverse.py"))
assert focal_function_coverage
assert focal_function_coverage.get("lines_covered") is not None
assert focal_function_coverage.get("lines_missed") is not None
assert set(focal_function_coverage.get("lines_covered")) == {4, 5, 7, 9, 10, 11, 12, 13, 14, 17}
assert set(focal_function_coverage.get("lines_missed")) == {18, 19, 6}
assert (
focal_function_coverage.get("executed_branches") > 0
), "executed_branches are a number greater than 0."
assert (
focal_function_coverage.get("total_branches") > 0
), "total_branches are a number greater than 0."
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
14 changes: 14 additions & 0 deletions python_files/tests/unittestadapter/.data/coverage_ex/reverse.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

def reverse_string(s):
if s is None or s == "":
return "Error: Input is None"
return s[::-1]

def reverse_sentence(sentence):
if sentence is None or sentence == "":
return "Error: Input is None"
words = sentence.split()
reversed_words = [reverse_string(word) for word in words]
return " ".join(reversed_words)
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

import unittest
from reverse import reverse_sentence, reverse_string

class TestReverseFunctions(unittest.TestCase):

def test_reverse_sentence(self):
"""
Tests the reverse_sentence function to ensure it correctly reverses each word in a sentence.
Test cases:
- "hello world" should be reversed to "olleh dlrow"
- "Python is fun" should be reversed to "nohtyP si nuf"
- "a b c" should remain "a b c" as each character is a single word
"""
self.assertEqual(reverse_sentence("hello world"), "olleh dlrow")
self.assertEqual(reverse_sentence("Python is fun"), "nohtyP si nuf")
self.assertEqual(reverse_sentence("a b c"), "a b c")

def test_reverse_sentence_error(self):
self.assertEqual(reverse_sentence(""), "Error: Input is None")
self.assertEqual(reverse_sentence(None), "Error: Input is None")

def test_reverse_string(self):
self.assertEqual(reverse_string("hello"), "olleh")
self.assertEqual(reverse_string("Python"), "nohtyP")
# this test specifically does not cover the error cases

if __name__ == '__main__':
unittest.main()
57 changes: 57 additions & 0 deletions python_files/tests/unittestadapter/test_coverage.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

import os
import pathlib
import sys

sys.path.append(os.fspath(pathlib.Path(__file__).parent))

python_files_path = pathlib.Path(__file__).parent.parent.parent
sys.path.insert(0, os.fspath(python_files_path))
sys.path.insert(0, os.fspath(python_files_path / "lib" / "python"))

from tests.pytestadapter import helpers # noqa: E402

TEST_DATA_PATH = pathlib.Path(__file__).parent / ".data"


def test_basic_coverage():
"""This test runs on a simple django project with three tests, two of which pass and one that fails."""
coverage_ex_folder: pathlib.Path = TEST_DATA_PATH / "coverage_ex"
execution_script: pathlib.Path = python_files_path / "unittestadapter" / "execution.py"
test_ids = [
"test_reverse.TestReverseFunctions.test_reverse_sentence",
"test_reverse.TestReverseFunctions.test_reverse_sentence_error",
"test_reverse.TestReverseFunctions.test_reverse_string",
]
argv = [os.fsdecode(execution_script), "--udiscovery", "-vv", "-s", ".", "-p", "*test*.py"]
argv = argv + test_ids

actual = helpers.runner_with_cwd_env(
argv,
coverage_ex_folder,
{"COVERAGE_ENABLED": os.fspath(coverage_ex_folder), "_TEST_VAR_UNITTEST": "True"},
)

assert actual
coverage = actual[-1]
assert coverage
results = coverage["result"]
assert results
assert len(results) == 3
focal_function_coverage = results.get(os.fspath(TEST_DATA_PATH / "coverage_ex" / "reverse.py"))
assert focal_function_coverage
assert focal_function_coverage.get("lines_covered") is not None
assert focal_function_coverage.get("lines_missed") is not None
assert set(focal_function_coverage.get("lines_covered")) == {4, 5, 7, 9, 10, 11, 12, 13, 14}
assert set(focal_function_coverage.get("lines_missed")) == {6}
assert (
focal_function_coverage.get("executed_branches") > 0
), "executed_branches are a number greater than 0."
assert (
focal_function_coverage.get("total_branches") > 0
), "total_branches are a number greater than 0."
Loading

0 comments on commit 8268131

Please sign in to comment.