From b2ba96105bb246b453a92e4befe1a25384c2abed Mon Sep 17 00:00:00 2001 From: Nikolay Kondratyev <4085884+kondratyev-nv@users.noreply.github.com> Date: Sat, 21 Mar 2020 20:23:29 +0300 Subject: [PATCH 1/4] Add support for custom python executable --- test/helpers.ts | 7 ++++-- test/index.ts | 24 +++--------------- test/pytestArguments.test.ts | 6 ----- test/pytestGeneral.test.ts | 6 +---- test/pythonTestAdapter.test.ts | 4 +-- test/testCancellation.test.ts | 4 +-- test/testConfiguration.ts | 25 +++++++++++++++++++ .../test_samples/pytest/.vscode/settings.json | 2 +- test/unittestGeneral.test.ts | 6 ++--- 9 files changed, 41 insertions(+), 43 deletions(-) create mode 100644 test/testConfiguration.ts diff --git a/test/helpers.ts b/test/helpers.ts index 7481bba..871f2fd 100644 --- a/test/helpers.ts +++ b/test/helpers.ts @@ -9,6 +9,7 @@ import { IWorkspaceConfiguration } from '../src/configuration/workspaceConfiguration'; import { ILogger } from '../src/logging/logger'; +import { getPythonExecutable } from './testConfiguration'; export function logger(): ILogger { return { @@ -60,7 +61,8 @@ export function findWorkspaceFolder(folder: string): vscode.WorkspaceFolder | un return vscode.workspace.workspaceFolders!.find(f => f.name === folder); } -export function createPytestConfiguration(python: string, folder: string, args?: string[]): IWorkspaceConfiguration { +export function createPytestConfiguration(folder: string, args?: string[]): IWorkspaceConfiguration { + const python = getPythonExecutable(); const wf = findWorkspaceFolder(folder)!; return new PlaceholderAwareWorkspaceConfiguration({ pythonPath(): string { @@ -84,7 +86,8 @@ export function createPytestConfiguration(python: string, folder: string, args?: }, wf, logger()); } -export function createUnittestConfiguration(python: string, folder: string): IWorkspaceConfiguration { +export function createUnittestConfiguration(folder: string): IWorkspaceConfiguration { + const python = getPythonExecutable(); const wf = findWorkspaceFolder(folder)!; return new PlaceholderAwareWorkspaceConfiguration({ pythonPath(): string { diff --git a/test/index.ts b/test/index.ts index 948ee81..b51447f 100644 --- a/test/index.ts +++ b/test/index.ts @@ -10,36 +10,18 @@ // to report the results back to the caller. When the tests are finished, return // a possible error to the callback or null if none. -import * as path from 'path'; import * as testRunner from 'vscode/lib/testrunner'; - +import { getReporter, getPythonExecutable } from './testConfiguration'; import { runScript } from '../src/pythonRunner'; -function getReporter() { - if (!process.env.JUNIT_REPORTER_ENABLED) { - console.log('JUNIT_REPORTER_ENABLED variable is not defined, using default reporter'); - return {}; - } - - const testResultsFile = path.resolve( - path.join(process.env.JUNIT_REPORTER_RESULT_DIRECTORY || './', 'test-results.xml') - ); - console.log(`Results will be placed in ${testResultsFile}`); - return { - reporter: 'xunit', - reporterOptions: { - output: testResultsFile, - }, - }; -} - runScript({ script: 'from __future__ import print_function; import sys; print(sys.executable, sys.version)', - pythonPath: 'python', + pythonPath: getPythonExecutable(), environment: {}, }).complete().then(({ output }) => console.log(`Using python ${output}`)); const reporter = getReporter(); +console.log(`Using ${reporter.reporter || 'default'} reporter`); testRunner.configure({ ...{ ui: 'tdd', // the TDD UI is being used in extension.test.ts (suite, test, etc.) diff --git a/test/pytestArguments.test.ts b/test/pytestArguments.test.ts index 7956eff..fe4d421 100644 --- a/test/pytestArguments.test.ts +++ b/test/pytestArguments.test.ts @@ -8,7 +8,6 @@ import { createPytestConfiguration, extractExpectedState, findTestSuiteByLabel, suite('Pytest test discovery with additional arguments', async () => { const config: IWorkspaceConfiguration = createPytestConfiguration( - 'python', 'pytest', [ '--rootdir=test/inner_tests', @@ -45,7 +44,6 @@ suite('Pytest test discovery with additional arguments', async () => { suite('Run pytest tests with additional arguments', () => { const config: IWorkspaceConfiguration = createPytestConfiguration( - 'python', 'pytest', [ '--rootdir', @@ -130,7 +128,6 @@ suite('Filter pytest tests by mark arguments', () => { test('should discover only tests with specific mark', async () => { const config: IWorkspaceConfiguration = createPytestConfiguration( - 'python', 'pytest', [ '--ignore=test/import_error_tests', @@ -147,7 +144,6 @@ suite('Filter pytest tests by mark arguments', () => { test('should run only tests with specific mark', async () => { const config: IWorkspaceConfiguration = createPytestConfiguration( - 'python', 'pytest', [ '--ignore=test/import_error_tests', @@ -170,7 +166,6 @@ suite('Filter pytest tests by mark arguments', () => { test('should not run tests with specific mark', async () => { const config: IWorkspaceConfiguration = createPytestConfiguration( - 'python', 'pytest', [ '--ignore=test/import_error_tests', @@ -194,7 +189,6 @@ suite('Filter pytest tests by mark arguments', () => { suite('Pytest tests with additional positional arguments', () => { const config: IWorkspaceConfiguration = createPytestConfiguration( - 'python', 'pytest', [ '--rootdir', diff --git a/test/pytestGeneral.test.ts b/test/pytestGeneral.test.ts index 846d1d7..9449d8b 100644 --- a/test/pytestGeneral.test.ts +++ b/test/pytestGeneral.test.ts @@ -8,7 +8,6 @@ import { createPytestConfiguration, extractExpectedState, findTestSuiteByLabel, suite('Pytest test discovery with errors', async () => { const config: IWorkspaceConfiguration = createPytestConfiguration( - 'python', 'pytest' ); const runner = new PytestTestRunner('some-id', logger()); @@ -36,7 +35,6 @@ suite('Pytest test discovery with errors', async () => { suite('Run pytest tests with discovery errors', () => { const config: IWorkspaceConfiguration = createPytestConfiguration( - 'python', 'pytest' ); const runner = new PytestTestRunner('some-id', logger()); @@ -59,7 +57,6 @@ suite('Run pytest tests with discovery errors', () => { suite('Pytest test discovery', async () => { const config: IWorkspaceConfiguration = createPytestConfiguration( - 'python', 'pytest', ['--ignore=test/import_error_tests'] ); @@ -72,7 +69,7 @@ suite('Pytest test discovery', async () => { test('should not return root suite when there is no tests', async () => { const configForEmptySuiteCollection: IWorkspaceConfiguration = createPytestConfiguration( - 'python', 'python_extension_configured_pytest' + 'python_extension_configured_pytest' ); expect(runner).to.be.not.null; const { suite: mainSuite, errors } = await runner.load(configForEmptySuiteCollection); @@ -110,7 +107,6 @@ suite('Pytest test discovery', async () => { suite('Run pytest tests', () => { const config: IWorkspaceConfiguration = createPytestConfiguration( - 'python', 'pytest', ['--ignore=test/import_error_tests'] ); diff --git a/test/pythonTestAdapter.test.ts b/test/pythonTestAdapter.test.ts index 634721b..76ea47b 100644 --- a/test/pythonTestAdapter.test.ts +++ b/test/pythonTestAdapter.test.ts @@ -22,7 +22,7 @@ import { { label: 'unittest', runner: new UnittestTestRunner('first-id', logger()), - configuration: createUnittestConfiguration('python', 'unittest'), + configuration: createUnittestConfiguration('unittest'), testsToRun: [ 'test_basic_two_plus_one_is_three_passed', 'test_basic_two_plus_two_is_five_failed', @@ -41,7 +41,6 @@ import { label: 'pytest', runner: new PytestTestRunner('second-id', logger()), configuration: createPytestConfiguration( - 'python', 'pytest', ['--ignore=test/import_error_tests'] ), @@ -170,7 +169,6 @@ suite('Adapter events with pytest runner and invalid files during discovery', () const configurationFactory: IConfigurationFactory = { get(_: vscode.WorkspaceFolder): IWorkspaceConfiguration { return createPytestConfiguration( - 'python', 'pytest' ); }, diff --git a/test/testCancellation.test.ts b/test/testCancellation.test.ts index d63a8e1..19b3409 100644 --- a/test/testCancellation.test.ts +++ b/test/testCancellation.test.ts @@ -18,13 +18,13 @@ import { { label: 'unittest', runner: new UnittestTestRunner('first-id', logger()), - configuration: createUnittestConfiguration('python', 'unittest_test_cancellation'), + configuration: createUnittestConfiguration('unittest_test_cancellation'), allowNoTestCompleted: false, }, { label: 'pytest', runner: new PytestTestRunner('second-id', logger()), - configuration: createPytestConfiguration('python', 'pytest_test_cancellation'), + configuration: createPytestConfiguration('pytest_test_cancellation'), allowNoTestCompleted: os.platform() === 'win32', } ].forEach(({ label, runner, configuration, allowNoTestCompleted }) => { diff --git a/test/testConfiguration.ts b/test/testConfiguration.ts new file mode 100644 index 0000000..959f688 --- /dev/null +++ b/test/testConfiguration.ts @@ -0,0 +1,25 @@ + +import * as path from 'path'; + +export function getReporter() { + if (!process.env.JUNIT_REPORTER_ENABLED) { + return {}; + } + + const testResultsFile = path.resolve( + path.join(process.env.JUNIT_REPORTER_RESULT_DIRECTORY || './', 'test-results.xml') + ); + return { + reporter: 'xunit', + reporterOptions: { + output: testResultsFile, + }, + }; +} + +export function getPythonExecutable() { + if (!process.env.VSCODE_PYTHON) { + return 'python'; + } + return process.env.VSCODE_PYTHON; +} diff --git a/test/test_samples/pytest/.vscode/settings.json b/test/test_samples/pytest/.vscode/settings.json index 93502f8..d1a1fc4 100644 --- a/test/test_samples/pytest/.vscode/settings.json +++ b/test/test_samples/pytest/.vscode/settings.json @@ -2,5 +2,5 @@ "python.pythonPath": "python", "python.envFile": "../.env", "python.testing.pytestEnabled": true, - "python.testing.pytestArgs": ["--rootdir", "test/inner_tests", "--doctest-module"] + "python.testing.pytestArgs": ["--rootdir", "test/inner_tests", "--doctest-modules"] } diff --git a/test/unittestGeneral.test.ts b/test/unittestGeneral.test.ts index cd9eb35..d73f3b7 100644 --- a/test/unittestGeneral.test.ts +++ b/test/unittestGeneral.test.ts @@ -7,7 +7,7 @@ import { UnittestTestRunner } from '../src/unittest/unittestTestRunner'; import { createUnittestConfiguration, extractExpectedState, findTestSuiteByLabel, logger } from './helpers'; suite('Unittest test discovery', () => { - const config: IWorkspaceConfiguration = createUnittestConfiguration('python', 'unittest'); + const config: IWorkspaceConfiguration = createUnittestConfiguration('unittest'); const runner = new UnittestTestRunner('some-id', logger()); test('should set runner id on initialization', () => { @@ -17,7 +17,7 @@ suite('Unittest test discovery', () => { test('should not return root suite when there is no tests', async () => { const configForEmptySuiteCollection: IWorkspaceConfiguration = createUnittestConfiguration( - 'python', 'python_extension_configured_unittest' + 'python_extension_configured_unittest' ); const { suite: mainSuite, errors } = await runner.load(configForEmptySuiteCollection); expect(errors).to.be.empty; @@ -51,7 +51,7 @@ suite('Unittest test discovery', () => { }); suite('Run unittest tests', () => { - const config: IWorkspaceConfiguration = createUnittestConfiguration('python', 'unittest'); + const config: IWorkspaceConfiguration = createUnittestConfiguration('unittest'); const runner = new UnittestTestRunner('some-id', logger()); test('should run all tests', async () => { From 3bebaf8e3e426a0e228eccab3f72ad3a5fb5af4e Mon Sep 17 00:00:00 2001 From: Nikolay Kondratyev <4085884+kondratyev-nv@users.noreply.github.com> Date: Mon, 23 Mar 2020 10:17:08 +0300 Subject: [PATCH 2/4] chai-string plugin --- .vscode/launch.json | 6 ++++-- package-lock.json | 15 +++++++++++++++ package.json | 4 +++- src/pytest/pytestTestRunner.ts | 2 +- test/index.ts | 4 ++++ test/pytestGeneral.test.ts | 2 +- 6 files changed, 28 insertions(+), 5 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index db88640..b651fa4 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -14,7 +14,8 @@ ], "preLaunchTask": "npm: build", "env": { - "SOME_PROCESS_VARIABLE": "HelloFromProcessEnv" + "SOME_PROCESS_VARIABLE": "HelloFromProcessEnv", + "VSCODE_PYTHON": "python3" } }, { @@ -38,7 +39,8 @@ ], "preLaunchTask": "npm: build", "env": { - "SOME_PROCESS_VARIABLE": "HelloFromProcessEnv" + "SOME_PROCESS_VARIABLE": "HelloFromProcessEnv", + "VSCODE_PYTHON": "python3" } } ] diff --git a/package-lock.json b/package-lock.json index fc1c4c2..91a3f0a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -36,6 +36,15 @@ "integrity": "sha512-U1bQiWbln41Yo6EeHMr+34aUhvrMVyrhn9lYfPSpLTCrZlGxU4Rtn1bocX+0p2Fc/Jkd2FanCEXdw0WNfHHM0w==", "dev": true }, + "@types/chai-string": { + "version": "1.4.2", + "resolved": "https://registry.npmjs.org/@types/chai-string/-/chai-string-1.4.2.tgz", + "integrity": "sha512-ld/1hV5qcPRGuwlPdvRfvM3Ka/iofOk2pH4VkasK4b1JJP1LjNmWWn0LsISf6RRzyhVOvs93rb9tM09e+UuF8Q==", + "dev": true, + "requires": { + "@types/chai": "*" + } + }, "@types/js-base64": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/@types/js-base64/-/js-base64-2.3.1.tgz", @@ -274,6 +283,12 @@ "type-detect": "^4.0.5" } }, + "chai-string": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/chai-string/-/chai-string-1.5.0.tgz", + "integrity": "sha512-sydDC3S3pNAQMYwJrs6dQX0oBQ6KfIPuOZ78n7rocW0eJJlsHPh2t3kwW7xfwYA/1Bf6/arGtSUo16rxR2JFlw==", + "dev": true + }, "chalk": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", diff --git a/package.json b/package.json index 2dba9b7..7970898 100644 --- a/package.json +++ b/package.json @@ -72,12 +72,14 @@ "devDependencies": { "@types/argparse": "^1.0.38", "@types/chai": "^4.2.8", + "@types/chai-string": "^1.4.2", "@types/js-base64": "^2.3.1", "@types/mocha": "^7.0.1", "@types/node": "^13.7.0", "@types/tmp": "^0.1.0", "@types/xml2js": "^0.4.5", "chai": "^4.2.0", + "chai-string": "^1.5.0", "cross-env": "^7.0.0", "mocha": "^7.0.1", "rimraf": "^3.0.1", @@ -122,4 +124,4 @@ } } } -} \ No newline at end of file +} diff --git a/src/pytest/pytestTestRunner.ts b/src/pytest/pytestTestRunner.ts index 5f047f4..213f7f2 100644 --- a/src/pytest/pytestTestRunner.ts +++ b/src/pytest/pytestTestRunner.ts @@ -154,7 +154,7 @@ pytest.main(sys.argv[1:], plugins=[PythonTestExplorerDiscoveryOutputPlugin()])`; const additionalEnvironment = await EnvironmentVariablesLoader.load(config.envFile(), process.env, this.logger); const { file, cleanupCallback } = await this.createTemporaryFile(); - const runArguments = [`--junitxml=${file}`].concat( + const runArguments = [`--junitxml=${file}`, '-o', 'junit_logging=system-out'].concat( this.getRunArguments(test, config.getPytestConfiguration().pytestArguments)); this.logger.log('info', `Running pytest wrapper with arguments: ${runArguments}`); const testExecution = runScript({ diff --git a/test/index.ts b/test/index.ts index b51447f..d2b81ca 100644 --- a/test/index.ts +++ b/test/index.ts @@ -13,6 +13,10 @@ import * as testRunner from 'vscode/lib/testrunner'; import { getReporter, getPythonExecutable } from './testConfiguration'; import { runScript } from '../src/pythonRunner'; +import * as chai from 'chai'; +import * as chaiString from 'chai-string'; + +chai.use(chaiString.default); runScript({ script: 'from __future__ import print_function; import sys; print(sys.executable, sys.version)', diff --git a/test/pytestGeneral.test.ts b/test/pytestGeneral.test.ts index 9449d8b..6eb8b7c 100644 --- a/test/pytestGeneral.test.ts +++ b/test/pytestGeneral.test.ts @@ -227,7 +227,7 @@ suite('Run pytest tests', () => { const expectedState = extractExpectedState(state.test as string); expect(state.state).to.be.eq(expectedState); expect(state.message).to.be.not.empty; - expect(state.message!.startsWith(`Hello from ${testMethod}`)).to.be.true; + expect(state.message).startsWith(`Hello from ${testMethod}`); }); }); }); From 1aa17d3d51739603a57e3a3fe5b2ba4e3bc55b0c Mon Sep 17 00:00:00 2001 From: Nikolay Kondratyev <4085884+kondratyev-nv@users.noreply.github.com> Date: Wed, 25 Mar 2020 23:03:48 +0300 Subject: [PATCH 3/4] Fix pytest output capture test --- test/pytestGeneral.test.ts | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/test/pytestGeneral.test.ts b/test/pytestGeneral.test.ts index 5413402..4c3aabe 100644 --- a/test/pytestGeneral.test.ts +++ b/test/pytestGeneral.test.ts @@ -211,24 +211,19 @@ suite('Run pytest tests', () => { }); }); - [ - 'test_one_plus_two_is_three_passed', - 'test_two_plus_two_is_five_failed' - ].forEach(testMethod => { - test.skip(`should capture output from ${testMethod} test`, async () => { - const { suite: mainSuite, errors } = await runner.load(config); - expect(errors).to.be.empty; - expect(mainSuite).to.be.not.undefined; - const suite = findTestSuiteByLabel(mainSuite!, testMethod); - expect(suite).to.be.not.undefined; - const states = await runner.run(config, suite!.id); - expect(states).to.be.not.empty; - states.forEach(state => { - const expectedState = extractExpectedState(state.test as string); - expect(state.state).to.be.eq(expectedState); - expect(state.message).to.be.not.empty; - expect(state.message).startsWith(`Hello from ${testMethod}`); - }); + test('should capture output from failing test', async () => { + const { suite: mainSuite, errors } = await runner.load(config); + expect(errors).to.be.empty; + expect(mainSuite).to.be.not.undefined; + const suite = findTestSuiteByLabel(mainSuite!, 'test_two_plus_two_is_five_failed'); + expect(suite).to.be.not.undefined; + const states = await runner.run(config, suite!.id); + expect(states).to.be.not.empty; + states.forEach(state => { + const expectedState = extractExpectedState(state.test as string); + expect(state.state).to.be.eq(expectedState); + expect(state.message).to.be.not.empty; + expect(state.message).contains('Hello from test_two_plus_two_is_five_failed'); }); }); From cd5dbe5a3c8d3f4874c38ee1ed13d2373abe0046 Mon Sep 17 00:00:00 2001 From: Nikolay Kondratyev <4085884+kondratyev-nv@users.noreply.github.com> Date: Thu, 26 Mar 2020 12:24:00 +0300 Subject: [PATCH 4/4] Remove unnecessary argument --- src/pytest/pytestTestRunner.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pytest/pytestTestRunner.ts b/src/pytest/pytestTestRunner.ts index 213f7f2..5f047f4 100644 --- a/src/pytest/pytestTestRunner.ts +++ b/src/pytest/pytestTestRunner.ts @@ -154,7 +154,7 @@ pytest.main(sys.argv[1:], plugins=[PythonTestExplorerDiscoveryOutputPlugin()])`; const additionalEnvironment = await EnvironmentVariablesLoader.load(config.envFile(), process.env, this.logger); const { file, cleanupCallback } = await this.createTemporaryFile(); - const runArguments = [`--junitxml=${file}`, '-o', 'junit_logging=system-out'].concat( + const runArguments = [`--junitxml=${file}`].concat( this.getRunArguments(test, config.getPytestConfiguration().pytestArguments)); this.logger.log('info', `Running pytest wrapper with arguments: ${runArguments}`); const testExecution = runScript({