Merge pull request #136 from kondratyev-nv/fix_output_capture

Fix output capture
kondratyev-nv · Mar 27, 2020 · 764b01d · 764b01d
2 parents caca2e4 + cd5dbe5
commit 764b01d
Show file tree

Hide file tree

Showing 12 changed files with 84 additions and 68 deletions.
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -14,7 +14,8 @@
       ],
       "preLaunchTask": "npm: build",
       "env": {
-        "SOME_PROCESS_VARIABLE": "HelloFromProcessEnv"
+        "SOME_PROCESS_VARIABLE": "HelloFromProcessEnv",
+        "VSCODE_PYTHON": "python3"
       }
     },
     {
@@ -38,7 +39,8 @@
       ],
       "preLaunchTask": "npm: build",
       "env": {
-        "SOME_PROCESS_VARIABLE": "HelloFromProcessEnv"
+        "SOME_PROCESS_VARIABLE": "HelloFromProcessEnv",
+        "VSCODE_PYTHON": "python3"
       }
     }
   ]

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -72,12 +72,14 @@
   "devDependencies": {
     "@types/argparse": "^1.0.38",
     "@types/chai": "^4.2.11",
+    "@types/chai-string": "^1.4.2",
     "@types/js-base64": "^2.3.1",
     "@types/mocha": "^7.0.2",
     "@types/node": "^13.9.3",
     "@types/tmp": "^0.1.0",
     "@types/xml2js": "^0.4.5",
     "chai": "^4.2.0",
+    "chai-string": "^1.5.0",
     "cross-env": "^7.0.2",
     "mocha": "^7.1.1",
     "rimraf": "^3.0.2",

diff --git a/test/helpers.ts b/test/helpers.ts
@@ -9,6 +9,7 @@ import {
     IWorkspaceConfiguration
 } from '../src/configuration/workspaceConfiguration';
 import { ILogger } from '../src/logging/logger';
+import { getPythonExecutable } from './testConfiguration';
 
 export function logger(): ILogger {
     return {
@@ -60,7 +61,8 @@ export function findWorkspaceFolder(folder: string): vscode.WorkspaceFolder | un
     return vscode.workspace.workspaceFolders!.find(f => f.name === folder);
 }
 
-export function createPytestConfiguration(python: string, folder: string, args?: string[]): IWorkspaceConfiguration {
+export function createPytestConfiguration(folder: string, args?: string[]): IWorkspaceConfiguration {
+    const python = getPythonExecutable();
     const wf = findWorkspaceFolder(folder)!;
     return new PlaceholderAwareWorkspaceConfiguration({
         pythonPath(): string {
@@ -84,7 +86,8 @@ export function createPytestConfiguration(python: string, folder: string, args?:
     }, wf, logger());
 }
 
-export function createUnittestConfiguration(python: string, folder: string): IWorkspaceConfiguration {
+export function createUnittestConfiguration(folder: string): IWorkspaceConfiguration {
+    const python = getPythonExecutable();
     const wf = findWorkspaceFolder(folder)!;
     return new PlaceholderAwareWorkspaceConfiguration({
         pythonPath(): string {

diff --git a/test/index.ts b/test/index.ts
@@ -10,36 +10,22 @@
 // to report the results back to the caller. When the tests are finished, return
 // a possible error to the callback or null if none.
 
-import * as path from 'path';
 import * as testRunner from 'vscode/lib/testrunner';
-
+import { getReporter, getPythonExecutable } from './testConfiguration';
 import { runScript } from '../src/pythonRunner';
+import * as chai from 'chai';
+import * as chaiString from 'chai-string';
 
-function getReporter() {
-    if (!process.env.JUNIT_REPORTER_ENABLED) {
-        console.log('JUNIT_REPORTER_ENABLED variable is not defined, using default reporter');
-        return {};
-    }
-
-    const testResultsFile = path.resolve(
-        path.join(process.env.JUNIT_REPORTER_RESULT_DIRECTORY || './', 'test-results.xml')
-    );
-    console.log(`Results will be placed in ${testResultsFile}`);
-    return {
-        reporter: 'xunit',
-        reporterOptions: {
-            output: testResultsFile,
-        },
-    };
-}
+chai.use(chaiString.default);
 
 runScript({
     script: 'from __future__ import print_function; import sys; print(sys.executable, sys.version)',
-    pythonPath: 'python',
+    pythonPath: getPythonExecutable(),
     environment: {},
 }).complete().then(({ output }) => console.log(`Using python ${output}`));
 
 const reporter = getReporter();
+console.log(`Using ${reporter.reporter || 'default'} reporter`);
 testRunner.configure({
     ...{
         ui: 'tdd',       // the TDD UI is being used in extension.test.ts (suite, test, etc.)

diff --git a/test/pytestArguments.test.ts b/test/pytestArguments.test.ts
@@ -8,7 +8,6 @@ import { createPytestConfiguration, extractExpectedState, findTestSuiteByLabel,
 
 suite('Pytest test discovery with additional arguments', async () => {
     const config: IWorkspaceConfiguration = createPytestConfiguration(
-        'python',
         'pytest',
         [
             '--rootdir=test/inner_tests',
@@ -45,7 +44,6 @@ suite('Pytest test discovery with additional arguments', async () => {
 
 suite('Run pytest tests with additional arguments', () => {
     const config: IWorkspaceConfiguration = createPytestConfiguration(
-        'python',
         'pytest',
         [
             '--rootdir',
@@ -130,7 +128,6 @@ suite('Filter pytest tests by mark arguments', () => {
 
     test('should discover only tests with specific mark', async () => {
         const config: IWorkspaceConfiguration = createPytestConfiguration(
-            'python',
             'pytest',
             [
                 '--ignore=test/import_error_tests',
@@ -147,7 +144,6 @@ suite('Filter pytest tests by mark arguments', () => {
 
     test('should run only tests with specific mark', async () => {
         const config: IWorkspaceConfiguration = createPytestConfiguration(
-            'python',
             'pytest',
             [
                 '--ignore=test/import_error_tests',
@@ -170,7 +166,6 @@ suite('Filter pytest tests by mark arguments', () => {
 
     test('should not run tests with specific mark', async () => {
         const config: IWorkspaceConfiguration = createPytestConfiguration(
-            'python',
             'pytest',
             [
                 '--ignore=test/import_error_tests',
@@ -194,7 +189,6 @@ suite('Filter pytest tests by mark arguments', () => {
 
 suite('Pytest tests with additional positional arguments', () => {
     const config: IWorkspaceConfiguration = createPytestConfiguration(
-        'python',
         'pytest',
         [
             '--rootdir',

diff --git a/test/pytestGeneral.test.ts b/test/pytestGeneral.test.ts
@@ -8,7 +8,6 @@ import { createPytestConfiguration, extractExpectedState, findTestSuiteByLabel,
 
 suite('Pytest test discovery with errors', async () => {
     const config: IWorkspaceConfiguration = createPytestConfiguration(
-        'python',
         'pytest'
     );
     const runner = new PytestTestRunner('some-id', logger());
@@ -36,7 +35,6 @@ suite('Pytest test discovery with errors', async () => {
 
 suite('Run pytest tests with discovery errors', () => {
     const config: IWorkspaceConfiguration = createPytestConfiguration(
-        'python',
         'pytest'
     );
     const runner = new PytestTestRunner('some-id', logger());
@@ -59,7 +57,6 @@ suite('Run pytest tests with discovery errors', () => {
 
 suite('Pytest test discovery', async () => {
     const config: IWorkspaceConfiguration = createPytestConfiguration(
-        'python',
         'pytest',
         ['--ignore=test/import_error_tests']
     );
@@ -72,7 +69,7 @@ suite('Pytest test discovery', async () => {
 
     test('should not return root suite when there is no tests', async () => {
         const configForEmptySuiteCollection: IWorkspaceConfiguration = createPytestConfiguration(
-            'python', 'python_extension_configured_pytest'
+            'python_extension_configured_pytest'
         );
         expect(runner).to.be.not.null;
         const { suite: mainSuite, errors } = await runner.load(configForEmptySuiteCollection);
@@ -110,7 +107,6 @@ suite('Pytest test discovery', async () => {
 
 suite('Run pytest tests', () => {
     const config: IWorkspaceConfiguration = createPytestConfiguration(
-        'python',
         'pytest',
         ['--ignore=test/import_error_tests']
     );
@@ -215,24 +211,19 @@ suite('Run pytest tests', () => {
         });
     });
 
-    [
-        'test_one_plus_two_is_three_passed',
-        'test_two_plus_two_is_five_failed'
-    ].forEach(testMethod => {
-        test.skip(`should capture output from ${testMethod} test`, async () => {
-            const { suite: mainSuite, errors } = await runner.load(config);
-            expect(errors).to.be.empty;
-            expect(mainSuite).to.be.not.undefined;
-            const suite = findTestSuiteByLabel(mainSuite!, testMethod);
-            expect(suite).to.be.not.undefined;
-            const states = await runner.run(config, suite!.id);
-            expect(states).to.be.not.empty;
-            states.forEach(state => {
-                const expectedState = extractExpectedState(state.test as string);
-                expect(state.state).to.be.eq(expectedState);
-                expect(state.message).to.be.not.empty;
-                expect(state.message!.startsWith(`Hello from ${testMethod}`)).to.be.true;
-            });
+    test('should capture output from failing test', async () => {
+        const { suite: mainSuite, errors } = await runner.load(config);
+        expect(errors).to.be.empty;
+        expect(mainSuite).to.be.not.undefined;
+        const suite = findTestSuiteByLabel(mainSuite!, 'test_two_plus_two_is_five_failed');
+        expect(suite).to.be.not.undefined;
+        const states = await runner.run(config, suite!.id);
+        expect(states).to.be.not.empty;
+        states.forEach(state => {
+            const expectedState = extractExpectedState(state.test as string);
+            expect(state.state).to.be.eq(expectedState);
+            expect(state.message).to.be.not.empty;
+            expect(state.message).contains('Hello from test_two_plus_two_is_five_failed');
         });
     });
 

diff --git a/test/pythonTestAdapter.test.ts b/test/pythonTestAdapter.test.ts
@@ -22,7 +22,7 @@ import {
     {
         label: 'unittest',
         runner: new UnittestTestRunner('first-id', logger()),
-        configuration: createUnittestConfiguration('python', 'unittest'),
+        configuration: createUnittestConfiguration('unittest'),
         testsToRun: [
             'test_basic_two_plus_one_is_three_passed',
             'test_basic_two_plus_two_is_five_failed',
@@ -41,7 +41,6 @@ import {
         label: 'pytest',
         runner: new PytestTestRunner('second-id', logger()),
         configuration: createPytestConfiguration(
-            'python',
             'pytest',
             ['--ignore=test/import_error_tests']
         ),
@@ -170,7 +169,6 @@ suite('Adapter events with pytest runner and invalid files during discovery', ()
     const configurationFactory: IConfigurationFactory = {
         get(_: vscode.WorkspaceFolder): IWorkspaceConfiguration {
             return createPytestConfiguration(
-                'python',
                 'pytest'
             );
         },

diff --git a/test/testCancellation.test.ts b/test/testCancellation.test.ts
@@ -18,13 +18,13 @@ import {
     {
         label: 'unittest',
         runner: new UnittestTestRunner('first-id', logger()),
-        configuration: createUnittestConfiguration('python', 'unittest_test_cancellation'),
+        configuration: createUnittestConfiguration('unittest_test_cancellation'),
         allowNoTestCompleted: false,
     },
     {
         label: 'pytest',
         runner: new PytestTestRunner('second-id', logger()),
-        configuration: createPytestConfiguration('python', 'pytest_test_cancellation'),
+        configuration: createPytestConfiguration('pytest_test_cancellation'),
         allowNoTestCompleted: os.platform() === 'win32',
     }
 ].forEach(({ label, runner, configuration, allowNoTestCompleted }) => {

diff --git a/test/testConfiguration.ts b/test/testConfiguration.ts
@@ -0,0 +1,25 @@
+
+import * as path from 'path';
+
+export function getReporter() {
+    if (!process.env.JUNIT_REPORTER_ENABLED) {
+        return {};
+    }
+
+    const testResultsFile = path.resolve(
+        path.join(process.env.JUNIT_REPORTER_RESULT_DIRECTORY || './', 'test-results.xml')
+    );
+    return {
+        reporter: 'xunit',
+        reporterOptions: {
+            output: testResultsFile,
+        },
+    };
+}
+
+export function getPythonExecutable() {
+    if (!process.env.VSCODE_PYTHON) {
+        return 'python';
+    }
+    return process.env.VSCODE_PYTHON;
+}
diff --git a/test/test_samples/pytest/.vscode/settings.json b/test/test_samples/pytest/.vscode/settings.json
@@ -2,5 +2,5 @@
    "python.pythonPath": "python",
    "python.envFile": "../.env",
    "python.testing.pytestEnabled": true,
-   "python.testing.pytestArgs": ["--rootdir", "test/inner_tests", "--doctest-module"]
+   "python.testing.pytestArgs": ["--rootdir", "test/inner_tests", "--doctest-modules"]
 }
diff --git a/test/unittestGeneral.test.ts b/test/unittestGeneral.test.ts
@@ -7,7 +7,7 @@ import { UnittestTestRunner } from '../src/unittest/unittestTestRunner';
 import { createUnittestConfiguration, extractExpectedState, findTestSuiteByLabel, logger } from './helpers';
 
 suite('Unittest test discovery', () => {
-    const config: IWorkspaceConfiguration = createUnittestConfiguration('python', 'unittest');
+    const config: IWorkspaceConfiguration = createUnittestConfiguration('unittest');
     const runner = new UnittestTestRunner('some-id', logger());
 
     test('should set runner id on initialization', () => {
@@ -17,7 +17,7 @@ suite('Unittest test discovery', () => {
 
     test('should not return root suite when there is no tests', async () => {
         const configForEmptySuiteCollection: IWorkspaceConfiguration = createUnittestConfiguration(
-            'python', 'python_extension_configured_unittest'
+            'python_extension_configured_unittest'
         );
         const { suite: mainSuite, errors } = await runner.load(configForEmptySuiteCollection);
         expect(errors).to.be.empty;
@@ -51,7 +51,7 @@ suite('Unittest test discovery', () => {
 });
 
 suite('Run unittest tests', () => {
-    const config: IWorkspaceConfiguration = createUnittestConfiguration('python', 'unittest');
+    const config: IWorkspaceConfiguration = createUnittestConfiguration('unittest');
     const runner = new UnittestTestRunner('some-id', logger());
 
     test('should run all tests', async () => {