From 9f5f1a8fe3f1c4827bf933ca8dbe4d25a6f611e4 Mon Sep 17 00:00:00 2001 From: Klaus Oberhofer Date: Sat, 19 Feb 2022 20:42:36 +0100 Subject: [PATCH 01/18] add support for behave BDD test framework --- README.md | 6 +- package.json | 4 +- src/behave/behaveTestJsonParser.ts | 108 ++++++++++ src/behave/behaveTestRunner.ts | 184 ++++++++++++++++++ .../placeholderAwareWorkspaceConfiguration.ts | 14 ++ ...honExtensionAwareWorkspaceConfiguration.ts | 5 + .../vscodeWorkspaceConfiguration.ts | 25 +++ src/configuration/workspaceConfiguration.ts | 8 + src/main.ts | 8 +- src/pythonTestAdapter.ts | 3 + test/test_samples/behave/behave_runner.bat | 9 + test/test_samples/behave/behave_runner.sh | 10 + .../behave/features/steps/tutorial.py | 18 ++ .../behave/features/tutorial.feature | 7 + .../samples-workspace.code-workspace | 3 + test/tests/environmentParsing.test.ts | 16 +- ...eholderAwareWorkspaceConfiguration.test.ts | 50 +++++ test/tests/pytestScript.test.ts | 4 + test/tests/unittestGeneral.test.ts | 3 + test/utils/helpers.ts | 18 ++ 20 files changed, 499 insertions(+), 4 deletions(-) create mode 100644 src/behave/behaveTestJsonParser.ts create mode 100644 src/behave/behaveTestRunner.ts create mode 100755 test/test_samples/behave/behave_runner.bat create mode 100755 test/test_samples/behave/behave_runner.sh create mode 100644 test/test_samples/behave/features/steps/tutorial.py create mode 100644 test/test_samples/behave/features/tutorial.feature diff --git a/README.md b/README.md index 99d7a3f..0801989 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Azure Pipelines CI](https://dev.azure.com/kondratyev-nv/Python%20Test%20Explorer%20for%20Visual%20Studio%20Code/_apis/build/status/Python%20Test%20Explorer%20for%20Visual%20Studio%20Code%20CI?branchName=master)](https://dev.azure.com/kondratyev-nv/Python%20Test%20Explorer%20for%20Visual%20Studio%20Code/_build/latest?definitionId=1&branchName=master) [![Dependencies Status](https://david-dm.org/kondratyev-nv/vscode-python-unittest-adapter/status.svg)](https://david-dm.org/kondratyev-nv/vscode-python-unittest-adapter) -This extension allows you to run your Python [Unittest](https://docs.python.org/3/library/unittest.html#module-unittest), [Pytest](https://docs.pytest.org/en/latest/) or [Testplan](https://testplan.readthedocs.io/) +This extension allows you to run your Python [Unittest](https://docs.python.org/3/library/unittest.html#module-unittest), [Pytest](https://docs.pytest.org/en/latest/), [Testplan](https://testplan.readthedocs.io/) or [Behave](https://behave.readthedocs.io/en/latest/) tests with the [Test Explorer UI](https://marketplace.visualstudio.com/items?itemName=hbenl.vscode-test-explorer). ![Screenshot](img/screenshot.png) @@ -17,6 +17,7 @@ tests with the [Test Explorer UI](https://marketplace.visualstudio.com/items?ite * [Unittest documentation](https://docs.python.org/3/library/unittest.html#module-unittest) * [Pytest documentation](https://docs.pytest.org/en/latest/getting-started.html) * [Testplan documentation](https://testplan.readthedocs.io/en/latest/getting_started.html) + * [Behave documentation](https://behave.readthedocs.io/en/latest/) * Open Test View sidebar * Run your tests using the ![Run](img/run-button.png) icon in the Test Explorer @@ -62,6 +63,9 @@ Property | Description `python.testing.pyTestEnabled` | Whether to enable or disable unit testing using pytest (enables or disables test discovery for Test Explorer). `python.testing.pytestPath` | Path to pytest executable or a pytest compatible module. `python.testing.pyTestArgs` | Arguments passed to the pytest. Each argument is a separate item in the array. +`python.testing.behaveEnabled` | Whether to enable or disable testing using behave (enables or disables test discovery for Test Explorer). +`python.testing.behavePath` | Path to behave executable. +`python.testing.behaveArgs` | Arguments passed to behave. Each argument is a separate item in the array. `python.testing.autoTestDiscoverOnSaveEnabled` | When `true` tests will be automatically rediscovered when saving a test file. `pythonTestExplorer.testFramework` | Test framework to use (overrides Python extension properties `python.testing.unittestEnabled` and `python.testing.pyTestEnabled`). `pythonTestExplorer.testplanPath` | Relative path to testplan main suite. diff --git a/package.json b/package.json index 366e5ee..7b46b06 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,8 @@ "test", "testing", "unittest", - "pytest" + "pytest", + "behave" ], "scripts": { "clean": "rimraf out *.vsix **/*.pyc **/__pycache__ **/.pytest_cache **/.some_venv **/.venv", @@ -119,6 +120,7 @@ "unittest", "pytest", "testplan", + "behave", null ], "default": null, diff --git a/src/behave/behaveTestJsonParser.ts b/src/behave/behaveTestJsonParser.ts new file mode 100644 index 0000000..e79f8aa --- /dev/null +++ b/src/behave/behaveTestJsonParser.ts @@ -0,0 +1,108 @@ +import * as path from 'path'; + +import { TestInfo, TestSuiteInfo } from 'vscode-test-adapter-api'; +import { TestEvent } from 'vscode-test-adapter-api'; + +// Typescript interfaces for behave json output +type IStatus = 'passed' | 'failed' | 'skipped'; + +interface IScenario { + type: string; + keyword: string; + name: string; + tags: any[]; + location: string; + steps: IStep[]; + status: IStatus; +} + +interface IFeature { + keyword: string; + name: string; + tags: any[]; + location: string; + status: IStatus; + elements: IScenario[]; +} +interface IStep { + keyword: string; + step_type: string; + name: string; + location: string; + match: any; + result: IResult; + text?: string[]; +} +interface IResult { + status: IStatus; + duration: number; + error_message?: string[]; +} + +export function parseTestSuites(content: string, cwd: string): (TestSuiteInfo | TestInfo)[] { + + const discoveryResult : IFeature[] = JSON.parse(content); + + let stepid = 0; + const suites = discoveryResult.map(feature => ({ + type: 'suite' as 'suite', + id: feature.location, + label: feature.name, + file: extractFile(feature.location, cwd), + line: extractLine(feature.location), + tooltip: feature.location, + children: feature.elements.map(scenario => ({ + type: 'suite' as 'suite', + id: scenario.location, + label: scenario.name, + file: extractFile(scenario.location, cwd), + line: extractLine(scenario.location), + tooltip: scenario.location, + children: scenario.steps.map(step => ({ + type: 'test' as 'test', + id: "step" + (stepid += 1), + label: step.name, + file: extractFile(step.location, cwd), + line: extractLine(step.location), + tooltip: step.location + })) + })), + })); + + return suites; +} + +function extractLine(text: string) : number { + const separatorIndex = text.indexOf(':'); + return Number(text.substring(separatorIndex + 1)); +} + +function extractFile(text: string, cwd : string) { + const separatorIndex = text.indexOf(':'); + return path.resolve(cwd, text.substring(0, separatorIndex)) +} + +export function parseTestStates(content: string): TestEvent[] { + const runtestResult : IFeature[] = JSON.parse(content); + + let states : TestEvent[] = []; + + let stepid = 0; + + runtestResult.forEach( (feature) => { + feature.elements.forEach( (scenario) => { + const steps = scenario.steps.map( (step) : TestEvent => ({ + type: 'test' as 'test', + state: step.result.status, + test: "step" + (stepid += 1), + message: (step.result.error_message ? step.result.error_message.join('\n') : ""), + decorations: [], + description: undefined, + })); + states = states.concat(steps); + }) + }); + + return states; +} + diff --git a/src/behave/behaveTestRunner.ts b/src/behave/behaveTestRunner.ts new file mode 100644 index 0000000..9fdc211 --- /dev/null +++ b/src/behave/behaveTestRunner.ts @@ -0,0 +1,184 @@ +import * as path from 'path'; + +import { + TestEvent, TestSuiteInfo +} from 'vscode-test-adapter-api'; + +import { ArgumentParser } from 'argparse'; +import { IWorkspaceConfiguration } from '../configuration/workspaceConfiguration'; +import { IEnvironmentVariables, EnvironmentVariablesLoader } from '../environmentVariablesLoader'; +import { ILogger } from '../logging/logger'; +import { IProcessExecution, runProcess } from '../processRunner'; +import { IDebugConfiguration, ITestRunner } from '../testRunner'; +import { empty } from '../utilities/collections'; +import { setDescriptionForEqualLabels } from '../utilities/tests'; +import { parseTestStates } from './behaveTestJsonParser'; +import { parseTestSuites } from './behaveTestJsonParser'; +import { runModule } from '../pythonRunner'; + +// --- Behave Exit Codes --- +// 0: All tests were collected and passed successfully +// 1: Some tests have failed +const BEHAVE_NON_ERROR_EXIT_CODES = [0, 1]; + +const DISCOVERY_OUTPUT_PLUGIN_INFO = { + PACKAGE_PATH: path.resolve(__dirname, '../../resources/python'), + MODULE_NAME: 'vscode_python_test_adapter.behave.discovery_output_plugin', +}; + + +export class BehaveTestRunner implements ITestRunner { + + private readonly testExecutions: Map = new Map(); + + constructor( + public readonly adapterId: string, + private readonly logger: ILogger + ) { } + + public cancel(): void { + this.testExecutions.forEach((execution, test) => { + this.logger.log('info', `Cancelling execution of ${test}`); + try { + execution.cancel(); + } catch (error) { + this.logger.log('crit', `Cancelling execution of ${test} failed: ${error}`); + } + }); + } + + public async debugConfiguration(config: IWorkspaceConfiguration, test: string): Promise { + const additionalEnvironment = await this.loadEnvironmentVariables(config); + const runArguments = this.getRunArguments(test, config.getBehaveConfiguration().behaveArguments); + return { + module: 'behave', + cwd: config.getCwd(), + args: runArguments, + env: additionalEnvironment, + }; + } + + public async load(config: IWorkspaceConfiguration): Promise { + if (!config.getBehaveConfiguration().isBehaveEnabled) { + this.logger.log('info', 'Behave test discovery is disabled'); + return undefined; + } + const additionalEnvironment = await this.loadEnvironmentVariables(config); + this.logger.log('info', `Discovering tests using python path '${config.pythonPath()}' in ${config.getCwd()}`); + + const discoveryArguments = this.getDiscoveryArguments(config.getBehaveConfiguration().behaveArguments); + this.logger.log('info', `Running behave with arguments: ${discoveryArguments.join(', ')}`); + + const result = await this.runBehave(config, additionalEnvironment, discoveryArguments).complete(); + const tests = parseTestSuites(result.output, config.getCwd()); + if (empty(tests)) { + this.logger.log('warn', 'No tests discovered'); + return undefined; + } + + setDescriptionForEqualLabels(tests, path.sep); + return { + type: 'suite', + id: this.adapterId, + label: 'Behave tests', + children: tests, + }; + } + + public async run(config: IWorkspaceConfiguration, test: string): Promise { + if (!config.getBehaveConfiguration().isBehaveEnabled) { + this.logger.log('info', 'Behave test execution is disabled'); + return []; + } + const additionalEnvironment = await this.loadEnvironmentVariables(config); + this.logger.log('info', `Running tests using python path '${config.pythonPath()}' in ${config.getCwd()}`); + + const testRunArguments = this.getRunArguments(test, config.getBehaveConfiguration().behaveArguments); + this.logger.log('info', `Running behave with arguments: ${testRunArguments.join(', ')}`); + + const result = await this.runBehave(config, additionalEnvironment, testRunArguments).complete(); + const states = parseTestStates(result.output); + if (empty(states)) { + this.logger.log('warn', 'No tests run'); + return []; + } + + return states; + } + + private runBehave(config: IWorkspaceConfiguration, env: IEnvironmentVariables, args: string[]): IProcessExecution { + const behavePath = config.getBehaveConfiguration().behavePath(); + if (behavePath === path.basename(behavePath)) { + this.logger.log('info', `Running ${behavePath} as a Python module`); + return runModule({ + pythonPath: config.pythonPath(), + module: config.getBehaveConfiguration().behavePath(), + environment: env, + args, + cwd: config.getCwd(), + acceptedExitCodes: BEHAVE_NON_ERROR_EXIT_CODES, + }); + } + + this.logger.log('info', `Running ${behavePath} as an executable`); + return runProcess( + behavePath, + args, + { + cwd: config.getCwd(), + environment: env, + acceptedExitCodes: BEHAVE_NON_ERROR_EXIT_CODES, + }); + } + + private async loadEnvironmentVariables(config: IWorkspaceConfiguration): Promise { + const envFileEnvironment = await EnvironmentVariablesLoader.load(config.envFile(), process.env, this.logger); + + const updatedPythonPath = [ + config.getCwd(), + envFileEnvironment.PYTHONPATH, + process.env.PYTHONPATH, + DISCOVERY_OUTPUT_PLUGIN_INFO.PACKAGE_PATH + ].filter(item => item).join(path.delimiter); + + const updatedBehavePlugins = [ + envFileEnvironment.BEHAVE_PLUGINS, + DISCOVERY_OUTPUT_PLUGIN_INFO.MODULE_NAME + ].filter(item => item).join(','); + + return { + ...envFileEnvironment, + PYTHONPATH: updatedPythonPath, + BEHAVE_PLUGINS: updatedBehavePlugins, + }; + } + + private getDiscoveryArguments(rawBehaveArguments: string[]): string[] { + const argumentParser = this.configureCommonArgumentParser(); + const [, argumentsToPass] = argumentParser.parse_known_args(rawBehaveArguments); + return ['-d', '-f', 'json', '--no-summary', '--no-snippets'].concat(argumentsToPass); + } + + // @ts-expect-error + private getRunArguments(test: string, rawBehaveArguments: string[]): string[] { + const argumentParser = this.configureCommonArgumentParser(); + const [, argumentsToPass] = argumentParser.parse_known_args(rawBehaveArguments); + return ['-f', 'json', '--no-summary', '--no-snippets'].concat(argumentsToPass); + } + + private configureCommonArgumentParser() { + const argumentParser = new ArgumentParser({ + exit_on_error: false, + }); + argumentParser.add_argument( + '-D', '--define', + { action: 'store', dest: 'define' }); + argumentParser.add_argument( + '-e', '--exclude', + { action: 'store', dest: 'exclude' }); + argumentParser.add_argument( + '-i', '--include', + { action: 'store', dest: 'include' }); + return argumentParser; + } +} diff --git a/src/configuration/placeholderAwareWorkspaceConfiguration.ts b/src/configuration/placeholderAwareWorkspaceConfiguration.ts index 13196dc..b6d4732 100644 --- a/src/configuration/placeholderAwareWorkspaceConfiguration.ts +++ b/src/configuration/placeholderAwareWorkspaceConfiguration.ts @@ -4,6 +4,7 @@ import { WorkspaceFolder } from 'vscode'; import { ILogger } from '../logging/logger'; import { + IBehaveConfiguration, IPytestConfiguration, ITestplanConfiguration, IUnittestConfiguration, @@ -62,6 +63,19 @@ export class PlaceholderAwareWorkspaceConfiguration implements IWorkspaceConfigu }; } + public getBehaveConfiguration(): IBehaveConfiguration { + const original = this.configuration.getBehaveConfiguration(); + return { + behavePath: () => this.getBehavePath(), + isBehaveEnabled: original.isBehaveEnabled, + behaveArguments: original.behaveArguments.map(argument => this.resolvePlaceholders(argument)), + }; + } + + private getBehavePath(): string { + return this.resolveExecutablePath(this.configuration.getBehaveConfiguration().behavePath()); + } + private getPytestPath(): string { return this.resolveExecutablePath(this.configuration.getPytestConfiguration().pytestPath()); } diff --git a/src/configuration/pythonExtensionAwareWorkspaceConfiguration.ts b/src/configuration/pythonExtensionAwareWorkspaceConfiguration.ts index e86db37..fcdf2ba 100644 --- a/src/configuration/pythonExtensionAwareWorkspaceConfiguration.ts +++ b/src/configuration/pythonExtensionAwareWorkspaceConfiguration.ts @@ -3,6 +3,7 @@ import { EOL } from 'os'; import { ILogger } from '../logging/logger'; import { + IBehaveConfiguration, IPytestConfiguration, ITestplanConfiguration, IUnittestConfiguration, @@ -100,4 +101,8 @@ export class PythonExtensionAwareWorkspaceConfiguration implements IWorkspaceCon public getTestplanConfiguration(): ITestplanConfiguration { return this.configuration.getTestplanConfiguration(); } + + public getBehaveConfiguration(): IBehaveConfiguration { + return this.configuration.getBehaveConfiguration(); + } } diff --git a/src/configuration/vscodeWorkspaceConfiguration.ts b/src/configuration/vscodeWorkspaceConfiguration.ts index 26d2190..5d0ccd1 100644 --- a/src/configuration/vscodeWorkspaceConfiguration.ts +++ b/src/configuration/vscodeWorkspaceConfiguration.ts @@ -2,6 +2,7 @@ import { ArgumentParser } from 'argparse'; import { workspace, WorkspaceConfiguration, WorkspaceFolder } from 'vscode'; import { + IBehaveConfiguration, IPytestConfiguration, ITestplanConfiguration, IUnittestArguments, @@ -66,6 +67,14 @@ export class VscodeWorkspaceConfiguration implements IWorkspaceConfiguration { }; } + public getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => this.getBehavePath(), + isBehaveEnabled: this.isBehaveTestEnabled(), + behaveArguments: this.getBehaveArguments(), + }; + } + private getConfigurationValueOrDefault( configuration: WorkspaceConfiguration, keys: string[], defaultValue: T @@ -143,6 +152,22 @@ export class VscodeWorkspaceConfiguration implements IWorkspaceConfiguration { return this.testExplorerConfiguration.get('testplanArgs', []); } + private isBehaveTestEnabled(): boolean { + const overriddenTestFramework = this.testExplorerConfiguration.get('testFramework', null); + if (overriddenTestFramework) { + return 'behave' === overriddenTestFramework; + } + return this.testExplorerConfiguration.get('behaveEnabled', false); + } + + private getBehavePath(): string { + return this.testExplorerConfiguration.get('behavePath', 'behave'); + } + + private getBehaveArguments(): string[] { + return this.testExplorerConfiguration.get('behaveArgs', []); + } + private configureUnittestArgumentParser() { const argumentParser = new ArgumentParser({ exit_on_error: false, diff --git a/src/configuration/workspaceConfiguration.ts b/src/configuration/workspaceConfiguration.ts index 3e8ad4b..74c2d8f 100644 --- a/src/configuration/workspaceConfiguration.ts +++ b/src/configuration/workspaceConfiguration.ts @@ -20,6 +20,12 @@ export interface ITestplanConfiguration { isTestplanEnabled: boolean; } +export interface IBehaveConfiguration { + behavePath(): string; + behaveArguments: string[]; + isBehaveEnabled: boolean; +} + export interface IWorkspaceConfiguration { pythonPath(): string; @@ -34,4 +40,6 @@ export interface IWorkspaceConfiguration { getPytestConfiguration(): IPytestConfiguration; getTestplanConfiguration(): ITestplanConfiguration; + + getBehaveConfiguration(): IBehaveConfiguration; } diff --git a/src/main.ts b/src/main.ts index 79cbfaf..3e6726c 100644 --- a/src/main.ts +++ b/src/main.ts @@ -11,6 +11,7 @@ import { PytestTestRunner } from './pytest/pytestTestRunner'; import { TestplanTestRunner } from './testplan/testplanTestRunner'; import { PythonTestAdapter } from './pythonTestAdapter'; import { UnittestTestRunner } from './unittest/unittestTestRunner'; +import { BehaveTestRunner } from './behave/behaveTestRunner'; type LoggerFactory = (framework: string, wf: vscode.WorkspaceFolder) => ILogger; @@ -28,14 +29,19 @@ function registerTestAdapters( const testplanLogger = loggerFactory('testplan', wf); const testplanRunner = new TestplanTestRunner(nextId(), pytestLogger); + const behaveLogger = loggerFactory('behave', wf); + const behaveRunner = new BehaveTestRunner(nextId(), behaveLogger); + const unittestConfigurationFactory = new DefaultConfigurationFactory(unittestLogger); const pytestConfigurationFactory = new DefaultConfigurationFactory(pytestLogger); const testplantConfigurationFactory = new DefaultConfigurationFactory(testplanLogger); + const behaveConfigurationFactory = new DefaultConfigurationFactory(behaveLogger); const adapters = [ new PythonTestAdapter(wf, unittestRunner, unittestConfigurationFactory, unittestLogger), new PythonTestAdapter(wf, pytestRunner, pytestConfigurationFactory, pytestLogger), - new PythonTestAdapter(wf, testplanRunner, testplantConfigurationFactory, testplanLogger) + new PythonTestAdapter(wf, testplanRunner, testplantConfigurationFactory, testplanLogger), + new PythonTestAdapter(wf, behaveRunner, behaveConfigurationFactory, behaveLogger) ]; adapters.forEach(adapter => extension.exports.registerTestAdapter(adapter)); return adapters; diff --git a/src/pythonTestAdapter.ts b/src/pythonTestAdapter.ts index a417685..85b1d66 100644 --- a/src/pythonTestAdapter.ts +++ b/src/pythonTestAdapter.ts @@ -83,6 +83,9 @@ export class PythonTestAdapter implements TestAdapter { 'python.testing.pytestEnabled', 'python.testing.pytestPath', 'python.testing.pytestArgs', + 'python.testing.behaveEnabled', + 'python.testing.behavePath', + 'python.testing.behaveArgs', 'pythonTestExplorer.testFramework' ]; diff --git a/test/test_samples/behave/behave_runner.bat b/test/test_samples/behave/behave_runner.bat new file mode 100755 index 0000000..ba8d298 --- /dev/null +++ b/test/test_samples/behave/behave_runner.bat @@ -0,0 +1,9 @@ +@echo off +echo "Hello from a script running behave" + +python -m venv .some_venv + +.some_venv\bin\activate +python -m pip install behave + +behave %* diff --git a/test/test_samples/behave/behave_runner.sh b/test/test_samples/behave/behave_runner.sh new file mode 100755 index 0000000..bf1e57e --- /dev/null +++ b/test/test_samples/behave/behave_runner.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +echo "Hello from a script running behave" + +python -m venv .some_venv + +. .some_venv/bin/activate +python -m pip install behave + +behave "$@" diff --git a/test/test_samples/behave/features/steps/tutorial.py b/test/test_samples/behave/features/steps/tutorial.py new file mode 100644 index 0000000..7722f7c --- /dev/null +++ b/test/test_samples/behave/features/steps/tutorial.py @@ -0,0 +1,18 @@ +from behave import * + +@given('we have behave installed') +def step_impl(context): + pass + +@when('we implement a test') +def step_impl(context): + assert True is not False + +@then('behave will test it for us!') +def step_impl(context): + assert context.failed is False + +@then(u'this step will fail') +def step_impl(context): + assert False + diff --git a/test/test_samples/behave/features/tutorial.feature b/test/test_samples/behave/features/tutorial.feature new file mode 100644 index 0000000..977f0ab --- /dev/null +++ b/test/test_samples/behave/features/tutorial.feature @@ -0,0 +1,7 @@ +Feature: showing off behave + + Scenario: run a simple test + Given we have behave installed + When we implement a test + Then behave will test it for us! + And this step will fail diff --git a/test/test_samples/samples-workspace.code-workspace b/test/test_samples/samples-workspace.code-workspace index 923ad5a..c87c39d 100644 --- a/test/test_samples/samples-workspace.code-workspace +++ b/test/test_samples/samples-workspace.code-workspace @@ -9,6 +9,9 @@ { "path": "testplan" }, + { + "path": "behave" + }, { "path": "workspaces/empty_configuration" }, diff --git a/test/tests/environmentParsing.test.ts b/test/tests/environmentParsing.test.ts index 14559f4..ada890c 100644 --- a/test/tests/environmentParsing.test.ts +++ b/test/tests/environmentParsing.test.ts @@ -2,7 +2,7 @@ import { expect } from 'chai'; import 'mocha'; import * as path from 'path'; -import { IPytestConfiguration, ITestplanConfiguration, IUnittestConfiguration } from '../../src/configuration/workspaceConfiguration'; +import { IBehaveConfiguration, IPytestConfiguration, ITestplanConfiguration, IUnittestConfiguration } from '../../src/configuration/workspaceConfiguration'; import { PytestTestRunner } from '../../src/pytest/pytestTestRunner'; import { TestplanTestRunner } from '../../src/testplan/testplanTestRunner'; import { UnittestTestRunner } from '../../src/unittest/unittestTestRunner'; @@ -62,6 +62,13 @@ import { getPythonExecutable } from '../utils/testConfiguration'; testplanArguments: [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: [], + }; + }, }; const suites = await runner.load(config); expect(suites).to.be.undefined; @@ -105,6 +112,13 @@ import { getPythonExecutable } from '../utils/testConfiguration'; testplanArguments: [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: [], + }; + }, }; const suites = await runner.load(config); expect(suites).to.be.undefined; diff --git a/test/tests/placeholderAwareWorkspaceConfiguration.test.ts b/test/tests/placeholderAwareWorkspaceConfiguration.test.ts index 1621126..3029736 100644 --- a/test/tests/placeholderAwareWorkspaceConfiguration.test.ts +++ b/test/tests/placeholderAwareWorkspaceConfiguration.test.ts @@ -5,6 +5,7 @@ import * as path from 'path'; import { PlaceholderAwareWorkspaceConfiguration } from '../../src/configuration/placeholderAwareWorkspaceConfiguration'; import { + IBehaveConfiguration, IPytestConfiguration, ITestplanConfiguration, IUnittestConfiguration, @@ -62,6 +63,13 @@ suite('Placeholder aware workspace configuration', () => { testplanArguments: [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: [], + }; + }, }); const wfPath = getWorkspaceFolder().uri.fsPath; @@ -115,6 +123,13 @@ suite('Placeholder aware workspace configuration', () => { testplanArguments: [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: [], + }; + }, }); const wfPath = getWorkspaceFolder().uri.fsPath; @@ -168,6 +183,13 @@ suite('Placeholder aware workspace configuration', () => { testplanArguments: [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: [], + }; + }, }); const wfPath = getWorkspaceFolder().uri.fsPath; @@ -212,6 +234,13 @@ suite('Placeholder aware workspace configuration', () => { testplanArguments: [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: [], + }; + }, }); const wfPath = getWorkspaceFolder().uri.fsPath; @@ -262,6 +291,13 @@ suite('Placeholder aware workspace configuration', () => { testplanArguments: [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: [], + }; + }, }); const homePath = os.homedir(); @@ -311,6 +347,13 @@ suite('Placeholder aware workspace configuration', () => { testplanArguments: [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: [], + }; + }, }); expect(configuration.pythonPath()).to.be.eq(path.resolve(expectedPath, 'some', 'local', 'python')); @@ -358,6 +401,13 @@ suite('Placeholder aware workspace configuration', () => { testplanArguments: [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: [], + }; + }, }); expect(configuration.pythonPath()).to.be.eq( diff --git a/test/tests/pytestScript.test.ts b/test/tests/pytestScript.test.ts index bf9d39e..925ada8 100644 --- a/test/tests/pytestScript.test.ts +++ b/test/tests/pytestScript.test.ts @@ -4,6 +4,7 @@ import * as path from 'path'; import * as os from 'os'; import { + IBehaveConfiguration, IPytestConfiguration, ITestplanConfiguration, IUnittestConfiguration, @@ -52,6 +53,9 @@ function createPytestConfiguration(args?: string[]): IWorkspaceConfiguration { getTestplanConfiguration(): ITestplanConfiguration { throw new Error(); }, + getBehaveConfiguration(): IBehaveConfiguration { + throw new Error(); + }, }, wf, logger()); } diff --git a/test/tests/unittestGeneral.test.ts b/test/tests/unittestGeneral.test.ts index 6eb4bad..55f081b 100644 --- a/test/tests/unittestGeneral.test.ts +++ b/test/tests/unittestGeneral.test.ts @@ -211,6 +211,9 @@ suite('Unittest run and discovery with start folder in config', () => { getTestplanConfiguration() { throw new Error('Testplan is not available'); }, + getBehaveConfiguration() { + throw new Error('Testplan is not available'); + }, }; const runner = new UnittestTestRunner('some-id', logger()); diff --git a/test/utils/helpers.ts b/test/utils/helpers.ts index 172c603..a33d368 100644 --- a/test/utils/helpers.ts +++ b/test/utils/helpers.ts @@ -5,6 +5,7 @@ import * as vscode from 'vscode'; import { TestInfo, TestSuiteInfo } from 'vscode-test-adapter-api'; import { PlaceholderAwareWorkspaceConfiguration } from '../../src/configuration/placeholderAwareWorkspaceConfiguration'; import { + IBehaveConfiguration, IPytestConfiguration, ITestplanConfiguration, IUnittestConfiguration, @@ -115,6 +116,13 @@ export function createPytestConfiguration(folder: string, args?: string[], cwd?: getTestplanConfiguration(): ITestplanConfiguration { throw new Error(); }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: args || [], + }; + }, }, wf, logger()); } @@ -149,6 +157,9 @@ export function createUnittestConfiguration(folder: string): IWorkspaceConfigura getTestplanConfiguration(): ITestplanConfiguration { throw new Error(); }, + getBehaveConfiguration(): IBehaveConfiguration { + throw new Error(); + }, }, wf, logger()); } @@ -181,6 +192,13 @@ export function createTestplanConfiguration(folder: string, args?: string[], cwd testplanArguments: args || [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: args || [], + }; + }, }, wf, logger()); } From a4061d4586b04f3ff653994a4cf9b99535587c6f Mon Sep 17 00:00:00 2001 From: Klaus Oberhofer Date: Mon, 21 Feb 2022 19:44:31 +0100 Subject: [PATCH 02/18] fix lint errors --- src/behave/behaveTestJsonParser.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/behave/behaveTestJsonParser.ts b/src/behave/behaveTestJsonParser.ts index e79f8aa..5f36a32 100644 --- a/src/behave/behaveTestJsonParser.ts +++ b/src/behave/behaveTestJsonParser.ts @@ -60,7 +60,7 @@ export function parseTestSuites(content: string, cwd: string): (TestSuiteInfo | tooltip: scenario.location, children: scenario.steps.map(step => ({ type: 'test' as 'test', - id: "step" + (stepid += 1), + id: 'step' + (stepid += 1), label: step.name, file: extractFile(step.location, cwd), line: extractLine(step.location), @@ -94,7 +94,7 @@ export function parseTestStates(content: string): TestEvent[] { const steps = scenario.steps.map( (step) : TestEvent => ({ type: 'test' as 'test', state: step.result.status, - test: "step" + (stepid += 1), + test: 'step' + (stepid += 1), message: (step.result.error_message ? step.result.error_message.join('\n') : ""), decorations: [], description: undefined, From 5bab31632a12ac6a4c6ff2f451a5927c3bd703d1 Mon Sep 17 00:00:00 2001 From: Klaus Oberhofer Date: Tue, 22 Feb 2022 21:34:09 +0100 Subject: [PATCH 03/18] - fix more lint errors - wrap json parsing to catch failures --- src/behave/behaveTestJsonParser.ts | 28 ++++++++++++++++++---------- src/behave/behaveTestRunner.ts | 3 ++- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/behave/behaveTestJsonParser.ts b/src/behave/behaveTestJsonParser.ts index 5f36a32..8435283 100644 --- a/src/behave/behaveTestJsonParser.ts +++ b/src/behave/behaveTestJsonParser.ts @@ -39,9 +39,17 @@ interface IResult { error_message?: string[]; } -export function parseTestSuites(content: string, cwd: string): (TestSuiteInfo | TestInfo)[] { +function safeJsonParse(text: string) : IFeature[] { + try { + return JSON.parse(text); + } catch (err) { + // this.logger.log('warn', 'parse json failed: ${text}'); + return []; + } +} - const discoveryResult : IFeature[] = JSON.parse(content); +export function parseTestSuites(content: string, cwd: string): (TestSuiteInfo | TestInfo)[] { + const discoveryResult = safeJsonParse(content); let stepid = 0; const suites = discoveryResult.map(feature => ({ @@ -64,8 +72,8 @@ export function parseTestSuites(content: string, cwd: string): (TestSuiteInfo | label: step.name, file: extractFile(step.location, cwd), line: extractLine(step.location), - tooltip: step.location - })) + tooltip: step.location, + })), })), })); @@ -79,28 +87,28 @@ function extractLine(text: string) : number { function extractFile(text: string, cwd : string) { const separatorIndex = text.indexOf(':'); - return path.resolve(cwd, text.substring(0, separatorIndex)) + return path.resolve(cwd, text.substring(0, separatorIndex)); } export function parseTestStates(content: string): TestEvent[] { - const runtestResult : IFeature[] = JSON.parse(content); + const runtestResult = safeJsonParse(content); let states : TestEvent[] = []; let stepid = 0; - runtestResult.forEach( (feature) => { - feature.elements.forEach( (scenario) => { + runtestResult.forEach( feature => { + feature.elements.forEach( scenario => { const steps = scenario.steps.map( (step) : TestEvent => ({ type: 'test' as 'test', state: step.result.status, test: 'step' + (stepid += 1), - message: (step.result.error_message ? step.result.error_message.join('\n') : ""), + message: (step.result.error_message ? step.result.error_message.join('\n') : ''), decorations: [], description: undefined, })); states = states.concat(steps); - }) + }); }); return states; diff --git a/src/behave/behaveTestRunner.ts b/src/behave/behaveTestRunner.ts index 9fdc211..7d2b1c9 100644 --- a/src/behave/behaveTestRunner.ts +++ b/src/behave/behaveTestRunner.ts @@ -99,8 +99,9 @@ export class BehaveTestRunner implements ITestRunner { const result = await this.runBehave(config, additionalEnvironment, testRunArguments).complete(); const states = parseTestStates(result.output); if (empty(states)) { + // maybe an error occured this.logger.log('warn', 'No tests run'); - return []; + this.logger.log('warn', 'Output: ${result.output}'); } return states; From 5aa87a08da3898454fb38382a50b0f831eab62da Mon Sep 17 00:00:00 2001 From: Klaus Oberhofer Date: Sat, 19 Feb 2022 20:42:36 +0100 Subject: [PATCH 04/18] add support for behave BDD test framework --- README.md | 6 +- package.json | 4 +- src/behave/behaveTestJsonParser.ts | 108 ++++++++++ src/behave/behaveTestRunner.ts | 184 ++++++++++++++++++ .../placeholderAwareWorkspaceConfiguration.ts | 14 ++ ...honExtensionAwareWorkspaceConfiguration.ts | 5 + .../vscodeWorkspaceConfiguration.ts | 25 +++ src/configuration/workspaceConfiguration.ts | 8 + src/main.ts | 8 +- src/pythonTestAdapter.ts | 3 + test/test_samples/behave/behave_runner.bat | 9 + test/test_samples/behave/behave_runner.sh | 10 + .../behave/features/steps/tutorial.py | 18 ++ .../behave/features/tutorial.feature | 7 + .../samples-workspace.code-workspace | 3 + test/tests/environmentParsing.test.ts | 16 +- ...eholderAwareWorkspaceConfiguration.test.ts | 50 +++++ test/tests/pytestScript.test.ts | 4 + test/tests/unittestGeneral.test.ts | 3 + test/utils/helpers.ts | 18 ++ 20 files changed, 499 insertions(+), 4 deletions(-) create mode 100644 src/behave/behaveTestJsonParser.ts create mode 100644 src/behave/behaveTestRunner.ts create mode 100755 test/test_samples/behave/behave_runner.bat create mode 100755 test/test_samples/behave/behave_runner.sh create mode 100644 test/test_samples/behave/features/steps/tutorial.py create mode 100644 test/test_samples/behave/features/tutorial.feature diff --git a/README.md b/README.md index 99d7a3f..0801989 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ [![Azure Pipelines CI](https://dev.azure.com/kondratyev-nv/Python%20Test%20Explorer%20for%20Visual%20Studio%20Code/_apis/build/status/Python%20Test%20Explorer%20for%20Visual%20Studio%20Code%20CI?branchName=master)](https://dev.azure.com/kondratyev-nv/Python%20Test%20Explorer%20for%20Visual%20Studio%20Code/_build/latest?definitionId=1&branchName=master) [![Dependencies Status](https://david-dm.org/kondratyev-nv/vscode-python-unittest-adapter/status.svg)](https://david-dm.org/kondratyev-nv/vscode-python-unittest-adapter) -This extension allows you to run your Python [Unittest](https://docs.python.org/3/library/unittest.html#module-unittest), [Pytest](https://docs.pytest.org/en/latest/) or [Testplan](https://testplan.readthedocs.io/) +This extension allows you to run your Python [Unittest](https://docs.python.org/3/library/unittest.html#module-unittest), [Pytest](https://docs.pytest.org/en/latest/), [Testplan](https://testplan.readthedocs.io/) or [Behave](https://behave.readthedocs.io/en/latest/) tests with the [Test Explorer UI](https://marketplace.visualstudio.com/items?itemName=hbenl.vscode-test-explorer). ![Screenshot](img/screenshot.png) @@ -17,6 +17,7 @@ tests with the [Test Explorer UI](https://marketplace.visualstudio.com/items?ite * [Unittest documentation](https://docs.python.org/3/library/unittest.html#module-unittest) * [Pytest documentation](https://docs.pytest.org/en/latest/getting-started.html) * [Testplan documentation](https://testplan.readthedocs.io/en/latest/getting_started.html) + * [Behave documentation](https://behave.readthedocs.io/en/latest/) * Open Test View sidebar * Run your tests using the ![Run](img/run-button.png) icon in the Test Explorer @@ -62,6 +63,9 @@ Property | Description `python.testing.pyTestEnabled` | Whether to enable or disable unit testing using pytest (enables or disables test discovery for Test Explorer). `python.testing.pytestPath` | Path to pytest executable or a pytest compatible module. `python.testing.pyTestArgs` | Arguments passed to the pytest. Each argument is a separate item in the array. +`python.testing.behaveEnabled` | Whether to enable or disable testing using behave (enables or disables test discovery for Test Explorer). +`python.testing.behavePath` | Path to behave executable. +`python.testing.behaveArgs` | Arguments passed to behave. Each argument is a separate item in the array. `python.testing.autoTestDiscoverOnSaveEnabled` | When `true` tests will be automatically rediscovered when saving a test file. `pythonTestExplorer.testFramework` | Test framework to use (overrides Python extension properties `python.testing.unittestEnabled` and `python.testing.pyTestEnabled`). `pythonTestExplorer.testplanPath` | Relative path to testplan main suite. diff --git a/package.json b/package.json index 7b516cf..f2207b5 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,8 @@ "test", "testing", "unittest", - "pytest" + "pytest", + "behave" ], "scripts": { "clean": "rimraf out *.vsix **/*.pyc **/__pycache__ **/.pytest_cache **/.some_venv **/.venv", @@ -119,6 +120,7 @@ "unittest", "pytest", "testplan", + "behave", null ], "default": null, diff --git a/src/behave/behaveTestJsonParser.ts b/src/behave/behaveTestJsonParser.ts new file mode 100644 index 0000000..e79f8aa --- /dev/null +++ b/src/behave/behaveTestJsonParser.ts @@ -0,0 +1,108 @@ +import * as path from 'path'; + +import { TestInfo, TestSuiteInfo } from 'vscode-test-adapter-api'; +import { TestEvent } from 'vscode-test-adapter-api'; + +// Typescript interfaces for behave json output +type IStatus = 'passed' | 'failed' | 'skipped'; + +interface IScenario { + type: string; + keyword: string; + name: string; + tags: any[]; + location: string; + steps: IStep[]; + status: IStatus; +} + +interface IFeature { + keyword: string; + name: string; + tags: any[]; + location: string; + status: IStatus; + elements: IScenario[]; +} +interface IStep { + keyword: string; + step_type: string; + name: string; + location: string; + match: any; + result: IResult; + text?: string[]; +} +interface IResult { + status: IStatus; + duration: number; + error_message?: string[]; +} + +export function parseTestSuites(content: string, cwd: string): (TestSuiteInfo | TestInfo)[] { + + const discoveryResult : IFeature[] = JSON.parse(content); + + let stepid = 0; + const suites = discoveryResult.map(feature => ({ + type: 'suite' as 'suite', + id: feature.location, + label: feature.name, + file: extractFile(feature.location, cwd), + line: extractLine(feature.location), + tooltip: feature.location, + children: feature.elements.map(scenario => ({ + type: 'suite' as 'suite', + id: scenario.location, + label: scenario.name, + file: extractFile(scenario.location, cwd), + line: extractLine(scenario.location), + tooltip: scenario.location, + children: scenario.steps.map(step => ({ + type: 'test' as 'test', + id: "step" + (stepid += 1), + label: step.name, + file: extractFile(step.location, cwd), + line: extractLine(step.location), + tooltip: step.location + })) + })), + })); + + return suites; +} + +function extractLine(text: string) : number { + const separatorIndex = text.indexOf(':'); + return Number(text.substring(separatorIndex + 1)); +} + +function extractFile(text: string, cwd : string) { + const separatorIndex = text.indexOf(':'); + return path.resolve(cwd, text.substring(0, separatorIndex)) +} + +export function parseTestStates(content: string): TestEvent[] { + const runtestResult : IFeature[] = JSON.parse(content); + + let states : TestEvent[] = []; + + let stepid = 0; + + runtestResult.forEach( (feature) => { + feature.elements.forEach( (scenario) => { + const steps = scenario.steps.map( (step) : TestEvent => ({ + type: 'test' as 'test', + state: step.result.status, + test: "step" + (stepid += 1), + message: (step.result.error_message ? step.result.error_message.join('\n') : ""), + decorations: [], + description: undefined, + })); + states = states.concat(steps); + }) + }); + + return states; +} + diff --git a/src/behave/behaveTestRunner.ts b/src/behave/behaveTestRunner.ts new file mode 100644 index 0000000..9fdc211 --- /dev/null +++ b/src/behave/behaveTestRunner.ts @@ -0,0 +1,184 @@ +import * as path from 'path'; + +import { + TestEvent, TestSuiteInfo +} from 'vscode-test-adapter-api'; + +import { ArgumentParser } from 'argparse'; +import { IWorkspaceConfiguration } from '../configuration/workspaceConfiguration'; +import { IEnvironmentVariables, EnvironmentVariablesLoader } from '../environmentVariablesLoader'; +import { ILogger } from '../logging/logger'; +import { IProcessExecution, runProcess } from '../processRunner'; +import { IDebugConfiguration, ITestRunner } from '../testRunner'; +import { empty } from '../utilities/collections'; +import { setDescriptionForEqualLabels } from '../utilities/tests'; +import { parseTestStates } from './behaveTestJsonParser'; +import { parseTestSuites } from './behaveTestJsonParser'; +import { runModule } from '../pythonRunner'; + +// --- Behave Exit Codes --- +// 0: All tests were collected and passed successfully +// 1: Some tests have failed +const BEHAVE_NON_ERROR_EXIT_CODES = [0, 1]; + +const DISCOVERY_OUTPUT_PLUGIN_INFO = { + PACKAGE_PATH: path.resolve(__dirname, '../../resources/python'), + MODULE_NAME: 'vscode_python_test_adapter.behave.discovery_output_plugin', +}; + + +export class BehaveTestRunner implements ITestRunner { + + private readonly testExecutions: Map = new Map(); + + constructor( + public readonly adapterId: string, + private readonly logger: ILogger + ) { } + + public cancel(): void { + this.testExecutions.forEach((execution, test) => { + this.logger.log('info', `Cancelling execution of ${test}`); + try { + execution.cancel(); + } catch (error) { + this.logger.log('crit', `Cancelling execution of ${test} failed: ${error}`); + } + }); + } + + public async debugConfiguration(config: IWorkspaceConfiguration, test: string): Promise { + const additionalEnvironment = await this.loadEnvironmentVariables(config); + const runArguments = this.getRunArguments(test, config.getBehaveConfiguration().behaveArguments); + return { + module: 'behave', + cwd: config.getCwd(), + args: runArguments, + env: additionalEnvironment, + }; + } + + public async load(config: IWorkspaceConfiguration): Promise { + if (!config.getBehaveConfiguration().isBehaveEnabled) { + this.logger.log('info', 'Behave test discovery is disabled'); + return undefined; + } + const additionalEnvironment = await this.loadEnvironmentVariables(config); + this.logger.log('info', `Discovering tests using python path '${config.pythonPath()}' in ${config.getCwd()}`); + + const discoveryArguments = this.getDiscoveryArguments(config.getBehaveConfiguration().behaveArguments); + this.logger.log('info', `Running behave with arguments: ${discoveryArguments.join(', ')}`); + + const result = await this.runBehave(config, additionalEnvironment, discoveryArguments).complete(); + const tests = parseTestSuites(result.output, config.getCwd()); + if (empty(tests)) { + this.logger.log('warn', 'No tests discovered'); + return undefined; + } + + setDescriptionForEqualLabels(tests, path.sep); + return { + type: 'suite', + id: this.adapterId, + label: 'Behave tests', + children: tests, + }; + } + + public async run(config: IWorkspaceConfiguration, test: string): Promise { + if (!config.getBehaveConfiguration().isBehaveEnabled) { + this.logger.log('info', 'Behave test execution is disabled'); + return []; + } + const additionalEnvironment = await this.loadEnvironmentVariables(config); + this.logger.log('info', `Running tests using python path '${config.pythonPath()}' in ${config.getCwd()}`); + + const testRunArguments = this.getRunArguments(test, config.getBehaveConfiguration().behaveArguments); + this.logger.log('info', `Running behave with arguments: ${testRunArguments.join(', ')}`); + + const result = await this.runBehave(config, additionalEnvironment, testRunArguments).complete(); + const states = parseTestStates(result.output); + if (empty(states)) { + this.logger.log('warn', 'No tests run'); + return []; + } + + return states; + } + + private runBehave(config: IWorkspaceConfiguration, env: IEnvironmentVariables, args: string[]): IProcessExecution { + const behavePath = config.getBehaveConfiguration().behavePath(); + if (behavePath === path.basename(behavePath)) { + this.logger.log('info', `Running ${behavePath} as a Python module`); + return runModule({ + pythonPath: config.pythonPath(), + module: config.getBehaveConfiguration().behavePath(), + environment: env, + args, + cwd: config.getCwd(), + acceptedExitCodes: BEHAVE_NON_ERROR_EXIT_CODES, + }); + } + + this.logger.log('info', `Running ${behavePath} as an executable`); + return runProcess( + behavePath, + args, + { + cwd: config.getCwd(), + environment: env, + acceptedExitCodes: BEHAVE_NON_ERROR_EXIT_CODES, + }); + } + + private async loadEnvironmentVariables(config: IWorkspaceConfiguration): Promise { + const envFileEnvironment = await EnvironmentVariablesLoader.load(config.envFile(), process.env, this.logger); + + const updatedPythonPath = [ + config.getCwd(), + envFileEnvironment.PYTHONPATH, + process.env.PYTHONPATH, + DISCOVERY_OUTPUT_PLUGIN_INFO.PACKAGE_PATH + ].filter(item => item).join(path.delimiter); + + const updatedBehavePlugins = [ + envFileEnvironment.BEHAVE_PLUGINS, + DISCOVERY_OUTPUT_PLUGIN_INFO.MODULE_NAME + ].filter(item => item).join(','); + + return { + ...envFileEnvironment, + PYTHONPATH: updatedPythonPath, + BEHAVE_PLUGINS: updatedBehavePlugins, + }; + } + + private getDiscoveryArguments(rawBehaveArguments: string[]): string[] { + const argumentParser = this.configureCommonArgumentParser(); + const [, argumentsToPass] = argumentParser.parse_known_args(rawBehaveArguments); + return ['-d', '-f', 'json', '--no-summary', '--no-snippets'].concat(argumentsToPass); + } + + // @ts-expect-error + private getRunArguments(test: string, rawBehaveArguments: string[]): string[] { + const argumentParser = this.configureCommonArgumentParser(); + const [, argumentsToPass] = argumentParser.parse_known_args(rawBehaveArguments); + return ['-f', 'json', '--no-summary', '--no-snippets'].concat(argumentsToPass); + } + + private configureCommonArgumentParser() { + const argumentParser = new ArgumentParser({ + exit_on_error: false, + }); + argumentParser.add_argument( + '-D', '--define', + { action: 'store', dest: 'define' }); + argumentParser.add_argument( + '-e', '--exclude', + { action: 'store', dest: 'exclude' }); + argumentParser.add_argument( + '-i', '--include', + { action: 'store', dest: 'include' }); + return argumentParser; + } +} diff --git a/src/configuration/placeholderAwareWorkspaceConfiguration.ts b/src/configuration/placeholderAwareWorkspaceConfiguration.ts index 13196dc..b6d4732 100644 --- a/src/configuration/placeholderAwareWorkspaceConfiguration.ts +++ b/src/configuration/placeholderAwareWorkspaceConfiguration.ts @@ -4,6 +4,7 @@ import { WorkspaceFolder } from 'vscode'; import { ILogger } from '../logging/logger'; import { + IBehaveConfiguration, IPytestConfiguration, ITestplanConfiguration, IUnittestConfiguration, @@ -62,6 +63,19 @@ export class PlaceholderAwareWorkspaceConfiguration implements IWorkspaceConfigu }; } + public getBehaveConfiguration(): IBehaveConfiguration { + const original = this.configuration.getBehaveConfiguration(); + return { + behavePath: () => this.getBehavePath(), + isBehaveEnabled: original.isBehaveEnabled, + behaveArguments: original.behaveArguments.map(argument => this.resolvePlaceholders(argument)), + }; + } + + private getBehavePath(): string { + return this.resolveExecutablePath(this.configuration.getBehaveConfiguration().behavePath()); + } + private getPytestPath(): string { return this.resolveExecutablePath(this.configuration.getPytestConfiguration().pytestPath()); } diff --git a/src/configuration/pythonExtensionAwareWorkspaceConfiguration.ts b/src/configuration/pythonExtensionAwareWorkspaceConfiguration.ts index e86db37..fcdf2ba 100644 --- a/src/configuration/pythonExtensionAwareWorkspaceConfiguration.ts +++ b/src/configuration/pythonExtensionAwareWorkspaceConfiguration.ts @@ -3,6 +3,7 @@ import { EOL } from 'os'; import { ILogger } from '../logging/logger'; import { + IBehaveConfiguration, IPytestConfiguration, ITestplanConfiguration, IUnittestConfiguration, @@ -100,4 +101,8 @@ export class PythonExtensionAwareWorkspaceConfiguration implements IWorkspaceCon public getTestplanConfiguration(): ITestplanConfiguration { return this.configuration.getTestplanConfiguration(); } + + public getBehaveConfiguration(): IBehaveConfiguration { + return this.configuration.getBehaveConfiguration(); + } } diff --git a/src/configuration/vscodeWorkspaceConfiguration.ts b/src/configuration/vscodeWorkspaceConfiguration.ts index 26d2190..5d0ccd1 100644 --- a/src/configuration/vscodeWorkspaceConfiguration.ts +++ b/src/configuration/vscodeWorkspaceConfiguration.ts @@ -2,6 +2,7 @@ import { ArgumentParser } from 'argparse'; import { workspace, WorkspaceConfiguration, WorkspaceFolder } from 'vscode'; import { + IBehaveConfiguration, IPytestConfiguration, ITestplanConfiguration, IUnittestArguments, @@ -66,6 +67,14 @@ export class VscodeWorkspaceConfiguration implements IWorkspaceConfiguration { }; } + public getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => this.getBehavePath(), + isBehaveEnabled: this.isBehaveTestEnabled(), + behaveArguments: this.getBehaveArguments(), + }; + } + private getConfigurationValueOrDefault( configuration: WorkspaceConfiguration, keys: string[], defaultValue: T @@ -143,6 +152,22 @@ export class VscodeWorkspaceConfiguration implements IWorkspaceConfiguration { return this.testExplorerConfiguration.get('testplanArgs', []); } + private isBehaveTestEnabled(): boolean { + const overriddenTestFramework = this.testExplorerConfiguration.get('testFramework', null); + if (overriddenTestFramework) { + return 'behave' === overriddenTestFramework; + } + return this.testExplorerConfiguration.get('behaveEnabled', false); + } + + private getBehavePath(): string { + return this.testExplorerConfiguration.get('behavePath', 'behave'); + } + + private getBehaveArguments(): string[] { + return this.testExplorerConfiguration.get('behaveArgs', []); + } + private configureUnittestArgumentParser() { const argumentParser = new ArgumentParser({ exit_on_error: false, diff --git a/src/configuration/workspaceConfiguration.ts b/src/configuration/workspaceConfiguration.ts index 3e8ad4b..74c2d8f 100644 --- a/src/configuration/workspaceConfiguration.ts +++ b/src/configuration/workspaceConfiguration.ts @@ -20,6 +20,12 @@ export interface ITestplanConfiguration { isTestplanEnabled: boolean; } +export interface IBehaveConfiguration { + behavePath(): string; + behaveArguments: string[]; + isBehaveEnabled: boolean; +} + export interface IWorkspaceConfiguration { pythonPath(): string; @@ -34,4 +40,6 @@ export interface IWorkspaceConfiguration { getPytestConfiguration(): IPytestConfiguration; getTestplanConfiguration(): ITestplanConfiguration; + + getBehaveConfiguration(): IBehaveConfiguration; } diff --git a/src/main.ts b/src/main.ts index 79cbfaf..3e6726c 100644 --- a/src/main.ts +++ b/src/main.ts @@ -11,6 +11,7 @@ import { PytestTestRunner } from './pytest/pytestTestRunner'; import { TestplanTestRunner } from './testplan/testplanTestRunner'; import { PythonTestAdapter } from './pythonTestAdapter'; import { UnittestTestRunner } from './unittest/unittestTestRunner'; +import { BehaveTestRunner } from './behave/behaveTestRunner'; type LoggerFactory = (framework: string, wf: vscode.WorkspaceFolder) => ILogger; @@ -28,14 +29,19 @@ function registerTestAdapters( const testplanLogger = loggerFactory('testplan', wf); const testplanRunner = new TestplanTestRunner(nextId(), pytestLogger); + const behaveLogger = loggerFactory('behave', wf); + const behaveRunner = new BehaveTestRunner(nextId(), behaveLogger); + const unittestConfigurationFactory = new DefaultConfigurationFactory(unittestLogger); const pytestConfigurationFactory = new DefaultConfigurationFactory(pytestLogger); const testplantConfigurationFactory = new DefaultConfigurationFactory(testplanLogger); + const behaveConfigurationFactory = new DefaultConfigurationFactory(behaveLogger); const adapters = [ new PythonTestAdapter(wf, unittestRunner, unittestConfigurationFactory, unittestLogger), new PythonTestAdapter(wf, pytestRunner, pytestConfigurationFactory, pytestLogger), - new PythonTestAdapter(wf, testplanRunner, testplantConfigurationFactory, testplanLogger) + new PythonTestAdapter(wf, testplanRunner, testplantConfigurationFactory, testplanLogger), + new PythonTestAdapter(wf, behaveRunner, behaveConfigurationFactory, behaveLogger) ]; adapters.forEach(adapter => extension.exports.registerTestAdapter(adapter)); return adapters; diff --git a/src/pythonTestAdapter.ts b/src/pythonTestAdapter.ts index a417685..85b1d66 100644 --- a/src/pythonTestAdapter.ts +++ b/src/pythonTestAdapter.ts @@ -83,6 +83,9 @@ export class PythonTestAdapter implements TestAdapter { 'python.testing.pytestEnabled', 'python.testing.pytestPath', 'python.testing.pytestArgs', + 'python.testing.behaveEnabled', + 'python.testing.behavePath', + 'python.testing.behaveArgs', 'pythonTestExplorer.testFramework' ]; diff --git a/test/test_samples/behave/behave_runner.bat b/test/test_samples/behave/behave_runner.bat new file mode 100755 index 0000000..ba8d298 --- /dev/null +++ b/test/test_samples/behave/behave_runner.bat @@ -0,0 +1,9 @@ +@echo off +echo "Hello from a script running behave" + +python -m venv .some_venv + +.some_venv\bin\activate +python -m pip install behave + +behave %* diff --git a/test/test_samples/behave/behave_runner.sh b/test/test_samples/behave/behave_runner.sh new file mode 100755 index 0000000..bf1e57e --- /dev/null +++ b/test/test_samples/behave/behave_runner.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +echo "Hello from a script running behave" + +python -m venv .some_venv + +. .some_venv/bin/activate +python -m pip install behave + +behave "$@" diff --git a/test/test_samples/behave/features/steps/tutorial.py b/test/test_samples/behave/features/steps/tutorial.py new file mode 100644 index 0000000..7722f7c --- /dev/null +++ b/test/test_samples/behave/features/steps/tutorial.py @@ -0,0 +1,18 @@ +from behave import * + +@given('we have behave installed') +def step_impl(context): + pass + +@when('we implement a test') +def step_impl(context): + assert True is not False + +@then('behave will test it for us!') +def step_impl(context): + assert context.failed is False + +@then(u'this step will fail') +def step_impl(context): + assert False + diff --git a/test/test_samples/behave/features/tutorial.feature b/test/test_samples/behave/features/tutorial.feature new file mode 100644 index 0000000..977f0ab --- /dev/null +++ b/test/test_samples/behave/features/tutorial.feature @@ -0,0 +1,7 @@ +Feature: showing off behave + + Scenario: run a simple test + Given we have behave installed + When we implement a test + Then behave will test it for us! + And this step will fail diff --git a/test/test_samples/samples-workspace.code-workspace b/test/test_samples/samples-workspace.code-workspace index 923ad5a..c87c39d 100644 --- a/test/test_samples/samples-workspace.code-workspace +++ b/test/test_samples/samples-workspace.code-workspace @@ -9,6 +9,9 @@ { "path": "testplan" }, + { + "path": "behave" + }, { "path": "workspaces/empty_configuration" }, diff --git a/test/tests/environmentParsing.test.ts b/test/tests/environmentParsing.test.ts index 14559f4..ada890c 100644 --- a/test/tests/environmentParsing.test.ts +++ b/test/tests/environmentParsing.test.ts @@ -2,7 +2,7 @@ import { expect } from 'chai'; import 'mocha'; import * as path from 'path'; -import { IPytestConfiguration, ITestplanConfiguration, IUnittestConfiguration } from '../../src/configuration/workspaceConfiguration'; +import { IBehaveConfiguration, IPytestConfiguration, ITestplanConfiguration, IUnittestConfiguration } from '../../src/configuration/workspaceConfiguration'; import { PytestTestRunner } from '../../src/pytest/pytestTestRunner'; import { TestplanTestRunner } from '../../src/testplan/testplanTestRunner'; import { UnittestTestRunner } from '../../src/unittest/unittestTestRunner'; @@ -62,6 +62,13 @@ import { getPythonExecutable } from '../utils/testConfiguration'; testplanArguments: [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: [], + }; + }, }; const suites = await runner.load(config); expect(suites).to.be.undefined; @@ -105,6 +112,13 @@ import { getPythonExecutable } from '../utils/testConfiguration'; testplanArguments: [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: [], + }; + }, }; const suites = await runner.load(config); expect(suites).to.be.undefined; diff --git a/test/tests/placeholderAwareWorkspaceConfiguration.test.ts b/test/tests/placeholderAwareWorkspaceConfiguration.test.ts index 1621126..3029736 100644 --- a/test/tests/placeholderAwareWorkspaceConfiguration.test.ts +++ b/test/tests/placeholderAwareWorkspaceConfiguration.test.ts @@ -5,6 +5,7 @@ import * as path from 'path'; import { PlaceholderAwareWorkspaceConfiguration } from '../../src/configuration/placeholderAwareWorkspaceConfiguration'; import { + IBehaveConfiguration, IPytestConfiguration, ITestplanConfiguration, IUnittestConfiguration, @@ -62,6 +63,13 @@ suite('Placeholder aware workspace configuration', () => { testplanArguments: [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: [], + }; + }, }); const wfPath = getWorkspaceFolder().uri.fsPath; @@ -115,6 +123,13 @@ suite('Placeholder aware workspace configuration', () => { testplanArguments: [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: [], + }; + }, }); const wfPath = getWorkspaceFolder().uri.fsPath; @@ -168,6 +183,13 @@ suite('Placeholder aware workspace configuration', () => { testplanArguments: [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: [], + }; + }, }); const wfPath = getWorkspaceFolder().uri.fsPath; @@ -212,6 +234,13 @@ suite('Placeholder aware workspace configuration', () => { testplanArguments: [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: [], + }; + }, }); const wfPath = getWorkspaceFolder().uri.fsPath; @@ -262,6 +291,13 @@ suite('Placeholder aware workspace configuration', () => { testplanArguments: [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: [], + }; + }, }); const homePath = os.homedir(); @@ -311,6 +347,13 @@ suite('Placeholder aware workspace configuration', () => { testplanArguments: [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: [], + }; + }, }); expect(configuration.pythonPath()).to.be.eq(path.resolve(expectedPath, 'some', 'local', 'python')); @@ -358,6 +401,13 @@ suite('Placeholder aware workspace configuration', () => { testplanArguments: [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: [], + }; + }, }); expect(configuration.pythonPath()).to.be.eq( diff --git a/test/tests/pytestScript.test.ts b/test/tests/pytestScript.test.ts index bf9d39e..925ada8 100644 --- a/test/tests/pytestScript.test.ts +++ b/test/tests/pytestScript.test.ts @@ -4,6 +4,7 @@ import * as path from 'path'; import * as os from 'os'; import { + IBehaveConfiguration, IPytestConfiguration, ITestplanConfiguration, IUnittestConfiguration, @@ -52,6 +53,9 @@ function createPytestConfiguration(args?: string[]): IWorkspaceConfiguration { getTestplanConfiguration(): ITestplanConfiguration { throw new Error(); }, + getBehaveConfiguration(): IBehaveConfiguration { + throw new Error(); + }, }, wf, logger()); } diff --git a/test/tests/unittestGeneral.test.ts b/test/tests/unittestGeneral.test.ts index 6eb4bad..55f081b 100644 --- a/test/tests/unittestGeneral.test.ts +++ b/test/tests/unittestGeneral.test.ts @@ -211,6 +211,9 @@ suite('Unittest run and discovery with start folder in config', () => { getTestplanConfiguration() { throw new Error('Testplan is not available'); }, + getBehaveConfiguration() { + throw new Error('Testplan is not available'); + }, }; const runner = new UnittestTestRunner('some-id', logger()); diff --git a/test/utils/helpers.ts b/test/utils/helpers.ts index 172c603..a33d368 100644 --- a/test/utils/helpers.ts +++ b/test/utils/helpers.ts @@ -5,6 +5,7 @@ import * as vscode from 'vscode'; import { TestInfo, TestSuiteInfo } from 'vscode-test-adapter-api'; import { PlaceholderAwareWorkspaceConfiguration } from '../../src/configuration/placeholderAwareWorkspaceConfiguration'; import { + IBehaveConfiguration, IPytestConfiguration, ITestplanConfiguration, IUnittestConfiguration, @@ -115,6 +116,13 @@ export function createPytestConfiguration(folder: string, args?: string[], cwd?: getTestplanConfiguration(): ITestplanConfiguration { throw new Error(); }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: args || [], + }; + }, }, wf, logger()); } @@ -149,6 +157,9 @@ export function createUnittestConfiguration(folder: string): IWorkspaceConfigura getTestplanConfiguration(): ITestplanConfiguration { throw new Error(); }, + getBehaveConfiguration(): IBehaveConfiguration { + throw new Error(); + }, }, wf, logger()); } @@ -181,6 +192,13 @@ export function createTestplanConfiguration(folder: string, args?: string[], cwd testplanArguments: args || [], }; }, + getBehaveConfiguration(): IBehaveConfiguration { + return { + behavePath: () => 'behave', + isBehaveEnabled: true, + behaveArguments: args || [], + }; + }, }, wf, logger()); } From 07c9a6a182f52892f624a8d5f96a0be6170dd978 Mon Sep 17 00:00:00 2001 From: Klaus Oberhofer Date: Mon, 21 Feb 2022 19:44:31 +0100 Subject: [PATCH 05/18] fix lint errors --- src/behave/behaveTestJsonParser.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/behave/behaveTestJsonParser.ts b/src/behave/behaveTestJsonParser.ts index e79f8aa..5f36a32 100644 --- a/src/behave/behaveTestJsonParser.ts +++ b/src/behave/behaveTestJsonParser.ts @@ -60,7 +60,7 @@ export function parseTestSuites(content: string, cwd: string): (TestSuiteInfo | tooltip: scenario.location, children: scenario.steps.map(step => ({ type: 'test' as 'test', - id: "step" + (stepid += 1), + id: 'step' + (stepid += 1), label: step.name, file: extractFile(step.location, cwd), line: extractLine(step.location), @@ -94,7 +94,7 @@ export function parseTestStates(content: string): TestEvent[] { const steps = scenario.steps.map( (step) : TestEvent => ({ type: 'test' as 'test', state: step.result.status, - test: "step" + (stepid += 1), + test: 'step' + (stepid += 1), message: (step.result.error_message ? step.result.error_message.join('\n') : ""), decorations: [], description: undefined, From 881d3be4612358ed3dca08fd3de5b01e4e3e70b6 Mon Sep 17 00:00:00 2001 From: Klaus Oberhofer Date: Tue, 22 Feb 2022 21:34:09 +0100 Subject: [PATCH 06/18] - fix more lint errors - wrap json parsing to catch failures --- src/behave/behaveTestJsonParser.ts | 28 ++++++++++++++++++---------- src/behave/behaveTestRunner.ts | 3 ++- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/behave/behaveTestJsonParser.ts b/src/behave/behaveTestJsonParser.ts index 5f36a32..8435283 100644 --- a/src/behave/behaveTestJsonParser.ts +++ b/src/behave/behaveTestJsonParser.ts @@ -39,9 +39,17 @@ interface IResult { error_message?: string[]; } -export function parseTestSuites(content: string, cwd: string): (TestSuiteInfo | TestInfo)[] { +function safeJsonParse(text: string) : IFeature[] { + try { + return JSON.parse(text); + } catch (err) { + // this.logger.log('warn', 'parse json failed: ${text}'); + return []; + } +} - const discoveryResult : IFeature[] = JSON.parse(content); +export function parseTestSuites(content: string, cwd: string): (TestSuiteInfo | TestInfo)[] { + const discoveryResult = safeJsonParse(content); let stepid = 0; const suites = discoveryResult.map(feature => ({ @@ -64,8 +72,8 @@ export function parseTestSuites(content: string, cwd: string): (TestSuiteInfo | label: step.name, file: extractFile(step.location, cwd), line: extractLine(step.location), - tooltip: step.location - })) + tooltip: step.location, + })), })), })); @@ -79,28 +87,28 @@ function extractLine(text: string) : number { function extractFile(text: string, cwd : string) { const separatorIndex = text.indexOf(':'); - return path.resolve(cwd, text.substring(0, separatorIndex)) + return path.resolve(cwd, text.substring(0, separatorIndex)); } export function parseTestStates(content: string): TestEvent[] { - const runtestResult : IFeature[] = JSON.parse(content); + const runtestResult = safeJsonParse(content); let states : TestEvent[] = []; let stepid = 0; - runtestResult.forEach( (feature) => { - feature.elements.forEach( (scenario) => { + runtestResult.forEach( feature => { + feature.elements.forEach( scenario => { const steps = scenario.steps.map( (step) : TestEvent => ({ type: 'test' as 'test', state: step.result.status, test: 'step' + (stepid += 1), - message: (step.result.error_message ? step.result.error_message.join('\n') : ""), + message: (step.result.error_message ? step.result.error_message.join('\n') : ''), decorations: [], description: undefined, })); states = states.concat(steps); - }) + }); }); return states; diff --git a/src/behave/behaveTestRunner.ts b/src/behave/behaveTestRunner.ts index 9fdc211..7d2b1c9 100644 --- a/src/behave/behaveTestRunner.ts +++ b/src/behave/behaveTestRunner.ts @@ -99,8 +99,9 @@ export class BehaveTestRunner implements ITestRunner { const result = await this.runBehave(config, additionalEnvironment, testRunArguments).complete(); const states = parseTestStates(result.output); if (empty(states)) { + // maybe an error occured this.logger.log('warn', 'No tests run'); - return []; + this.logger.log('warn', 'Output: ${result.output}'); } return states; From 0b4c4f9318209b3327c9d7d26ef56e499fa1e153 Mon Sep 17 00:00:00 2001 From: Klaus Oberhofer Date: Fri, 25 Feb 2022 20:21:58 +0100 Subject: [PATCH 07/18] fix line endings (LF) --- src/behave/behaveTestRunner.ts | 370 ++++++++++++++++----------------- 1 file changed, 185 insertions(+), 185 deletions(-) diff --git a/src/behave/behaveTestRunner.ts b/src/behave/behaveTestRunner.ts index 7d2b1c9..6511f6e 100644 --- a/src/behave/behaveTestRunner.ts +++ b/src/behave/behaveTestRunner.ts @@ -1,185 +1,185 @@ -import * as path from 'path'; - -import { - TestEvent, TestSuiteInfo -} from 'vscode-test-adapter-api'; - -import { ArgumentParser } from 'argparse'; -import { IWorkspaceConfiguration } from '../configuration/workspaceConfiguration'; -import { IEnvironmentVariables, EnvironmentVariablesLoader } from '../environmentVariablesLoader'; -import { ILogger } from '../logging/logger'; -import { IProcessExecution, runProcess } from '../processRunner'; -import { IDebugConfiguration, ITestRunner } from '../testRunner'; -import { empty } from '../utilities/collections'; -import { setDescriptionForEqualLabels } from '../utilities/tests'; -import { parseTestStates } from './behaveTestJsonParser'; -import { parseTestSuites } from './behaveTestJsonParser'; -import { runModule } from '../pythonRunner'; - -// --- Behave Exit Codes --- -// 0: All tests were collected and passed successfully -// 1: Some tests have failed -const BEHAVE_NON_ERROR_EXIT_CODES = [0, 1]; - -const DISCOVERY_OUTPUT_PLUGIN_INFO = { - PACKAGE_PATH: path.resolve(__dirname, '../../resources/python'), - MODULE_NAME: 'vscode_python_test_adapter.behave.discovery_output_plugin', -}; - - -export class BehaveTestRunner implements ITestRunner { - - private readonly testExecutions: Map = new Map(); - - constructor( - public readonly adapterId: string, - private readonly logger: ILogger - ) { } - - public cancel(): void { - this.testExecutions.forEach((execution, test) => { - this.logger.log('info', `Cancelling execution of ${test}`); - try { - execution.cancel(); - } catch (error) { - this.logger.log('crit', `Cancelling execution of ${test} failed: ${error}`); - } - }); - } - - public async debugConfiguration(config: IWorkspaceConfiguration, test: string): Promise { - const additionalEnvironment = await this.loadEnvironmentVariables(config); - const runArguments = this.getRunArguments(test, config.getBehaveConfiguration().behaveArguments); - return { - module: 'behave', - cwd: config.getCwd(), - args: runArguments, - env: additionalEnvironment, - }; - } - - public async load(config: IWorkspaceConfiguration): Promise { - if (!config.getBehaveConfiguration().isBehaveEnabled) { - this.logger.log('info', 'Behave test discovery is disabled'); - return undefined; - } - const additionalEnvironment = await this.loadEnvironmentVariables(config); - this.logger.log('info', `Discovering tests using python path '${config.pythonPath()}' in ${config.getCwd()}`); - - const discoveryArguments = this.getDiscoveryArguments(config.getBehaveConfiguration().behaveArguments); - this.logger.log('info', `Running behave with arguments: ${discoveryArguments.join(', ')}`); - - const result = await this.runBehave(config, additionalEnvironment, discoveryArguments).complete(); - const tests = parseTestSuites(result.output, config.getCwd()); - if (empty(tests)) { - this.logger.log('warn', 'No tests discovered'); - return undefined; - } - - setDescriptionForEqualLabels(tests, path.sep); - return { - type: 'suite', - id: this.adapterId, - label: 'Behave tests', - children: tests, - }; - } - - public async run(config: IWorkspaceConfiguration, test: string): Promise { - if (!config.getBehaveConfiguration().isBehaveEnabled) { - this.logger.log('info', 'Behave test execution is disabled'); - return []; - } - const additionalEnvironment = await this.loadEnvironmentVariables(config); - this.logger.log('info', `Running tests using python path '${config.pythonPath()}' in ${config.getCwd()}`); - - const testRunArguments = this.getRunArguments(test, config.getBehaveConfiguration().behaveArguments); - this.logger.log('info', `Running behave with arguments: ${testRunArguments.join(', ')}`); - - const result = await this.runBehave(config, additionalEnvironment, testRunArguments).complete(); - const states = parseTestStates(result.output); - if (empty(states)) { - // maybe an error occured - this.logger.log('warn', 'No tests run'); - this.logger.log('warn', 'Output: ${result.output}'); - } - - return states; - } - - private runBehave(config: IWorkspaceConfiguration, env: IEnvironmentVariables, args: string[]): IProcessExecution { - const behavePath = config.getBehaveConfiguration().behavePath(); - if (behavePath === path.basename(behavePath)) { - this.logger.log('info', `Running ${behavePath} as a Python module`); - return runModule({ - pythonPath: config.pythonPath(), - module: config.getBehaveConfiguration().behavePath(), - environment: env, - args, - cwd: config.getCwd(), - acceptedExitCodes: BEHAVE_NON_ERROR_EXIT_CODES, - }); - } - - this.logger.log('info', `Running ${behavePath} as an executable`); - return runProcess( - behavePath, - args, - { - cwd: config.getCwd(), - environment: env, - acceptedExitCodes: BEHAVE_NON_ERROR_EXIT_CODES, - }); - } - - private async loadEnvironmentVariables(config: IWorkspaceConfiguration): Promise { - const envFileEnvironment = await EnvironmentVariablesLoader.load(config.envFile(), process.env, this.logger); - - const updatedPythonPath = [ - config.getCwd(), - envFileEnvironment.PYTHONPATH, - process.env.PYTHONPATH, - DISCOVERY_OUTPUT_PLUGIN_INFO.PACKAGE_PATH - ].filter(item => item).join(path.delimiter); - - const updatedBehavePlugins = [ - envFileEnvironment.BEHAVE_PLUGINS, - DISCOVERY_OUTPUT_PLUGIN_INFO.MODULE_NAME - ].filter(item => item).join(','); - - return { - ...envFileEnvironment, - PYTHONPATH: updatedPythonPath, - BEHAVE_PLUGINS: updatedBehavePlugins, - }; - } - - private getDiscoveryArguments(rawBehaveArguments: string[]): string[] { - const argumentParser = this.configureCommonArgumentParser(); - const [, argumentsToPass] = argumentParser.parse_known_args(rawBehaveArguments); - return ['-d', '-f', 'json', '--no-summary', '--no-snippets'].concat(argumentsToPass); - } - - // @ts-expect-error - private getRunArguments(test: string, rawBehaveArguments: string[]): string[] { - const argumentParser = this.configureCommonArgumentParser(); - const [, argumentsToPass] = argumentParser.parse_known_args(rawBehaveArguments); - return ['-f', 'json', '--no-summary', '--no-snippets'].concat(argumentsToPass); - } - - private configureCommonArgumentParser() { - const argumentParser = new ArgumentParser({ - exit_on_error: false, - }); - argumentParser.add_argument( - '-D', '--define', - { action: 'store', dest: 'define' }); - argumentParser.add_argument( - '-e', '--exclude', - { action: 'store', dest: 'exclude' }); - argumentParser.add_argument( - '-i', '--include', - { action: 'store', dest: 'include' }); - return argumentParser; - } -} +import * as path from 'path'; + +import { + TestEvent, TestSuiteInfo +} from 'vscode-test-adapter-api'; + +import { ArgumentParser } from 'argparse'; +import { IWorkspaceConfiguration } from '../configuration/workspaceConfiguration'; +import { IEnvironmentVariables, EnvironmentVariablesLoader } from '../environmentVariablesLoader'; +import { ILogger } from '../logging/logger'; +import { IProcessExecution, runProcess } from '../processRunner'; +import { IDebugConfiguration, ITestRunner } from '../testRunner'; +import { empty } from '../utilities/collections'; +import { setDescriptionForEqualLabels } from '../utilities/tests'; +import { parseTestStates } from './behaveTestJsonParser'; +import { parseTestSuites } from './behaveTestJsonParser'; +import { runModule } from '../pythonRunner'; + +// --- Behave Exit Codes --- +// 0: All tests were collected and passed successfully +// 1: Some tests have failed +const BEHAVE_NON_ERROR_EXIT_CODES = [0, 1]; + +const DISCOVERY_OUTPUT_PLUGIN_INFO = { + PACKAGE_PATH: path.resolve(__dirname, '../../resources/python'), + MODULE_NAME: 'vscode_python_test_adapter.behave.discovery_output_plugin', +}; + + +export class BehaveTestRunner implements ITestRunner { + + private readonly testExecutions: Map = new Map(); + + constructor( + public readonly adapterId: string, + private readonly logger: ILogger + ) { } + + public cancel(): void { + this.testExecutions.forEach((execution, test) => { + this.logger.log('info', `Cancelling execution of ${test}`); + try { + execution.cancel(); + } catch (error) { + this.logger.log('crit', `Cancelling execution of ${test} failed: ${error}`); + } + }); + } + + public async debugConfiguration(config: IWorkspaceConfiguration, test: string): Promise { + const additionalEnvironment = await this.loadEnvironmentVariables(config); + const runArguments = this.getRunArguments(test, config.getBehaveConfiguration().behaveArguments); + return { + module: 'behave', + cwd: config.getCwd(), + args: runArguments, + env: additionalEnvironment, + }; + } + + public async load(config: IWorkspaceConfiguration): Promise { + if (!config.getBehaveConfiguration().isBehaveEnabled) { + this.logger.log('info', 'Behave test discovery is disabled'); + return undefined; + } + const additionalEnvironment = await this.loadEnvironmentVariables(config); + this.logger.log('info', `Discovering tests using python path '${config.pythonPath()}' in ${config.getCwd()}`); + + const discoveryArguments = this.getDiscoveryArguments(config.getBehaveConfiguration().behaveArguments); + this.logger.log('info', `Running behave with arguments: ${discoveryArguments.join(', ')}`); + + const result = await this.runBehave(config, additionalEnvironment, discoveryArguments).complete(); + const tests = parseTestSuites(result.output, config.getCwd()); + if (empty(tests)) { + this.logger.log('warn', 'No tests discovered'); + return undefined; + } + + setDescriptionForEqualLabels(tests, path.sep); + return { + type: 'suite', + id: this.adapterId, + label: 'Behave tests', + children: tests, + }; + } + + public async run(config: IWorkspaceConfiguration, test: string): Promise { + if (!config.getBehaveConfiguration().isBehaveEnabled) { + this.logger.log('info', 'Behave test execution is disabled'); + return []; + } + const additionalEnvironment = await this.loadEnvironmentVariables(config); + this.logger.log('info', `Running tests using python path '${config.pythonPath()}' in ${config.getCwd()}`); + + const testRunArguments = this.getRunArguments(test, config.getBehaveConfiguration().behaveArguments); + this.logger.log('info', `Running behave with arguments: ${testRunArguments.join(', ')}`); + + const result = await this.runBehave(config, additionalEnvironment, testRunArguments).complete(); + const states = parseTestStates(result.output); + if (empty(states)) { + // maybe an error occured + this.logger.log('warn', 'No tests run'); + this.logger.log('warn', 'Output: ${result.output}'); + } + + return states; + } + + private runBehave(config: IWorkspaceConfiguration, env: IEnvironmentVariables, args: string[]): IProcessExecution { + const behavePath = config.getBehaveConfiguration().behavePath(); + if (behavePath === path.basename(behavePath)) { + this.logger.log('info', `Running ${behavePath} as a Python module`); + return runModule({ + pythonPath: config.pythonPath(), + module: config.getBehaveConfiguration().behavePath(), + environment: env, + args, + cwd: config.getCwd(), + acceptedExitCodes: BEHAVE_NON_ERROR_EXIT_CODES, + }); + } + + this.logger.log('info', `Running ${behavePath} as an executable`); + return runProcess( + behavePath, + args, + { + cwd: config.getCwd(), + environment: env, + acceptedExitCodes: BEHAVE_NON_ERROR_EXIT_CODES, + }); + } + + private async loadEnvironmentVariables(config: IWorkspaceConfiguration): Promise { + const envFileEnvironment = await EnvironmentVariablesLoader.load(config.envFile(), process.env, this.logger); + + const updatedPythonPath = [ + config.getCwd(), + envFileEnvironment.PYTHONPATH, + process.env.PYTHONPATH, + DISCOVERY_OUTPUT_PLUGIN_INFO.PACKAGE_PATH + ].filter(item => item).join(path.delimiter); + + const updatedBehavePlugins = [ + envFileEnvironment.BEHAVE_PLUGINS, + DISCOVERY_OUTPUT_PLUGIN_INFO.MODULE_NAME + ].filter(item => item).join(','); + + return { + ...envFileEnvironment, + PYTHONPATH: updatedPythonPath, + BEHAVE_PLUGINS: updatedBehavePlugins, + }; + } + + private getDiscoveryArguments(rawBehaveArguments: string[]): string[] { + const argumentParser = this.configureCommonArgumentParser(); + const [, argumentsToPass] = argumentParser.parse_known_args(rawBehaveArguments); + return ['-d', '-f', 'json', '--no-summary', '--no-snippets'].concat(argumentsToPass); + } + + // @ts-expect-error + private getRunArguments(test: string, rawBehaveArguments: string[]): string[] { + const argumentParser = this.configureCommonArgumentParser(); + const [, argumentsToPass] = argumentParser.parse_known_args(rawBehaveArguments); + return ['-f', 'json', '--no-summary', '--no-snippets'].concat(argumentsToPass); + } + + private configureCommonArgumentParser() { + const argumentParser = new ArgumentParser({ + exit_on_error: false, + }); + argumentParser.add_argument( + '-D', '--define', + { action: 'store', dest: 'define' }); + argumentParser.add_argument( + '-e', '--exclude', + { action: 'store', dest: 'exclude' }); + argumentParser.add_argument( + '-i', '--include', + { action: 'store', dest: 'include' }); + return argumentParser; + } +} From 1a87a0ee0387c43941721d4b26d2f83c50a22ea0 Mon Sep 17 00:00:00 2001 From: Klaus Oberhofer Date: Fri, 25 Feb 2022 20:23:44 +0100 Subject: [PATCH 08/18] behaveTestRunner: fix handling of behave specific arguments (e.g. behaveArgs) --- src/behave/behaveTestRunner.ts | 39 ++++++++++++++----- .../vscodeWorkspaceConfiguration.ts | 18 +++++++-- test/tests/unittestGeneral.test.ts | 2 +- 3 files changed, 46 insertions(+), 13 deletions(-) diff --git a/src/behave/behaveTestRunner.ts b/src/behave/behaveTestRunner.ts index 6511f6e..85cbb0e 100644 --- a/src/behave/behaveTestRunner.ts +++ b/src/behave/behaveTestRunner.ts @@ -26,6 +26,11 @@ const DISCOVERY_OUTPUT_PLUGIN_INFO = { MODULE_NAME: 'vscode_python_test_adapter.behave.discovery_output_plugin', }; +interface IBehaveArguments { + argumentsToPass: string[]; + locations: string[]; +} + export class BehaveTestRunner implements ITestRunner { @@ -50,10 +55,11 @@ export class BehaveTestRunner implements ITestRunner { public async debugConfiguration(config: IWorkspaceConfiguration, test: string): Promise { const additionalEnvironment = await this.loadEnvironmentVariables(config); const runArguments = this.getRunArguments(test, config.getBehaveConfiguration().behaveArguments); + const params = [ ...runArguments.argumentsToPass, ...runArguments.locations]; return { module: 'behave', cwd: config.getCwd(), - args: runArguments, + args: params, env: additionalEnvironment, }; } @@ -67,9 +73,12 @@ export class BehaveTestRunner implements ITestRunner { this.logger.log('info', `Discovering tests using python path '${config.pythonPath()}' in ${config.getCwd()}`); const discoveryArguments = this.getDiscoveryArguments(config.getBehaveConfiguration().behaveArguments); - this.logger.log('info', `Running behave with arguments: ${discoveryArguments.join(', ')}`); + this.logger.log('info', `Running behave with arguments: ${discoveryArguments.argumentsToPass.join(', ')}`); + this.logger.log('info', `Running behave with locations: ${discoveryArguments.locations.join(', ')}`); + + const params = [ ...discoveryArguments.argumentsToPass, ...discoveryArguments.locations]; - const result = await this.runBehave(config, additionalEnvironment, discoveryArguments).complete(); + const result = await this.runBehave(config, additionalEnvironment, params).complete(); const tests = parseTestSuites(result.output, config.getCwd()); if (empty(tests)) { this.logger.log('warn', 'No tests discovered'); @@ -154,17 +163,23 @@ export class BehaveTestRunner implements ITestRunner { }; } - private getDiscoveryArguments(rawBehaveArguments: string[]): string[] { + private getDiscoveryArguments(rawBehaveArguments: string[]): IBehaveArguments { const argumentParser = this.configureCommonArgumentParser(); - const [, argumentsToPass] = argumentParser.parse_known_args(rawBehaveArguments); - return ['-d', '-f', 'json', '--no-summary', '--no-snippets'].concat(argumentsToPass); + const [knownArguments, argumentsToPass] = argumentParser.parse_known_args(rawBehaveArguments); + return { + locations: (knownArguments as { locations?: string[] }).locations || [], + argumentsToPass: ['-d', '-f', 'json', '--no-summary', '--no-snippets'].concat(argumentsToPass), + }; } // @ts-expect-error - private getRunArguments(test: string, rawBehaveArguments: string[]): string[] { + private getRunArguments(test: string, rawBehaveArguments: string[]): IBehaveArguments { const argumentParser = this.configureCommonArgumentParser(); - const [, argumentsToPass] = argumentParser.parse_known_args(rawBehaveArguments); - return ['-f', 'json', '--no-summary', '--no-snippets'].concat(argumentsToPass); + const [knownArguments, argumentsToPass] = argumentParser.parse_known_args(rawBehaveArguments); + return { + locations: (knownArguments as { locations?: string[] }).locations || [], + argumentsToPass: ['-f', 'json', '--no-summary', '--no-snippets'].concat(argumentsToPass), + }; } private configureCommonArgumentParser() { @@ -180,6 +195,12 @@ export class BehaveTestRunner implements ITestRunner { argumentParser.add_argument( '-i', '--include', { action: 'store', dest: 'include' }); + + // Handle positional arguments (list of testsuite directories to run behave in). + argumentParser.add_argument( + 'locations', + { nargs: '*' }); + return argumentParser; } } diff --git a/src/configuration/vscodeWorkspaceConfiguration.ts b/src/configuration/vscodeWorkspaceConfiguration.ts index 5d0ccd1..68d4dfb 100644 --- a/src/configuration/vscodeWorkspaceConfiguration.ts +++ b/src/configuration/vscodeWorkspaceConfiguration.ts @@ -157,15 +157,27 @@ export class VscodeWorkspaceConfiguration implements IWorkspaceConfiguration { if (overriddenTestFramework) { return 'behave' === overriddenTestFramework; } - return this.testExplorerConfiguration.get('behaveEnabled', false); + return this.getConfigurationValueOrDefault( + this.pythonConfiguration, + ['testing.behaveEnabled', 'testing.BehaveEnabled'], + false + ); } private getBehavePath(): string { - return this.testExplorerConfiguration.get('behavePath', 'behave'); + return this.getConfigurationValueOrDefault( + this.pythonConfiguration, + ['testing.behavePath', 'testing.BehavePath'], + 'behave' + ); } private getBehaveArguments(): string[] { - return this.testExplorerConfiguration.get('behaveArgs', []); + return this.getConfigurationValueOrDefault( + this.pythonConfiguration, + ['testing.behaveArgs', 'testing.BehaveArgs'], + [] + ); } private configureUnittestArgumentParser() { diff --git a/test/tests/unittestGeneral.test.ts b/test/tests/unittestGeneral.test.ts index 55f081b..f6c587c 100644 --- a/test/tests/unittestGeneral.test.ts +++ b/test/tests/unittestGeneral.test.ts @@ -212,7 +212,7 @@ suite('Unittest run and discovery with start folder in config', () => { throw new Error('Testplan is not available'); }, getBehaveConfiguration() { - throw new Error('Testplan is not available'); + throw new Error('Behave is not available'); }, }; const runner = new UnittestTestRunner('some-id', logger()); From 22344ae2c948c16ba353e3ea7e4dae6d052e194e Mon Sep 17 00:00:00 2001 From: Klaus Oberhofer Date: Fri, 25 Feb 2022 21:26:21 +0100 Subject: [PATCH 09/18] behaveTestRunner: fix handling of behave specific arguments in run() --- src/behave/behaveTestRunner.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/behave/behaveTestRunner.ts b/src/behave/behaveTestRunner.ts index 85cbb0e..080171f 100644 --- a/src/behave/behaveTestRunner.ts +++ b/src/behave/behaveTestRunner.ts @@ -103,9 +103,12 @@ export class BehaveTestRunner implements ITestRunner { this.logger.log('info', `Running tests using python path '${config.pythonPath()}' in ${config.getCwd()}`); const testRunArguments = this.getRunArguments(test, config.getBehaveConfiguration().behaveArguments); - this.logger.log('info', `Running behave with arguments: ${testRunArguments.join(', ')}`); + this.logger.log('info', `Running behave with arguments: ${testRunArguments.argumentsToPass.join(', ')}`); + this.logger.log('info', `Running behave with locations: ${testRunArguments.locations.join(', ')}`); - const result = await this.runBehave(config, additionalEnvironment, testRunArguments).complete(); + const params = [ ...testRunArguments.argumentsToPass, ...testRunArguments.locations]; + + const result = await this.runBehave(config, additionalEnvironment, params).complete(); const states = parseTestStates(result.output); if (empty(states)) { // maybe an error occured From c5cec0a2d804cdaf408e19a5040f173f41b840e4 Mon Sep 17 00:00:00 2001 From: Klaus Oberhofer Date: Tue, 1 Mar 2022 21:38:46 +0100 Subject: [PATCH 10/18] fix json parsing for feature files with no valid scenarios --- src/behave/behaveTestJsonParser.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/behave/behaveTestJsonParser.ts b/src/behave/behaveTestJsonParser.ts index 8435283..fd4d007 100644 --- a/src/behave/behaveTestJsonParser.ts +++ b/src/behave/behaveTestJsonParser.ts @@ -22,7 +22,7 @@ interface IFeature { tags: any[]; location: string; status: IStatus; - elements: IScenario[]; + elements?: IScenario[]; } interface IStep { keyword: string; @@ -59,7 +59,7 @@ export function parseTestSuites(content: string, cwd: string): (TestSuiteInfo | file: extractFile(feature.location, cwd), line: extractLine(feature.location), tooltip: feature.location, - children: feature.elements.map(scenario => ({ + children: (feature.elements || []).map(scenario => ({ type: 'suite' as 'suite', id: scenario.location, label: scenario.name, @@ -98,7 +98,7 @@ export function parseTestStates(content: string): TestEvent[] { let stepid = 0; runtestResult.forEach( feature => { - feature.elements.forEach( scenario => { + (feature.elements || []).forEach( scenario => { const steps = scenario.steps.map( (step) : TestEvent => ({ type: 'test' as 'test', state: step.result.status, From ca8c3ef23c1f960ce3897f6f8eafa5191552b1e0 Mon Sep 17 00:00:00 2001 From: Klaus Oberhofer Date: Tue, 1 Mar 2022 21:40:39 +0100 Subject: [PATCH 11/18] fix line endings (LF) --- src/behave/behaveTestJsonParser.ts | 232 ++++++++++++++--------------- 1 file changed, 116 insertions(+), 116 deletions(-) diff --git a/src/behave/behaveTestJsonParser.ts b/src/behave/behaveTestJsonParser.ts index fd4d007..814db98 100644 --- a/src/behave/behaveTestJsonParser.ts +++ b/src/behave/behaveTestJsonParser.ts @@ -1,116 +1,116 @@ -import * as path from 'path'; - -import { TestInfo, TestSuiteInfo } from 'vscode-test-adapter-api'; -import { TestEvent } from 'vscode-test-adapter-api'; - -// Typescript interfaces for behave json output -type IStatus = 'passed' | 'failed' | 'skipped'; - -interface IScenario { - type: string; - keyword: string; - name: string; - tags: any[]; - location: string; - steps: IStep[]; - status: IStatus; -} - -interface IFeature { - keyword: string; - name: string; - tags: any[]; - location: string; - status: IStatus; - elements?: IScenario[]; -} -interface IStep { - keyword: string; - step_type: string; - name: string; - location: string; - match: any; - result: IResult; - text?: string[]; -} -interface IResult { - status: IStatus; - duration: number; - error_message?: string[]; -} - -function safeJsonParse(text: string) : IFeature[] { - try { - return JSON.parse(text); - } catch (err) { - // this.logger.log('warn', 'parse json failed: ${text}'); - return []; - } -} - -export function parseTestSuites(content: string, cwd: string): (TestSuiteInfo | TestInfo)[] { - const discoveryResult = safeJsonParse(content); - - let stepid = 0; - const suites = discoveryResult.map(feature => ({ - type: 'suite' as 'suite', - id: feature.location, - label: feature.name, - file: extractFile(feature.location, cwd), - line: extractLine(feature.location), - tooltip: feature.location, - children: (feature.elements || []).map(scenario => ({ - type: 'suite' as 'suite', - id: scenario.location, - label: scenario.name, - file: extractFile(scenario.location, cwd), - line: extractLine(scenario.location), - tooltip: scenario.location, - children: scenario.steps.map(step => ({ - type: 'test' as 'test', - id: 'step' + (stepid += 1), - label: step.name, - file: extractFile(step.location, cwd), - line: extractLine(step.location), - tooltip: step.location, - })), - })), - })); - - return suites; -} - -function extractLine(text: string) : number { - const separatorIndex = text.indexOf(':'); - return Number(text.substring(separatorIndex + 1)); -} - -function extractFile(text: string, cwd : string) { - const separatorIndex = text.indexOf(':'); - return path.resolve(cwd, text.substring(0, separatorIndex)); -} - -export function parseTestStates(content: string): TestEvent[] { - const runtestResult = safeJsonParse(content); - - let states : TestEvent[] = []; - - let stepid = 0; - - runtestResult.forEach( feature => { - (feature.elements || []).forEach( scenario => { - const steps = scenario.steps.map( (step) : TestEvent => ({ - type: 'test' as 'test', - state: step.result.status, - test: 'step' + (stepid += 1), - message: (step.result.error_message ? step.result.error_message.join('\n') : ''), - decorations: [], - description: undefined, - })); - states = states.concat(steps); - }); - }); - - return states; -} - +import * as path from 'path'; + +import { TestInfo, TestSuiteInfo } from 'vscode-test-adapter-api'; +import { TestEvent } from 'vscode-test-adapter-api'; + +// Typescript interfaces for behave json output +type IStatus = 'passed' | 'failed' | 'skipped'; + +interface IScenario { + type: string; + keyword: string; + name: string; + tags: any[]; + location: string; + steps: IStep[]; + status: IStatus; +} + +interface IFeature { + keyword: string; + name: string; + tags: any[]; + location: string; + status: IStatus; + elements?: IScenario[]; +} +interface IStep { + keyword: string; + step_type: string; + name: string; + location: string; + match: any; + result: IResult; + text?: string[]; +} +interface IResult { + status: IStatus; + duration: number; + error_message?: string[]; +} + +function safeJsonParse(text: string) : IFeature[] { + try { + return JSON.parse(text); + } catch (err) { + // this.logger.log('warn', 'parse json failed: ${text}'); + return []; + } +} + +export function parseTestSuites(content: string, cwd: string): (TestSuiteInfo | TestInfo)[] { + const discoveryResult = safeJsonParse(content); + + let stepid = 0; + const suites = discoveryResult.map(feature => ({ + type: 'suite' as 'suite', + id: feature.location, + label: feature.name, + file: extractFile(feature.location, cwd), + line: extractLine(feature.location), + tooltip: feature.location, + children: (feature.elements || []).map(scenario => ({ + type: 'suite' as 'suite', + id: scenario.location, + label: scenario.name, + file: extractFile(scenario.location, cwd), + line: extractLine(scenario.location), + tooltip: scenario.location, + children: scenario.steps.map(step => ({ + type: 'test' as 'test', + id: 'step' + (stepid += 1), + label: step.name, + file: extractFile(step.location, cwd), + line: extractLine(step.location), + tooltip: step.location, + })), + })), + })); + + return suites; +} + +function extractLine(text: string) : number { + const separatorIndex = text.indexOf(':'); + return Number(text.substring(separatorIndex + 1)); +} + +function extractFile(text: string, cwd : string) { + const separatorIndex = text.indexOf(':'); + return path.resolve(cwd, text.substring(0, separatorIndex)); +} + +export function parseTestStates(content: string): TestEvent[] { + const runtestResult = safeJsonParse(content); + + let states : TestEvent[] = []; + + let stepid = 0; + + runtestResult.forEach( feature => { + (feature.elements || []).forEach( scenario => { + const steps = scenario.steps.map( (step) : TestEvent => ({ + type: 'test' as 'test', + state: step.result.status, + test: 'step' + (stepid += 1), + message: (step.result.error_message ? step.result.error_message.join('\n') : ''), + decorations: [], + description: undefined, + })); + states = states.concat(steps); + }); + }); + + return states; +} + From 987a087b60da02d7afe51263974ffa6901648ade Mon Sep 17 00:00:00 2001 From: Klaus Oberhofer Date: Sun, 20 Mar 2022 20:09:36 +0100 Subject: [PATCH 12/18] add behave to requirements.txt --- requirements.txt | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 7469de1..6ce9920 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,4 +5,7 @@ https://github.com/morganstanley/testplan/archive/main.zip; python_version > '3. plotly # temporary fix for testplan -markupsafe==2.0.1; python_version > '3.6' \ No newline at end of file +markupsafe==2.0.1; python_version > '3.6' + +# behave test framework +behave From b827ce0160a9d57b5bebfb8eec6a4e3309cd6c4d Mon Sep 17 00:00:00 2001 From: Klaus Oberhofer Date: Sun, 20 Mar 2022 20:20:26 +0100 Subject: [PATCH 13/18] fix comment --- src/behave/behaveTestJsonParser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/behave/behaveTestJsonParser.ts b/src/behave/behaveTestJsonParser.ts index 814db98..a19f954 100644 --- a/src/behave/behaveTestJsonParser.ts +++ b/src/behave/behaveTestJsonParser.ts @@ -43,7 +43,7 @@ function safeJsonParse(text: string) : IFeature[] { try { return JSON.parse(text); } catch (err) { - // this.logger.log('warn', 'parse json failed: ${text}'); + // parse json has failed, return empty array return []; } } From 1ead2a6fb85790f6cec07e267837cc662a94fb4a Mon Sep 17 00:00:00 2001 From: Klaus Oberhofer Date: Sun, 20 Mar 2022 20:24:47 +0100 Subject: [PATCH 14/18] use parseInt() instead of Number() to convert line number --- src/behave/behaveTestJsonParser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/behave/behaveTestJsonParser.ts b/src/behave/behaveTestJsonParser.ts index a19f954..1539311 100644 --- a/src/behave/behaveTestJsonParser.ts +++ b/src/behave/behaveTestJsonParser.ts @@ -82,7 +82,7 @@ export function parseTestSuites(content: string, cwd: string): (TestSuiteInfo | function extractLine(text: string) : number { const separatorIndex = text.indexOf(':'); - return Number(text.substring(separatorIndex + 1)); + return parseInt(text.substring(separatorIndex + 1), 10); } function extractFile(text: string, cwd : string) { From 974a3bc170f78d974f743c251ac9c38e44ebdf85 Mon Sep 17 00:00:00 2001 From: Klaus Oberhofer Date: Sun, 20 Mar 2022 20:37:15 +0100 Subject: [PATCH 15/18] fix identation --- src/behave/behaveTestJsonParser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/behave/behaveTestJsonParser.ts b/src/behave/behaveTestJsonParser.ts index 1539311..522c13b 100644 --- a/src/behave/behaveTestJsonParser.ts +++ b/src/behave/behaveTestJsonParser.ts @@ -59,7 +59,7 @@ export function parseTestSuites(content: string, cwd: string): (TestSuiteInfo | file: extractFile(feature.location, cwd), line: extractLine(feature.location), tooltip: feature.location, - children: (feature.elements || []).map(scenario => ({ + children: (feature.elements || []).map(scenario => ({ type: 'suite' as 'suite', id: scenario.location, label: scenario.name, From dc3c76e2918102e4be1bcb71f9120468249f3206 Mon Sep 17 00:00:00 2001 From: Klaus Oberhofer Date: Sun, 20 Mar 2022 20:48:00 +0100 Subject: [PATCH 16/18] remove unecessary attribute tests --- src/configuration/vscodeWorkspaceConfiguration.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/configuration/vscodeWorkspaceConfiguration.ts b/src/configuration/vscodeWorkspaceConfiguration.ts index 68d4dfb..12359f1 100644 --- a/src/configuration/vscodeWorkspaceConfiguration.ts +++ b/src/configuration/vscodeWorkspaceConfiguration.ts @@ -159,7 +159,7 @@ export class VscodeWorkspaceConfiguration implements IWorkspaceConfiguration { } return this.getConfigurationValueOrDefault( this.pythonConfiguration, - ['testing.behaveEnabled', 'testing.BehaveEnabled'], + ['testing.behaveEnabled'], false ); } @@ -167,7 +167,7 @@ export class VscodeWorkspaceConfiguration implements IWorkspaceConfiguration { private getBehavePath(): string { return this.getConfigurationValueOrDefault( this.pythonConfiguration, - ['testing.behavePath', 'testing.BehavePath'], + ['testing.behavePath'], 'behave' ); } @@ -175,7 +175,7 @@ export class VscodeWorkspaceConfiguration implements IWorkspaceConfiguration { private getBehaveArguments(): string[] { return this.getConfigurationValueOrDefault( this.pythonConfiguration, - ['testing.behaveArgs', 'testing.BehaveArgs'], + ['testing.behaveArgs'], [] ); } From 1519b1f092c7de514963c74c1b0786c5c1b853d0 Mon Sep 17 00:00:00 2001 From: Klaus Oberhofer Date: Sun, 20 Mar 2022 21:58:33 +0100 Subject: [PATCH 17/18] behave test sample: add .vscode settings --- test/test_samples/behave/.vscode/launch.json | 16 ++++++++++++++++ test/test_samples/behave/.vscode/settings.json | 5 +++++ 2 files changed, 21 insertions(+) create mode 100644 test/test_samples/behave/.vscode/launch.json create mode 100644 test/test_samples/behave/.vscode/settings.json diff --git a/test/test_samples/behave/.vscode/launch.json b/test/test_samples/behave/.vscode/launch.json new file mode 100644 index 0000000..10fddb6 --- /dev/null +++ b/test/test_samples/behave/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Debug configuration", + "type": "python", + "request": "test", + "console": "externalTerminal", + "justMyCode": false, + "stopOnEntry": true + } + ] +} \ No newline at end of file diff --git a/test/test_samples/behave/.vscode/settings.json b/test/test_samples/behave/.vscode/settings.json new file mode 100644 index 0000000..cc3441c --- /dev/null +++ b/test/test_samples/behave/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "python.envFile": "../.env", + "python.testing.behaveEnabled": true, + "python.testing.behaveArgs": [] +} From 1b12d4d27ad11db9389bb0fea79222cb42c2b427 Mon Sep 17 00:00:00 2001 From: Klaus Oberhofer Date: Sun, 20 Mar 2022 23:57:47 +0100 Subject: [PATCH 18/18] behave runner: fix use of test parameter in getRunArguments() --- src/behave/behaveTestRunner.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/behave/behaveTestRunner.ts b/src/behave/behaveTestRunner.ts index 080171f..8455ff5 100644 --- a/src/behave/behaveTestRunner.ts +++ b/src/behave/behaveTestRunner.ts @@ -175,13 +175,14 @@ export class BehaveTestRunner implements ITestRunner { }; } - // @ts-expect-error private getRunArguments(test: string, rawBehaveArguments: string[]): IBehaveArguments { const argumentParser = this.configureCommonArgumentParser(); const [knownArguments, argumentsToPass] = argumentParser.parse_known_args(rawBehaveArguments); return { - locations: (knownArguments as { locations?: string[] }).locations || [], - argumentsToPass: ['-f', 'json', '--no-summary', '--no-snippets'].concat(argumentsToPass), + locations: ((knownArguments as { locations?: string[] }).locations || []) + .concat(test !== this.adapterId ? [test] : []), + argumentsToPass: ['-f', 'json', '--no-summary', '--no-snippets'] + .concat(argumentsToPass) }; }