Skip to content

Commit

Permalink
pytestExecutionAdapter tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@eleanorjboyd
eleanorjboyd committed Sep 13, 2023
1 parent cafe6b5 commit 738c494
Show file tree
Hide file tree
Showing 9 changed files with 84 additions and 88 deletions.
2 changes: 1 addition & 1 deletion src/client/testing/testController/common/server.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ import { traceError, traceInfo, traceLog } from '../../../logging';
import { DataReceivedEvent, ITestServer, TestCommandOptions } from './types';
import { ITestDebugLauncher, LaunchOptions } from '../../common/types';
import { UNITTEST_PROVIDER } from '../../common/constants';
import { containsHeaders, extractJsonPayload } from './utils';
import { createExecutionErrorPayload, extractJsonPayload } from './utils';
import { createDeferred } from '../../../common/utils/async';

export class PythonTestServer implements ITestServer, Disposable {
Expand Down
16 changes: 5 additions & 11 deletions src/client/testing/testController/common/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import { IExperimentService } from '../../../common/types';
import { IServiceContainer } from '../../../ioc/types';
import { DebugTestTag, ErrorTestItemOptions, RunTestTag } from './testItemUtilities';
import { DiscoveredTestItem, DiscoveredTestNode, ExecutionTestPayload, ITestResultResolver } from './types';
import { Deferred, createDeferred } from '../../../common/utils/async';

export function fixLogLines(content: string): string {
const lines = content.split(/\r?\n/g);
Expand All @@ -35,6 +36,10 @@ export const JSONRPC_UUID_HEADER = 'Request-uuid';
export const JSONRPC_CONTENT_LENGTH_HEADER = 'Content-Length';
export const JSONRPC_CONTENT_TYPE_HEADER = 'Content-Type';

export function createEOTDeferred(): Deferred<void> {
return createDeferred<void>();
}

export function extractJsonPayload(rawData: string, uuids: Array<string>): ExtractOutput {
/**
* Extracts JSON-RPC payload from the provided raw data.
Expand Down Expand Up @@ -66,17 +71,6 @@ export function extractJsonPayload(rawData: string, uuids: Array<string>): Extra
return { uuid: undefined, cleanedJsonData: undefined, remainingRawData: rawData };
}

export function containsHeaders(rawData: string): boolean {
/**
* Checks if the provided raw data contains JSON-RPC specific headers.
*/
return (
rawData.includes(JSONRPC_CONTENT_LENGTH_HEADER) &&
rawData.includes(JSONRPC_CONTENT_TYPE_HEADER) &&
rawData.includes(JSONRPC_UUID_HEADER)
);
}

export function checkUuid(uuid: string | undefined, uuids: Array<string>): string | undefined {
if (!uuid) {
// no UUID found, this could occurred if the payload is full yet so send back without erroring
Expand Down
4 changes: 2 additions & 2 deletions src/client/testing/testController/pytest/pytestExecutionAdapter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import { TestRun, Uri } from 'vscode';
import * as path from 'path';
import { IConfigurationService, ITestOutputChannel } from '../../../common/types';
import { Deferred, createDeferred } from '../../../common/utils/async';
import { traceError, traceInfo, traceLog, traceVerbose } from '../../../logging';
import { traceError, traceInfo, traceVerbose } from '../../../logging';
import {
DataReceivedEvent,
ExecutionTestPayload,
Expand Down Expand Up @@ -43,7 +43,7 @@ export class PytestTestExecutionAdapter implements ITestExecutionAdapter {
): Promise<ExecutionTestPayload> {
const uuid = this.testServer.createUUID(uri.fsPath);
traceVerbose(uri, testIds, debugBool);
const deferredTillEOT: Deferred<void> = createDeferred<void>();
const deferredTillEOT: Deferred<void> = utils.createEOTDeferred();
const dataReceivedDisposable = this.testServer.onRunDataReceived((e: DataReceivedEvent) => {
console.log('data received');
if (runInstance) {
Expand Down
4 changes: 3 additions & 1 deletion src/client/testing/testController/unittest/testDiscoveryAdapter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,9 @@ export class UnittestTestDiscoveryAdapter implements ITestDiscoveryAdapter {
dataReceivedDisposable.dispose();
};

await this.callSendCommand(options);
await this.callSendCommand(options, () => {
disposeDataReceiver?.(this.testServer);
});
await deferredTillEOT.promise;
disposeDataReceiver(this.testServer);
// placeholder until after the rewrite is adopted
Expand Down
70 changes: 24 additions & 46 deletions src/test/testing/common/testingAdapter.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,12 @@ suite('End to End Tests: test adapters', () => {
let pythonExecFactory: IPythonExecutionFactory;
let debugLauncher: ITestDebugLauncher;
let configService: IConfigurationService;
let testOutputChannel: ITestOutputChannel;
let serviceContainer: IServiceContainer;
let workspaceUri: Uri;
let testOutputChannel: typeMoq.IMock<ITestOutputChannel>;
let testController: TestController;
const unittestProvider: TestProvider = UNITTEST_PROVIDER;
const pytestProvider: TestProvider = PYTEST_PROVIDER;
const rootPathSmallWorkspace = path.join(
EXTENSION_ROOT_DIR_FOR_TESTS,
'src',
Expand Down Expand Up @@ -57,11 +60,8 @@ suite('End to End Tests: test adapters', () => {
configService = serviceContainer.get<IConfigurationService>(IConfigurationService);
pythonExecFactory = serviceContainer.get<IPythonExecutionFactory>(IPythonExecutionFactory);
debugLauncher = serviceContainer.get<ITestDebugLauncher>(ITestDebugLauncher);
testOutputChannel = serviceContainer.get<ITestOutputChannel>(ITestOutputChannel);
testController = serviceContainer.get<TestController>(ITestController);

// create mock resultResolver object

// create objects that were not injected
pythonTestServer = new PythonTestServer(pythonExecFactory, debugLauncher);
await pythonTestServer.serverReady();
Expand Down Expand Up @@ -368,17 +368,8 @@ suite('End to End Tests: test adapters', () => {
pythonExecFactory,
)
.then(() => {
// verification after discovery is complete
resultResolver.verify(
(x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny()),
typeMoq.Times.exactly(2),
);
// 1. Check the status is "success"
assert.strictEqual(actualData.status, 'success', "Expected status to be 'success'");
// 2. Confirm no errors
assert.strictEqual(actualData.error, null, "Expected no errors in 'error' field");
// 3. Confirm tests are found
assert.ok(actualData.result, 'Expected results to be present');
// verify that the _resolveExecution was called once per test
assert.strictEqual(callCount, 1, 'Expected _resolveExecution to be called once');
});
});
test('pytest execution adapter large workspace', async () => {
Expand Down Expand Up @@ -420,31 +411,17 @@ suite('End to End Tests: test adapters', () => {
onCancellationRequested: () => undefined,
} as any),
);
console.log('FROM TEST, do the run large');
await executionAdapter
.runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory)
.then(() => {
// resolve execution should be called 200 times since there are 200 tests run.
console.log('hit then');
assert.strictEqual(
errorMessages.length,
0,
['Test run was unsuccessful, the following errors were produced: \n', ...errorMessages].join('\n'),
);
resultResolver.verify(
(x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()),
typeMoq.Times.atLeast(2000),
);
})
.finally(() => {
console.log('hit finally large');
});
await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory).then(() => {
// verify that the _resolveExecution was called once per test
assert.strictEqual(callCount, 3, 'Expected _resolveExecution to be called once');
});
});
test('unittest execution adapter seg fault error handling', async () => {
const resultResolverMock: typeMoq.IMock<ITestResultResolver> = typeMoq.Mock.ofType<ITestResultResolver>();
const testId = `test_seg_fault.TestSegmentationFault.test_segfault`;
const testIds: string[] = [testId];
resultResolver
.setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()))
resultResolverMock
.setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny()))
.returns((data) => {
// do the following asserts for each time resolveExecution is called, should be called once per test.
// 1. Check the status is "success"
Expand All @@ -469,8 +446,8 @@ suite('End to End Tests: test adapters', () => {
const executionAdapter = new UnittestTestExecutionAdapter(
pythonTestServer,
configService,
testOutputChannel,
resultResolver.object,
testOutputChannel.object,
resultResolverMock.object,
);
const testRun = typeMoq.Mock.ofType<TestRun>();
testRun
Expand All @@ -482,17 +459,18 @@ suite('End to End Tests: test adapters', () => {
} as any),
);
await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object).finally(() => {
resultResolver.verify(
(x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()),
resultResolverMock.verify(
(x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny()),
typeMoq.Times.exactly(1),
);
});
});
test('pytest execution adapter seg fault error handling', async () => {
const resultResolverMock: typeMoq.IMock<ITestResultResolver> = typeMoq.Mock.ofType<ITestResultResolver>();
const testId = `${rootPathErrorWorkspace}/test_seg_fault.py::TestSegmentationFault::test_segfault`;
const testIds: string[] = [testId];
resultResolver
.setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny()))
resultResolverMock
.setup((x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny()))
.returns((data) => {
// do the following asserts for each time resolveExecution is called, should be called once per test.
// 1. Check the status is "success"
Expand All @@ -517,8 +495,8 @@ suite('End to End Tests: test adapters', () => {
const executionAdapter = new PytestTestExecutionAdapter(
pythonTestServer,
configService,
testOutputChannel,
resultResolver.object,
testOutputChannel.object,
resultResolverMock.object,
);
const testRun = typeMoq.Mock.ofType<TestRun>();
testRun
Expand All @@ -530,9 +508,9 @@ suite('End to End Tests: test adapters', () => {
} as any),
);
await executionAdapter.runTests(workspaceUri, testIds, false, testRun.object, pythonExecFactory).finally(() => {
resultResolver.verify(
resultResolverMock.verify(
(x) => x.resolveExecution(typeMoq.It.isAny(), typeMoq.It.isAny(), typeMoq.It.isAny()),
typeMoq.Times.exactly(4),
typeMoq.Times.exactly(1),
);
});
});
Expand Down
2 changes: 1 addition & 1 deletion src/test/testing/testController/helper.server.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import { PythonTestServer } from '../../../client/testing/testController/common/
import { ITestDebugLauncher } from '../../../client/testing/common/types';
import { Deferred, createDeferred } from '../../../client/common/utils/async';
import { MockChildProcess } from '../../mocks/mockChildProcess';
import { PAYLOAD_MULTI_CHUNK, PAYLOAD_SINGLE_CHUNK, PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY } from './payloadTestCases';
import { PAYLOAD_SINGLE_CHUNK, PAYLOAD_SPLIT_ACROSS_CHUNKS_ARRAY } from './payloadTestCases';

suite('Python Test Server', () => {
const FAKE_UUID = 'fake-uuid';
Expand Down
44 changes: 28 additions & 16 deletions src/test/testing/testController/pytest/pytestExecutionAdapter.unit.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import { ITestDebugLauncher, LaunchOptions } from '../../../../client/testing/co
import * as util from '../../../../client/testing/testController/common/utils';
import { EXTENSION_ROOT_DIR } from '../../../../client/constants';
import { MockChildProcess } from '../../../mocks/mockChildProcess';
import { traceInfo } from '../../../../client/logging';

suite('pytest test execution adapter', () => {
let testServer: typeMoq.IMock<ITestServer>;
Expand All @@ -33,7 +34,7 @@ suite('pytest test execution adapter', () => {
(global as any).EXTENSION_ROOT_DIR = EXTENSION_ROOT_DIR;
let myTestPath: string;
let mockProc: MockChildProcess;
let utilsStub: sinon.SinonStub;
let utilsStartServerStub: sinon.SinonStub;
setup(() => {
testServer = typeMoq.Mock.ofType<ITestServer>();
testServer.setup((t) => t.getPort()).returns(() => 12345);
Expand All @@ -51,6 +52,8 @@ suite('pytest test execution adapter', () => {
isTestExecution: () => false,
} as unknown) as IConfigurationService;

// mock out the result resolver

// set up exec service with child process
mockProc = new MockChildProcess('', ['']);
const output = new Observable<Output<string>>(() => {
Expand All @@ -67,7 +70,7 @@ suite('pytest test execution adapter', () => {
},
}));
execFactory = typeMoq.Mock.ofType<IPythonExecutionFactory>();
utilsStub = sinon.stub(util, 'startTestIdServer');
utilsStartServerStub = sinon.stub(util, 'startTestIdServer');
debugLauncher = typeMoq.Mock.ofType<ITestDebugLauncher>();
execFactory
.setup((x) => x.createActivatedEnvironment(typeMoq.It.isAny()))
Expand All @@ -79,13 +82,6 @@ suite('pytest test execution adapter', () => {
deferred.resolve();
return Promise.resolve({ stdout: '{}' });
});
debugLauncher
.setup((d) => d.launchDebugger(typeMoq.It.isAny(), typeMoq.It.isAny()))
.returns(() => {
deferred.resolve();
return Promise.resolve();
});

execFactory.setup((p) => ((p as unknown) as any).then).returns(() => undefined);
execService.setup((p) => ((p as unknown) as any).then).returns(() => undefined);
debugLauncher.setup((p) => ((p as unknown) as any).then).returns(() => undefined);
Expand All @@ -104,7 +100,7 @@ suite('pytest test execution adapter', () => {
deferred2.resolve();
return Promise.resolve(execService.object);
});
utilsStub.callsFake(() => {
utilsStartServerStub.callsFake(() => {
deferred3.resolve();
return Promise.resolve(54321);
});
Expand All @@ -131,7 +127,7 @@ suite('pytest test execution adapter', () => {
mockProc.trigger('close');

// assert
sinon.assert.calledWithExactly(utilsStub, testIds);
sinon.assert.calledWithExactly(utilsStartServerStub, testIds);
});
test('pytest execution called with correct args', async () => {
const deferred2 = createDeferred();
Expand All @@ -143,7 +139,7 @@ suite('pytest test execution adapter', () => {
deferred2.resolve();
return Promise.resolve(execService.object);
});
utilsStub.callsFake(() => {
utilsStartServerStub.callsFake(() => {
deferred3.resolve();
return Promise.resolve(54321);
});
Expand Down Expand Up @@ -175,7 +171,6 @@ suite('pytest test execution adapter', () => {
TEST_UUID: 'uuid123',
TEST_PORT: '12345',
};
// execService.verify((x) => x.exec(expectedArgs, typeMoq.It.isAny()), typeMoq.Times.once());
execService.verify(
(x) =>
x.execObservable(
Expand Down Expand Up @@ -203,7 +198,7 @@ suite('pytest test execution adapter', () => {
deferred2.resolve();
return Promise.resolve(execService.object);
});
utilsStub.callsFake(() => {
utilsStartServerStub.callsFake(() => {
deferred3.resolve();
return Promise.resolve(54321);
});
Expand Down Expand Up @@ -262,12 +257,28 @@ suite('pytest test execution adapter', () => {
});
test('Debug launched correctly for pytest', async () => {
const deferred3 = createDeferred();
utilsStub.callsFake(() => {
const deferredEOT = createDeferred();
utilsStartServerStub.callsFake(() => {
deferred3.resolve();
return Promise.resolve(54321);
});
debugLauncher
.setup((dl) => dl.launchDebugger(typeMoq.It.isAny(), typeMoq.It.isAny()))
.returns(async () => {
traceInfo('stubs launch debugger');
deferredEOT.resolve();
});
const utilsCreateEOTStub: sinon.SinonStub = sinon.stub(util, 'createEOTDeferred');
utilsCreateEOTStub.callsFake(() => deferredEOT);
const testRun = typeMoq.Mock.ofType<TestRun>();
testRun.setup((t) => t.token).returns(() => ({ onCancellationRequested: () => undefined } as any));
testRun
.setup((t) => t.token)
.returns(
() =>
({
onCancellationRequested: () => undefined,
} as any),
);
const uri = Uri.file(myTestPath);
const uuid = 'uuid123';
testServer
Expand Down Expand Up @@ -298,5 +309,6 @@ suite('pytest test execution adapter', () => {
),
typeMoq.Times.once(),
);
testServer.verify((x) => x.deleteUUID(typeMoq.It.isAny()), typeMoq.Times.once());
});
});
Loading

0 comments on commit 738c494

Please sign in to comment.