Skip to content

Commit

Permalink
♻️ Refactor reporter code (#31)
Browse files Browse the repository at this point in the history
* Add example test

* Refactor reporter code

* Fix verbosity levels again

* Fix eslint warnings
  • Loading branch information
tolauwae authored Dec 12, 2024
1 parent 66f715c commit 02dfd40
Show file tree
Hide file tree
Showing 15 changed files with 630 additions and 345 deletions.
6 changes: 4 additions & 2 deletions .eslint.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,17 @@ import stylisticJs from '@stylistic/eslint-plugin-js'

/** @type {import('eslint').Linter.Config[]} */
export default [
pluginJs.configs.recommended,
...tseslint.configs.recommended,
{ plugins: {
'@stylistic/js': stylisticJs
}, rules: {
'@stylistic/js/indent': ['error', 4],
'@typescript-eslint/no-wrapper-object-types': 'off'
}
},
{files: ["**/*.{js,mjs,cjs,ts}"]},
{files: ["**/*.js"], languageOptions: {sourceType: "commonjs"}},
{languageOptions: { globals: globals.browser }},
pluginJs.configs.recommended,
...tseslint.configs.recommended,

];
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
"test:prebuild": "npm run build && npm run build:tests",
"test:all": "npm run test:prebuild && npm run test:ava",
"test:ava": "ava",
"test:example": "npx ts-node ./tests/examples/example.ts",
"coverage:test:ava": "c8 --src src/ --all ava"
},
"dependencies": {
Expand Down
17 changes: 9 additions & 8 deletions src/framework/Framework.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@ import {HybridScheduler, Scheduler} from './Scheduler';
import {TestScenario} from './scenario/TestScenario';

import {TestbedSpecification} from '../testbeds/TestbedSpecification';
import {Reporter, SuiteResults} from '../reporter/Reporter';

import {StyleType} from '../reporter';
import {styling} from '../reporter/Style';
import {SuiteResult} from '../reporter/Results';
import {Reporter} from '../reporter/Reporter';

export interface Suite {

Expand Down Expand Up @@ -90,10 +91,10 @@ export class Framework {
for (const suite of suites) {
for (const testee of suite.testees) {
const order: TestScenario[] = suite.scheduler.sequential(suite);
const result: SuiteResults = new SuiteResults(suite, testee);
const result: SuiteResult = new SuiteResult(suite);

const first: TestScenario = order[0];
await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error = e));
await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message)));

await this.runSuite(result, testee, order);
this.reporter.report(result);
Expand All @@ -114,10 +115,10 @@ export class Framework {
await Promise.all(suites.map(async (suite: Suite) => {
await Promise.all(suite.testees.map(async (testee: Testee) => {
const order: TestScenario[] = suite.scheduler.sequential(suite);
const result: SuiteResults = new SuiteResults(suite, testee);
const result: SuiteResult = new SuiteResult(suite);

const first: TestScenario = order[0];
await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error = e));
await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message)));

await this.runSuite(result, testee, order);
this.reporter.report(result);
Expand All @@ -139,10 +140,10 @@ export class Framework {
const order: TestScenario[][] = suite.scheduler.parallel(suite, suite.testees.length);
await Promise.all(suite.testees.map(async (testee: Testee, i: number) => {
// console.log(`scheduling on ${testee.name}`)
const result: SuiteResults = new SuiteResults(suite, testee);
const result: SuiteResult = new SuiteResult(suite);

const first: TestScenario = order[i][0];
await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error = e));
await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message)));

for (let j = i; j < order.length; j += suite.testees.length) {
await this.runSuite(result, testee, order[j]);
Expand All @@ -159,7 +160,7 @@ export class Framework {
this.reporter.results(t1 - t0);
}

private async runSuite(result: SuiteResults, testee: Testee, order: TestScenario[]) {
private async runSuite(result: SuiteResult, testee: Testee, order: TestScenario[]) {
for (const test of order) {
await testee.describe(test, result, this.runs);
}
Expand Down
79 changes: 39 additions & 40 deletions src/framework/Testee.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,11 @@ import {TestScenario} from './scenario/TestScenario';
import {OutofPlaceSpecification, PlatformType, TestbedSpecification} from '../testbeds/TestbedSpecification';
import {CompileOutput, CompilerFactory} from '../manage/Compiler';
import {WABT} from '../util/env';
import {Completion, expect, ScenarioResult, SuiteResults} from '../reporter/Reporter';
import {Outcome} from '../reporter/describers/Describer';
import {WASM} from '../sourcemap/Wasm';
import {DummyProxy} from '../testbeds/Emulator';
import {Result} from '../reporter/Result';
import {ScenarioResult, Skipped, StepOutcome, SuiteResult} from '../reporter/Results';
import {Verifier} from './Verifier';

export function timeout<T>(label: string, time: number, promise: Promise<T>): Promise<T> {
if (time === 0) {
Expand Down Expand Up @@ -46,14 +47,14 @@ export function getValue(object: any, field: string): any {
}

export enum Target {
supervisor,
proxy
supervisor = 'supervisor',
proxy = 'proxy'
}

export class Testee { // TODO unified with testbed interface

/** The current state for each described test */
private states: Map<string, Result> = new Map<string, Result>();
private states: Map<string, StepOutcome> = new Map<string, StepOutcome>();

/** Factory to establish new connections to VMs */
public readonly connector: TestbedFactory;
Expand Down Expand Up @@ -132,16 +133,16 @@ export class Testee { // TODO unified with testbed interface
}

private run(name: string, limit: number, fn: () => Promise<any>) {
return timeout<Object | void>(name, limit, fn());
return timeout<object | void>(name, limit, fn());
}

private step(name: string, limit: number, fn: () => Promise<any>) {
return timeout<Object | void>(name, limit, fn());
return timeout<object | void>(name, limit, fn());
}

public async describe(description: TestScenario, suiteResult: SuiteResults, runs: number = 1) {
public async describe(description: TestScenario, suiteResult: SuiteResult, runs: number = 1) {
const testee = this;
const scenarioResult: ScenarioResult = new ScenarioResult(description, testee);
const scenarioResult: ScenarioResult = new ScenarioResult(description);

if (description.skip) {
return;
Expand All @@ -154,11 +155,11 @@ export class Testee { // TODO unified with testbed interface
await this.run('Check for failing dependencies', testee.timeout, async function () {
const failedDependencies: TestScenario[] = testee.failedDependencies(description);
if (failedDependencies.length > 0) {
testee.states.set(description.title, new Result('Skipping', 'Test has failing dependencies', Completion.skipped));
testee.states.set(description.title, new Skipped('Skipping', 'Test has failing dependencies'));
throw new Error(`Skipped: failed dependent tests: ${failedDependencies.map(dependence => dependence.title)}`);
}
}).catch((e: Error) => {
scenarioResult.error = e;
scenarioResult.error(e.message);
});

await this.run('Compile and upload program', testee.connector.timeout, async function () {
Expand All @@ -169,31 +170,31 @@ export class Testee { // TODO unified with testbed interface

const compiled: CompileOutput = await new CompilerFactory(WABT).pickCompiler(description.program).compile(description.program);
try {
await timeout<Object | void>(`uploading module`, testee.timeout, testee.bed()!.sendRequest(new SourceMap.Mapping(), Message.updateModule(compiled.file))).catch((e) => Promise.reject(e));
await timeout<object | void>(`uploading module`, testee.timeout, testee.bed()!.sendRequest(new SourceMap.Mapping(), Message.updateModule(compiled.file))).catch((e) => Promise.reject(e));
testee.current = description.program;
} catch (e) {
await testee.initialize(description.program, description.args ?? []).catch((o) => Promise.reject(o));
}
}).catch((e: Error | string) => {
if (typeof e === 'string') {
scenarioResult.error = new Error(e);
scenarioResult.error(e);
} else {
scenarioResult.error = e;
scenarioResult.error(e.toString());
}
});

await this.run('Get source mapping', testee.connector.timeout, async function () {
map = await testee.mapper.map(description.program);
}).catch((e: Error | string) => {
if (typeof e === 'string') {
scenarioResult.error = new Error(e);
scenarioResult.error(e);
} else {
scenarioResult.error = e;
scenarioResult.error(e.toString());
}
});

if (scenarioResult.error) {
suiteResult.scenarios.push(scenarioResult);
if (scenarioResult.outcome === Outcome.error) {
suiteResult.add(scenarioResult);
return;
}

Expand All @@ -205,62 +206,60 @@ export class Testee { // TODO unified with testbed interface
await this.run('resetting before retry', testee.timeout, async function () {
await testee.reset(testee.testbed);
}).catch((e: Error) => {
scenarioResult.error = e;
scenarioResult.error(e.toString());
});
}

for (const step of description.steps ?? []) {
/** Perform the step and check if expectations were met */
await this.step(step.title, testee.timeout, async function () {
let result: Result = new Result(step.title, 'incomplete');
const verifier: Verifier = new Verifier(step);
if (testee.bed(step.target ?? Target.supervisor) === undefined) {
testee.states.set(description.title, result);
result.error('Cannot run test: no debugger connection.');
testee.states.set(description.title, result);
testee.states.set(description.title, verifier.error('Cannot run test: no debugger connection.'));
return;
}

let actual: Object | void;
let actual: object | void;
if (step.instruction.kind === Kind.Action) {
actual = await timeout<Object | void>(`performing action . ${step.title}`, testee.timeout,
actual = await timeout<object | void>(`performing action . ${step.title}`, testee.timeout,
step.instruction.value.act(testee)).catch((err) => {
result.error(err);
testee.states.set(description.title, verifier.error(err));
return;
});
} else {
actual = await testee.recoverable(testee, step.instruction.value, map,
(testee, req, map) => timeout<Object | void>(`sending instruction ${req.type}`, testee.timeout,
(testee, req, map) => timeout<object | void>(`sending instruction ${req.type}`, testee.timeout,
testee.bed(step.target ?? Target.supervisor)!.sendRequest(map, req)),
(testee) => testee.run(`Recover: re-initialize ${testee.testbed?.name}`, testee.connector.timeout, async function () {
await testee.initialize(description.program, description.args ?? []).catch((o) => {
return Promise.reject(o)
});
}), 1).catch((e: Error) => {
result.completion = (e.message.includes('timeout')) ? Completion.timedout : Completion.error;
result.description = e.message;
const result = new StepOutcome(step);
testee.states.set(description.title, result.update((e.message.includes('timeout')) ? Outcome.timedout : Outcome.error, e.message));
});
}

if (result.completion === Completion.uncommenced) {
result = expect(step, actual, previous);
}
const result = verifier.verify(actual, previous);

if (actual !== undefined) {
previous = actual;
}

testee.states.set(description.title, result);
scenarioResult.results.push(result);
scenarioResult.add(result);
});
}
suiteResult.scenarios.push(scenarioResult);
suiteResult.add(scenarioResult);
}
}

/* eslint @typescript-eslint/no-explicit-any: off */
private async recoverable(testee: Testee, step: Request<any>, map: SourceMap.Mapping,
attempt: (t: Testee, req: Request<any>, m: SourceMap.Mapping) => Promise<Object | void>,
attempt: (t: Testee, req: Request<any>, m: SourceMap.Mapping) => Promise<object | void>,
recover: (t: Testee) => Promise<any>,
retries: number = 0): Promise<Object | void> {
let result: Object | void = undefined;
retries: number = 0): Promise<object | void> {
let result: object | void = undefined;
let error;
while (0 <= retries && result === undefined) {
result = await attempt(testee, step, map).catch(async (err) => {
Expand All @@ -281,7 +280,7 @@ export class Testee { // TODO unified with testbed interface
if (instance === undefined) {
this.framework.reporter.error('Cannot run test: no debugger connection.'); // todo
} else {
await timeout<Object | void>('resetting vm', this.timeout, this.testbed!.sendRequest(new SourceMap.Mapping(), Message.reset));
await timeout<object | void>('resetting vm', this.timeout, this.testbed!.sendRequest(new SourceMap.Mapping(), Message.reset));
}
}

Expand All @@ -293,8 +292,8 @@ export class Testee { // TODO unified with testbed interface
private failedDependencies(description: TestScenario): TestScenario[] {
return (description?.dependencies ?? []).filter(dependence => {
if (this.states.get(dependence.title)) {
const c = this.states.get(dependence.title)!.completion;
return !(c === Completion.succeeded || c === Completion.uncommenced);
const c = this.states.get(dependence.title)!.outcome;
return !(c === Outcome.succeeded || c === Outcome.uncommenced);
} else {
return false;
}
Expand Down
Loading

0 comments on commit 02dfd40

Please sign in to comment.