Skip to content

Commit

Permalink
Refactor reporter code
Browse files Browse the repository at this point in the history
  • Loading branch information
tolauwae committed Dec 11, 2024
1 parent edffed5 commit 1edcadd
Show file tree
Hide file tree
Showing 11 changed files with 569 additions and 325 deletions.
17 changes: 9 additions & 8 deletions src/framework/Framework.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,11 @@ import {HybridScheduler, Scheduler} from './Scheduler';
import {TestScenario} from './scenario/TestScenario';

import {TestbedSpecification} from '../testbeds/TestbedSpecification';
import {Reporter, SuiteResults} from '../reporter/Reporter';

import {StyleType} from '../reporter';
import {styling} from '../reporter/Style';
import {SuiteResult} from '../reporter/Outcomes';
import {Reporter} from '../reporter/Reporter';

export interface Suite {

Expand Down Expand Up @@ -90,10 +91,10 @@ export class Framework {
for (const suite of suites) {
for (const testee of suite.testees) {
const order: TestScenario[] = suite.scheduler.sequential(suite);
const result: SuiteResults = new SuiteResults(suite, testee);
const result: SuiteResult = new SuiteResult(suite);

const first: TestScenario = order[0];
await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error = e));
await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message)));

await this.runSuite(result, testee, order);
this.reporter.report(result);
Expand All @@ -114,10 +115,10 @@ export class Framework {
await Promise.all(suites.map(async (suite: Suite) => {
await Promise.all(suite.testees.map(async (testee: Testee) => {
const order: TestScenario[] = suite.scheduler.sequential(suite);
const result: SuiteResults = new SuiteResults(suite, testee);
const result: SuiteResult = new SuiteResult(suite);

const first: TestScenario = order[0];
await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error = e));
await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message)));

await this.runSuite(result, testee, order);
this.reporter.report(result);
Expand All @@ -139,10 +140,10 @@ export class Framework {
const order: TestScenario[][] = suite.scheduler.parallel(suite, suite.testees.length);
await Promise.all(suite.testees.map(async (testee: Testee, i: number) => {
// console.log(`scheduling on ${testee.name}`)
const result: SuiteResults = new SuiteResults(suite, testee);
const result: SuiteResult = new SuiteResult(suite);

const first: TestScenario = order[i][0];
await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error = e));
await timeout<Object | void>('Initialize testbed', testee.connector.timeout, testee.initialize(first.program, first.args ?? []).catch((e: Error) => result.error(e.message)));

for (let j = i; j < order.length; j += suite.testees.length) {
await this.runSuite(result, testee, order[j]);
Expand All @@ -159,7 +160,7 @@ export class Framework {
this.reporter.results(t1 - t0);
}

private async runSuite(result: SuiteResults, testee: Testee, order: TestScenario[]) {
private async runSuite(result: SuiteResult, testee: Testee, order: TestScenario[]) {
for (const test of order) {
await testee.describe(test, result, this.runs);
}
Expand Down
58 changes: 28 additions & 30 deletions src/framework/Testee.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,11 @@ import {TestScenario} from './scenario/TestScenario';
import {OutofPlaceSpecification, PlatformType, TestbedSpecification} from '../testbeds/TestbedSpecification';
import {CompileOutput, CompilerFactory} from '../manage/Compiler';
import {WABT} from '../util/env';
import {Completion, expect, ScenarioResult, SuiteResults} from '../reporter/Reporter';
import {Outcome} from '../reporter/verbosity/Describer';
import {WASM} from '../sourcemap/Wasm';
import {DummyProxy} from '../testbeds/Emulator';
import {Result} from '../reporter/Result';
import {ScenarioResult, Skipped, StepOutcome, SuiteResult} from '../reporter/Outcomes';
import {Verifier} from './Verifier';

export function timeout<T>(label: string, time: number, promise: Promise<T>): Promise<T> {
if (time === 0) {
Expand Down Expand Up @@ -46,14 +47,14 @@ export function getValue(object: any, field: string): any {
}

export enum Target {
supervisor,
proxy
supervisor = 'supervisor',
proxy = 'proxy'
}

export class Testee { // TODO unified with testbed interface

/** The current state for each described test */
private states: Map<string, Result> = new Map<string, Result>();
private states: Map<string, StepOutcome> = new Map<string, StepOutcome>();

/** Factory to establish new connections to VMs */
public readonly connector: TestbedFactory;
Expand Down Expand Up @@ -139,9 +140,9 @@ export class Testee { // TODO unified with testbed interface
return timeout<Object | void>(name, limit, fn());
}

public async describe(description: TestScenario, suiteResult: SuiteResults, runs: number = 1) {
public async describe(description: TestScenario, suiteResult: SuiteResult, runs: number = 1) {
const testee = this;
const scenarioResult: ScenarioResult = new ScenarioResult(description, testee);
const scenarioResult: ScenarioResult = new ScenarioResult(description);

if (description.skip) {
return;
Expand All @@ -154,11 +155,11 @@ export class Testee { // TODO unified with testbed interface
await this.run('Check for failing dependencies', testee.timeout, async function () {
const failedDependencies: TestScenario[] = testee.failedDependencies(description);
if (failedDependencies.length > 0) {
testee.states.set(description.title, new Result('Skipping', 'Test has failing dependencies', Completion.skipped));
testee.states.set(description.title, new Skipped('Skipping', 'Test has failing dependencies'));
throw new Error(`Skipped: failed dependent tests: ${failedDependencies.map(dependence => dependence.title)}`);
}
}).catch((e: Error) => {
scenarioResult.error = e;
scenarioResult.error(e.message);
});

await this.run('Compile and upload program', testee.connector.timeout, async function () {
Expand All @@ -176,24 +177,24 @@ export class Testee { // TODO unified with testbed interface
}
}).catch((e: Error | string) => {
if (typeof e === 'string') {
scenarioResult.error = new Error(e);
scenarioResult.error(e);
} else {
scenarioResult.error = e;
scenarioResult.error(e.toString());
}
});

await this.run('Get source mapping', testee.connector.timeout, async function () {
map = await testee.mapper.map(description.program);
}).catch((e: Error | string) => {
if (typeof e === 'string') {
scenarioResult.error = new Error(e);
scenarioResult.error(e);
} else {
scenarioResult.error = e;
scenarioResult.error(e.toString());
}
});

if (scenarioResult.error) {
suiteResult.scenarios.push(scenarioResult);
if (scenarioResult.outcome === Outcome.error) {
suiteResult.add(scenarioResult);
return;
}

Expand All @@ -205,26 +206,25 @@ export class Testee { // TODO unified with testbed interface
await this.run('resetting before retry', testee.timeout, async function () {
await testee.reset(testee.testbed);
}).catch((e: Error) => {
scenarioResult.error = e;
scenarioResult.error(e.toString());
});
}

for (const step of description.steps ?? []) {
/** Perform the step and check if expectations were met */
await this.step(step.title, testee.timeout, async function () {
let result: Result = new Result(step.title, 'incomplete');
const verifier: Verifier = new Verifier(step);
if (testee.bed(step.target ?? Target.supervisor) === undefined) {
testee.states.set(description.title, result);
result.error('Cannot run test: no debugger connection.');
testee.states.set(description.title, result);
testee.states.set(description.title, verifier.error('Cannot run test: no debugger connection.'));
return;
}

let actual: Object | void;
if (step.instruction.kind === Kind.Action) {
actual = await timeout<Object | void>(`performing action . ${step.title}`, testee.timeout,
step.instruction.value.act(testee)).catch((err) => {
result.error(err);
testee.states.set(description.title, verifier.error(err));
return;
});
} else {
actual = await testee.recoverable(testee, step.instruction.value, map,
Expand All @@ -235,24 +235,22 @@ export class Testee { // TODO unified with testbed interface
return Promise.reject(o)
});
}), 1).catch((e: Error) => {
result.completion = (e.message.includes('timeout')) ? Completion.timedout : Completion.error;
result.description = e.message;
const result = new StepOutcome(step);
testee.states.set(description.title, result.update((e.message.includes('timeout')) ? Outcome.timedout : Outcome.error, e.message));
});
}

if (result.completion === Completion.uncommenced) {
result = expect(step, actual, previous);
}
const result = verifier.verify(actual, previous);

if (actual !== undefined) {
previous = actual;
}

testee.states.set(description.title, result);
scenarioResult.results.push(result);
scenarioResult.add(result);
});
}
suiteResult.scenarios.push(scenarioResult);
suiteResult.add(scenarioResult);
}
}

Expand Down Expand Up @@ -293,8 +291,8 @@ export class Testee { // TODO unified with testbed interface
private failedDependencies(description: TestScenario): TestScenario[] {
return (description?.dependencies ?? []).filter(dependence => {
if (this.states.get(dependence.title)) {
const c = this.states.get(dependence.title)!.completion;
return !(c === Completion.succeeded || c === Completion.uncommenced);
const c = this.states.get(dependence.title)!.outcome;
return !(c === Outcome.succeeded || c === Outcome.uncommenced);
} else {
return false;
}
Expand Down
127 changes: 127 additions & 0 deletions src/framework/Verifier.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
import {Behaviour, Description, Step} from './scenario/Step';
import {StepOutcome} from '../reporter/Outcomes';
import {getValue} from './Testee';
import {Outcome} from '../reporter/verbosity/Describer';
import {bold} from 'ansi-colors';

// decorator for Step class
export class Verifier {
public readonly step: Step;

constructor(step: Step) {
this.step = step;
}

public verify(actual: Object | void, previous?: Object): StepOutcome {
let result = new StepOutcome(this.step).update(Outcome.succeeded);
for (const expectation of this.step.expected ?? []) {
for (const [field, entry] of Object.entries(expectation)) {
try {
const value = getValue(actual, field);

if (entry.kind === 'primitive') {
result = this.expectPrimitive(value, entry.value);
} else if (entry.kind === 'description') {
result = this.expectDescription(value, entry.value);
} else if (entry.kind === 'comparison') {
result = this.expectComparison(actual, value, entry.value, entry.message);
} else if (entry.kind === 'behaviour') {
if (previous === undefined) {
return this.error('Invalid test: no [previous] to compare behaviour to.');
}
result = this.expectBehaviour(value, getValue(previous, field), entry.value);
}
} catch (e) {
return this.error(`Failure: ${JSON.stringify(actual)} state does not contain '${field}'.`);
}

if (result.outcome !== Outcome.succeeded) {
return result;
}
}
}
return result;
}

public error(clarification: string): StepOutcome {
const result: StepOutcome = new StepOutcome(this.step);
result.update(Outcome.succeeded);
return result.update(Outcome.error, clarification);
}

private expectPrimitive<T>(actual: T, expected: T): StepOutcome {
const result: StepOutcome = new StepOutcome(this.step);
if (deepEqual(actual, expected)) {
result.update(Outcome.succeeded);
} else {
result.update(Outcome.failed, `Expected ${bold(`${expected}`)} got ${bold(`${actual}`)}`);
}
return result;
}

private expectDescription<T>(actual: T, value: Description): StepOutcome {
const result: StepOutcome = new StepOutcome(this.step);
if ((value === Description.defined && actual !== undefined) ||
value === Description.notDefined && actual === undefined) {
result.update(Outcome.succeeded);
} else {
result.update(Outcome.failed, value === Description.defined ? 'Should exist' : 'Unexpected field');
}
return result;
}

private expectComparison<T>(state: Object | void, actual: T, comparator: (state: Object, value: T) => boolean, message?: string): StepOutcome {
const result: StepOutcome = new StepOutcome(this.step);
if (state === undefined) {
result.update(Outcome.failed, `Got unexpected ${state}`);
return result;
}

if (comparator(state, actual)) {
result.update(Outcome.succeeded);
} else {
result.update(Outcome.failed, 'custom comparator failed');
}

return result;
}

private expectBehaviour(actual: any, previous: any, behaviour: Behaviour): StepOutcome {
const result: StepOutcome = new StepOutcome(this.step);
switch (behaviour) {
case Behaviour.unchanged:
if (deepEqual(actual, previous)) {
result.update(Outcome.succeeded);
} else {
result.update(Outcome.failed, `Expected ${actual} to equal ${previous}`);
}
break;
case Behaviour.changed:
if (!deepEqual(actual, previous)) {
result.update(Outcome.succeeded);
} else {
result.update(Outcome.failed, `Expected ${actual} to be different from ${previous}`);
}
break;
case Behaviour.increased:
if (actual > previous) {
result.update(Outcome.succeeded);
} else {
result.update(Outcome.failed, `Expected ${actual} to be greater than ${previous}`);
}
break;
case Behaviour.decreased:
if (actual < previous) {
result.update(Outcome.succeeded);
} else {
result.update(Outcome.failed, `Expected ${actual} to be less than ${previous}`);
}
break;
}
return result;
}
}

function deepEqual(a: any, b: any): boolean {
return a === b || (isNaN(a) && isNaN(b));
}
Loading

0 comments on commit 1edcadd

Please sign in to comment.