diff --git a/package-lock.json b/package-lock.json index 041e305..3eee890 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "playwright-trx-reporter", - "version": "1.0.7", + "version": "1.0.8", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "playwright-trx-reporter", - "version": "1.0.7", + "version": "1.0.8", "license": "ISC", "dependencies": { "uuid": "^9.0.0" diff --git a/src/trxReporter.ts b/src/trxReporter.ts index 3ae158a..d5e4e0e 100644 --- a/src/trxReporter.ts +++ b/src/trxReporter.ts @@ -42,6 +42,13 @@ export interface TrxReporterOptions { * @default false */ verbose?: boolean; + + /** + * Determines how to treat playwright test retries. By default they are listed as individual tests any of which could fail. + * + * @default false + */ + groupRetriesAsSingleTest?: boolean; } const outputFileEnv = 'PLAYWRIGHT_TRX_OUTPUT_NAME'; @@ -67,12 +74,15 @@ export class TrxReporter implements Reporter { private priorityAnnotation: string; + private groupRetriesAsSingleTest = false; + constructor(options: TrxReporterOptions = {}) { const outputFilePath = (typeof options.outputFile === 'string' ? options.outputFile : undefined) || process.env[outputFileEnv]; this.outputFileInfo = outputFilePath ?? options.outputFile; this.ownerAnnotation = options.ownerAnnotation ?? 'owner'; this.priorityAnnotation = options.priorityAnnotation ?? 'priority'; this.verbose = options.verbose ?? false; + this.groupRetriesAsSingleTest = options.groupRetriesAsSingleTest ?? false; } log(str: string) { @@ -104,6 +114,7 @@ export class TrxReporter implements Reporter { const testRuns = finalTrxsBuilder.analytics(this.suite, { ownerAnnotation: this.ownerAnnotation, priorityAnnotation: this.priorityAnnotation, + groupRetriesAsSingleTest: this.groupRetriesAsSingleTest, testRunBuilderOptions: { id: createUuid(), name: `${finalRunUser} ${this.startTimeDate.toISOString()}`, diff --git a/src/trxWriter.ts b/src/trxWriter.ts index 80785d1..3af50a7 100644 --- a/src/trxWriter.ts +++ b/src/trxWriter.ts @@ -35,6 +35,11 @@ export interface TrxWriterOptions { */ priorityAnnotation: string, + /** + * How to treat playwright test retries. By default they are listed as individual tests any of which could fail. + */ + groupRetriesAsSingleTest: boolean, + /** * The constructor options of `TestRunBuilder` */ @@ -70,8 +75,8 @@ function mergeFileOrGroupSuite(testRunsBuilder: TestRunsBuilder, suite: Suite, o } function mergeTestCase(testRunsBuilder: TestRunsBuilder, test: TestCase, options: TrxWriterOptions) { - const { ownerAnnotation, priorityAnnotation } = options; - const trxUnitTestResults = buildTrxUnitTestResultByPwTestCase(test); + const { ownerAnnotation, priorityAnnotation, groupRetriesAsSingleTest } = options; + const trxUnitTestResults = buildTrxUnitTestResultByPwTestCase(test, groupRetriesAsSingleTest); // TODO: use `formatTestTitle`? // remove root title, which is just empty @@ -93,9 +98,43 @@ function mergeTestCase(testRunsBuilder: TestRunsBuilder, test: TestCase, options }); } -function buildTrxUnitTestResultByPwTestCase(test: TestCase): UnitTestResultType[] { - // TODO: assert test.results is sorted by retry index. - return test.results.map((result) => buildTrxUnitTestResultByPwTestResult(test, result)); +function buildTrxUnitTestResultByPwTestCase(test: TestCase, groupRetriesAsSingleTest: boolean = false): UnitTestResultType[] { + + if (groupRetriesAsSingleTest) { + // Tests are grouped togeter with any retries. The last retry will have the final result (i.e. if every retry passed, the test will be marked as passed). + const overallResult = pwTestCaseOutcome2TrxOutcome(test.outcome()); + + const firstResult = test.results[0]; + const lastResult = test.results[test.results.length - 1]; + + const endTime = new Date(lastResult.startTime); + endTime.setMilliseconds(lastResult.startTime.getMilliseconds() + lastResult.duration); + + const totalDuration = lastResult.startTime.getMilliseconds() + lastResult.duration - firstResult.startTime.getMilliseconds(); + const unitTestResult = new UnitTestResultType({ + $computerName: computerName, + $testId: convertPwId2Uuid(test.id), + $testListId: RESULT_NOT_IN_A_LIST_ID, + $testName: test.titlePath().slice(1).join(NAME_SPLITTER), + $testType: UNIT_TEST_TYPE, + $duration: formatMs2TimeSpanString(totalDuration), + $startTime: firstResult.startTime.toISOString(), + $endTime: endTime.toISOString(), + $executionId: createUuid(), + $outcome: overallResult, + }); + + for (const result of test.results) { + bindAttachment(unitTestResult, test, result); + bindOutput(unitTestResult, test, result); + } + + return [unitTestResult]; + + } else { + // TODO: assert test.results is sorted by retry index. + return test.results.map((result) => buildTrxUnitTestResultByPwTestResult(test, result)); + } } // TODO: copy from playwright implementation, so that it would be easy to migrate @@ -144,9 +183,12 @@ function bindOutput(unitTestResult: UnitTestResultType, test: TestCase, result: } : undefined; if (stdOutString || stdErrString || errorInfo) { unitTestResult.Output = { - StdOut: stdOutString, - StdErr: stdErrString, - ErrorInfo: errorInfo, + StdOut: (unitTestResult.Output?.StdOut ?? '') + stdOutString, + StdErr: (unitTestResult.Output?.StdErr ?? '') + stdErrString, + ErrorInfo: { + Message: (unitTestResult.Output?.ErrorInfo?.Message ?? '') + (errorInfo?.Message ?? ''), + StackTrace: (unitTestResult.Output?.ErrorInfo?.StackTrace ?? '') + (errorInfo?.StackTrace ?? ''), + }, }; } } @@ -166,8 +208,9 @@ function bindAttachment(unitTestResult: UnitTestResultType, test: TestCase, resu } if (attachmentPaths.length !== 0) { + const oldAttachments = unitTestResult.ResultFiles?.ResultFile ?? []; unitTestResult.ResultFiles = { - ResultFile: attachmentPaths.map((p) => ({ $path: p })), + ResultFile: oldAttachments.concat(attachmentPaths.map((p) => ({ $path: p }))) }; } } @@ -193,6 +236,21 @@ function pwOutcome2TrxOutcome(outcome: TestStatus) { } } +function pwTestCaseOutcome2TrxOutcome(outcome: ReturnType) { + switch (outcome) { + case 'unexpected': + return TestOutcome.Failed; + case 'expected': + return TestOutcome.Passed; + case 'flaky': // count flaky tests as eventually passing + return TestOutcome.Passed + case 'skipped': + return TestOutcome.NotExecuted; + default: + assertNever(outcome); + } +} + function getFromAnnotationByType(annotations: TestCase['annotations'], type: string) { for (let index = annotations.length - 1; index >= 0; index -= 1) { const annotation = annotations[index];