diff --git a/libs/execution/src/lib/transforms/transform-executor.spec.ts b/libs/execution/src/lib/transforms/transform-executor.spec.ts new file mode 100644 index 000000000..b7ee6bb23 --- /dev/null +++ b/libs/execution/src/lib/transforms/transform-executor.spec.ts @@ -0,0 +1,397 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import assert = require('assert'); +import * as path from 'path'; + +import { + InternalValueRepresentation, + PrimitiveValuetypes, + TransformDefinition, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { constructTable, getTestExecutionContext } from '../../../test/utils'; +import { Table, TableColumn } from '../types/io-types/table'; + +import { PortDetails, TransformExecutor } from './transform-executor'; + +describe('Validation of TransformExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../../test/assets/', + ); + + function getColumnsMap( + inputColumnNames: string[], + inputTable: Table, + transformInputDetailsList: PortDetails[], + ): Map { + const variableToColumnMap: Map = new Map(); + for (let i = 0; i < inputColumnNames.length; ++i) { + const inputColumnName = inputColumnNames[i]; + assert(inputColumnName !== undefined); + const inputColumn = inputTable.getColumn(inputColumnName); + assert(inputColumn !== undefined); + + const matchingInputDetails = transformInputDetailsList[i]; + assert(matchingInputDetails !== undefined); + + const variableName = matchingInputDetails.port.name; + variableToColumnMap.set(variableName, inputColumn); + } + return variableToColumnMap; + } + + async function parseAndExecuteTransform( + input: string, + inputTable: Table, + columnNames: string[], + ): Promise<{ + resultingColumn: TableColumn; + rowsToDelete: number[]; + }> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const transform = locator.getAstNode( + document.parseResult.value, + 'transforms@0', + ) as TransformDefinition; + const executor = new TransformExecutor(transform); + + return executor.executeTransform( + getColumnsMap(columnNames, inputTable, executor.getInputDetails()), + inputTable.getNumberOfRows(), + getTestExecutionContext(locator, document), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + + await loadTestExtensions(services, [ + path.resolve( + __dirname, + '../../../test/assets/transform-executor/test-extension/TestBlockTypes.jv', + ), + ]); + + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + it('should diagnose no error on valid value', async () => { + const text = readJvTestAsset( + 'transform-executor/valid-decimal-integer-transform.jv', + ); + + const inputTable = constructTable( + [ + { + columnName: 'Column1', + column: { + values: ['value 1'], + valuetype: PrimitiveValuetypes.Text, + }, + }, + { + columnName: 'Column2', + column: { + values: [20.2], + valuetype: PrimitiveValuetypes.Decimal, + }, + }, + ], + 1, + ); + const transformColumnNames: string[] = ['Column2']; + + const result = await parseAndExecuteTransform( + text, + inputTable, + transformColumnNames, + ); + + expect(result.rowsToDelete).toHaveLength(0); + expect(result.resultingColumn.valuetype).toEqual( + PrimitiveValuetypes.Integer, + ); + expect(result.resultingColumn.values).toHaveLength(1); + expect(result.resultingColumn.values).toEqual(expect.arrayContaining([21])); + }); + + it('should diagnose no error on invalid value representation', async () => { + const text = readJvTestAsset( + 'transform-executor/invalid-input-output-type-transform.jv', + ); + + const inputTable = constructTable( + [ + { + columnName: 'Column1', + column: { + values: ['value 1'], + valuetype: PrimitiveValuetypes.Text, + }, + }, + { + columnName: 'Column2', + column: { + values: [20.0], + valuetype: PrimitiveValuetypes.Decimal, + }, + }, + ], + 1, + ); + const transformColumnNames: string[] = ['Column2']; + + const result = await parseAndExecuteTransform( + text, + inputTable, + transformColumnNames, + ); + + expect(result.rowsToDelete).toHaveLength(1); + expect(result.rowsToDelete).toEqual(expect.arrayContaining([0])); + expect(result.resultingColumn.valuetype).toEqual(PrimitiveValuetypes.Text); + expect(result.resultingColumn.values).toHaveLength(0); + }); + + it('should diagnose no error on valid value', async () => { + const text = readJvTestAsset( + 'transform-executor/valid-multiple-input-transform.jv', + ); + + const inputTable = constructTable( + [ + { + columnName: 'Column1', + column: { + values: ['value 1'], + valuetype: PrimitiveValuetypes.Text, + }, + }, + { + columnName: 'Column2', + column: { + values: [20.2], + valuetype: PrimitiveValuetypes.Decimal, + }, + }, + { + columnName: 'Column3', + column: { + values: [85.978], + valuetype: PrimitiveValuetypes.Decimal, + }, + }, + ], + 1, + ); + const transformColumnNames: string[] = ['Column2', 'Column3']; + + const result = await parseAndExecuteTransform( + text, + inputTable, + transformColumnNames, + ); + + expect(result.rowsToDelete).toHaveLength(0); + expect(result.resultingColumn.valuetype).toEqual( + PrimitiveValuetypes.Integer, + ); + expect(result.resultingColumn.values).toHaveLength(1); + expect(result.resultingColumn.values).toEqual( + expect.arrayContaining([106]), + ); + }); + + it('should diagnose error on empty columns map', async () => { + const text = readJvTestAsset( + 'transform-executor/valid-decimal-integer-transform.jv', + ); + + const inputTable = constructTable( + [ + { + columnName: 'Column1', + column: { + values: ['value 1'], + valuetype: PrimitiveValuetypes.Text, + }, + }, + { + columnName: 'Column2', + column: { + values: [20.2], + valuetype: PrimitiveValuetypes.Decimal, + }, + }, + ], + 1, + ); + const transformColumnNames: string[] = []; + + try { + const result = await parseAndExecuteTransform( + text, + inputTable, + transformColumnNames, + ); + expect(result).toEqual(undefined); + } catch (e) { + expect(e).toBeInstanceOf(assert.AssertionError); + expect((e as assert.AssertionError).stack).toEqual( + expect.stringContaining('at TransformExecutor.addVariablesToContext'), + ); + expect((e as assert.AssertionError).expected).toEqual(true); + expect((e as assert.AssertionError).actual).toEqual(false); + } + }); + + it('should diagnose no error on invalid column type', async () => { + const text = readJvTestAsset( + 'transform-executor/valid-decimal-integer-transform.jv', + ); + + const inputTable = constructTable( + [ + { + columnName: 'Column1', + column: { + values: ['value 1'], + valuetype: PrimitiveValuetypes.Text, + }, + }, + { + columnName: 'Column2', + column: { + values: [20.2], + valuetype: PrimitiveValuetypes.Decimal, + }, + }, + ], + 1, + ); + const transformColumnNames: string[] = ['Column1']; + + const result = await parseAndExecuteTransform( + text, + inputTable, + transformColumnNames, + ); + + expect(result.rowsToDelete).toHaveLength(1); + expect(result.resultingColumn.valuetype).toEqual( + PrimitiveValuetypes.Integer, + ); + expect(result.resultingColumn.values).toHaveLength(0); + }); + + it('should diagnose no error on invalid row value', async () => { + const text = readJvTestAsset( + 'transform-executor/valid-decimal-integer-transform.jv', + ); + + const inputTable = constructTable( + [ + { + columnName: 'Column1', + column: { + values: ['value 1', 'value 2'], + valuetype: PrimitiveValuetypes.Text, + }, + }, + { + columnName: 'Column2', + column: { + values: ['20.2', 20.1], + valuetype: PrimitiveValuetypes.Decimal, + }, + }, + ], + 2, + ); + const transformColumnNames: string[] = ['Column2']; + + const result = await parseAndExecuteTransform( + text, + inputTable, + transformColumnNames, + ); + + expect(result.rowsToDelete).toHaveLength(1); + expect(result.resultingColumn.valuetype).toEqual( + PrimitiveValuetypes.Integer, + ); + expect(result.resultingColumn.values).toHaveLength(1); + expect(result.resultingColumn.values).toEqual(expect.arrayContaining([21])); + }); + + it('should diagnose no error on expression evaluation error', async () => { + const text = readJvTestAsset( + 'transform-executor/invalid-expression-evaluation-error.jv', + ); + + const inputTable = constructTable( + [ + { + columnName: 'Column1', + column: { + values: ['value 1'], + valuetype: PrimitiveValuetypes.Text, + }, + }, + { + columnName: 'Column2', + column: { + values: [20.2], + valuetype: PrimitiveValuetypes.Decimal, + }, + }, + { + columnName: 'Column3', + column: { + values: [85.978], + valuetype: PrimitiveValuetypes.Decimal, + }, + }, + ], + 1, + ); + const transformColumnNames: string[] = ['Column2', 'Column1']; + + const result = await parseAndExecuteTransform( + text, + inputTable, + transformColumnNames, + ); + + expect(result.rowsToDelete).toHaveLength(1); + expect(result.resultingColumn.valuetype).toEqual( + PrimitiveValuetypes.Decimal, + ); + expect(result.resultingColumn.values).toHaveLength(0); + }); +}); diff --git a/libs/execution/src/lib/transforms/transform-executor.ts b/libs/execution/src/lib/transforms/transform-executor.ts index fda61bb49..98ce44dbc 100644 --- a/libs/execution/src/lib/transforms/transform-executor.ts +++ b/libs/execution/src/lib/transforms/transform-executor.ts @@ -95,10 +95,19 @@ export class TransformExecutor { for (let rowIndex = 0; rowIndex < numberOfRows; ++rowIndex) { this.addVariablesToContext(inputDetailsList, columns, rowIndex, context); - const newValue = evaluateExpression( - this.getOutputAssignment().expression, - context.evaluationContext, - ); + let newValue: InternalValueRepresentation | undefined = undefined; + try { + newValue = evaluateExpression( + this.getOutputAssignment().expression, + context.evaluationContext, + ); + } catch (e) { + if (e instanceof Error) { + context.logger.logDebug(e.message); + } else { + context.logger.logDebug(String(e)); + } + } if (newValue === undefined) { context.logger.logDebug( diff --git a/libs/execution/src/lib/types/io-types/table.ts b/libs/execution/src/lib/types/io-types/table.ts index 7ea3e24d3..baf6bacee 100644 --- a/libs/execution/src/lib/types/io-types/table.ts +++ b/libs/execution/src/lib/types/io-types/table.ts @@ -35,6 +35,10 @@ export class Table implements IOTypeImplementation { private columns: Map = new Map(); + public constructor(numberOfRows = 0) { + this.numberOfRows = numberOfRows; + } + addColumn(name: string, column: TableColumn): void { assert(column.values.length === this.numberOfRows); this.columns.set(name, column); diff --git a/libs/execution/test/assets/transform-executor/invalid-expression-evaluation-error.jv b/libs/execution/test/assets/transform-executor/invalid-expression-evaluation-error.jv new file mode 100644 index 000000000..ab8c77193 --- /dev/null +++ b/libs/execution/test/assets/transform-executor/invalid-expression-evaluation-error.jv @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +transform TestTransform { + from firstInputParam oftype decimal; + from secondInputParam oftype text; + to result oftype decimal; + + result: firstInputParam + secondInputParam; +} + +pipeline TestPipeline { + + block TestExtractor oftype TestFileExtractor { + } + + block TestLoader oftype TestFileLoader { + } + + TestExtractor -> TestLoader; +} diff --git a/libs/execution/test/assets/transform-executor/invalid-input-output-type-transform.jv b/libs/execution/test/assets/transform-executor/invalid-input-output-type-transform.jv new file mode 100644 index 000000000..ddb7a316c --- /dev/null +++ b/libs/execution/test/assets/transform-executor/invalid-input-output-type-transform.jv @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +transform TestTransform { + from inputParam oftype decimal; + to result oftype text; + + result: ceil(inputParam); +} + +pipeline TestPipeline { + + block TestExtractor oftype TestFileExtractor { + } + + block TestLoader oftype TestFileLoader { + } + + TestExtractor -> TestLoader; +} diff --git a/libs/execution/test/assets/transform-executor/test-extension/TestBlockTypes.jv b/libs/execution/test/assets/transform-executor/test-extension/TestBlockTypes.jv new file mode 100644 index 000000000..bf5bb4357 --- /dev/null +++ b/libs/execution/test/assets/transform-executor/test-extension/TestBlockTypes.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +builtin blocktype TestFileExtractor { + input inPort oftype None; + output outPort oftype File; +} + +builtin blocktype TestFileLoader { + input inPort oftype File; + output outPort oftype None; +} + +builtin blocktype TestTableLoader { + input inPort oftype Table; + output outPort oftype None; +} diff --git a/libs/execution/test/assets/transform-executor/valid-decimal-integer-transform.jv b/libs/execution/test/assets/transform-executor/valid-decimal-integer-transform.jv new file mode 100644 index 000000000..99a2f97b9 --- /dev/null +++ b/libs/execution/test/assets/transform-executor/valid-decimal-integer-transform.jv @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +transform TestTransform { + from inputParam oftype decimal; + to result oftype integer; + + result: ceil(inputParam); +} + +pipeline TestPipeline { + + block TestExtractor oftype TestFileExtractor { + } + + block TestLoader oftype TestFileLoader { + } + + TestExtractor -> TestLoader; +} diff --git a/libs/execution/test/assets/transform-executor/valid-multiple-input-transform.jv b/libs/execution/test/assets/transform-executor/valid-multiple-input-transform.jv new file mode 100644 index 000000000..9c56ce5ab --- /dev/null +++ b/libs/execution/test/assets/transform-executor/valid-multiple-input-transform.jv @@ -0,0 +1,22 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +transform TestTransform { + from firstInputParam oftype decimal; + from secondInputParam oftype decimal; + to result oftype integer; + + result: ceil(firstInputParam) + floor(secondInputParam); +} + +pipeline TestPipeline { + + block TestExtractor oftype TestFileExtractor { + } + + block TestLoader oftype TestFileLoader { + } + + TestExtractor -> TestLoader; +} diff --git a/libs/execution/test/utils.ts b/libs/execution/test/utils.ts index 639c4239e..671f10873 100644 --- a/libs/execution/test/utils.ts +++ b/libs/execution/test/utils.ts @@ -15,6 +15,8 @@ import { DebugTargets, ExecutionContext, StackNode, + Table, + TableColumn, blockExecutorRegistry, constraintExecutorRegistry, } from '../src'; @@ -65,3 +67,17 @@ export function getTestExecutionContext( return executionContext; } + +export interface TableColumnDefinition { + columnName: string; + column: TableColumn; +} + +export function constructTable( + columns: TableColumnDefinition[], + numberOfRows: number, +): Table { + const table = new Table(numberOfRows); + columns.forEach((col) => table.addColumn(col.columnName, col.column)); + return table; +} diff --git a/libs/interpreter-lib/src/parsing-util.spec.ts b/libs/interpreter-lib/src/parsing-util.spec.ts index 2975927b6..7f8ab9abf 100644 --- a/libs/interpreter-lib/src/parsing-util.spec.ts +++ b/libs/interpreter-lib/src/parsing-util.spec.ts @@ -8,11 +8,11 @@ import { CachedLogger } from '@jvalue/jayvee-execution'; import { JayveeServices, createJayveeServices, - initializeWorkspace, } from '@jvalue/jayvee-language-server'; import { ParseHelperOptions, expectNoParserAndLexerErrors, + loadTestExtensions, parseHelper, readJvTestAssetHelper, } from '@jvalue/jayvee-language-server/test'; @@ -52,14 +52,11 @@ describe('Validation of parsing-util', () => { // Create language services services = createJayveeServices(NodeFileSystem).Jayvee; - await initializeWorkspace(services, [ - { - uri: path.resolve( - __dirname, - '../test/assets/parsing-util/test-extension', - ), - name: 'TestBlockTypes.jv', - }, + await loadTestExtensions(services, [ + path.resolve( + __dirname, + '../test/assets/parsing-util/test-extension/TestBlockTypes.jv', + ), ]); // Parse function for Jayvee (without validation) diff --git a/libs/interpreter-lib/src/validation-checks/runtime-parameter-literal.spec.ts b/libs/interpreter-lib/src/validation-checks/runtime-parameter-literal.spec.ts index 9e61c42f6..f835f4c9e 100644 --- a/libs/interpreter-lib/src/validation-checks/runtime-parameter-literal.spec.ts +++ b/libs/interpreter-lib/src/validation-checks/runtime-parameter-literal.spec.ts @@ -11,11 +11,11 @@ import { RuntimeParameterProvider, ValidationContext, createJayveeServices, - initializeWorkspace, } from '@jvalue/jayvee-language-server'; import { ParseHelperOptions, expectNoParserAndLexerErrors, + loadTestExtensions, parseHelper, readJvTestAssetHelper, validationAcceptorMockImpl, @@ -71,14 +71,11 @@ describe('Validation of validateRuntimeParameterLiteral', () => { // Create language services const services = createJayveeServices(NodeFileSystem).Jayvee; - await initializeWorkspace(services, [ - { - uri: path.resolve( - __dirname, - '../../test/assets/runtime-parameter-literal/test-extension', - ), - name: 'TestBlockTypes.jv', - }, + await loadTestExtensions(services, [ + path.resolve( + __dirname, + '../../test/assets/runtime-parameter-literal/test-extension/TestBlockTypes.jv', + ), ]); locator = services.workspace.AstNodeLocator; diff --git a/libs/language-server/src/test/utils.ts b/libs/language-server/src/test/utils.ts index c5c9e72e9..159e01b56 100644 --- a/libs/language-server/src/test/utils.ts +++ b/libs/language-server/src/test/utils.ts @@ -2,11 +2,15 @@ // // SPDX-License-Identifier: AGPL-3.0-only +import * as assert from 'assert'; import { readFileSync } from 'fs'; import * as path from 'path'; import { AstNode, LangiumDocument, ValidationAcceptor } from 'langium'; +import { WorkspaceFolder } from 'vscode-languageserver-protocol'; +import { JayveeServices } from '../lib'; +import { initializeWorkspace } from '../lib/builtin-library/jayvee-workspace-manager'; import { constraintMetaInfRegistry } from '../lib/meta-information/meta-inf-registry'; // eslint-disable-next-line @typescript-eslint/no-empty-function @@ -46,3 +50,15 @@ export function expectNoParserAndLexerErrors( export function clearMetaInfRegistry() { constraintMetaInfRegistry.clear(); } + +export async function loadTestExtensions( + services: JayveeServices, + testExtensionJvFiles: string[], +) { + assert(testExtensionJvFiles.every((file) => file.endsWith('.jv'))); + const extensions: WorkspaceFolder[] = testExtensionJvFiles.map((file) => ({ + uri: path.dirname(file), + name: path.basename(file), + })); + return initializeWorkspace(services, extensions); +}