diff --git a/.eslintrc.json b/.eslintrc.json index ca3801f25..65f42ce94 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -43,10 +43,15 @@ }, { "files": ["*.spec.ts", "*.spec.tsx", "*.spec.js", "*.spec.jsx"], + "plugins": ["jest"], "env": { "jest": true }, - "rules": {} + "rules": { + // you should turn the original rule off *only* for test files + "@typescript-eslint/unbound-method": "off", + "jest/unbound-method": "error" + } } ] } diff --git a/.github/workflows/check-paths-for-windows.yaml b/.github/workflows/check-paths-for-windows.yaml new file mode 100644 index 000000000..74e2cc1f5 --- /dev/null +++ b/.github/workflows/check-paths-for-windows.yaml @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +# +# SPDX-License-Identifier: AGPL-3.0-only + +name: CI + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +jobs: + check-paths-for-windows: + name: Check paths for Windows + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - uses: actions/setup-node@v3 + with: + node-version: 'lts/*' + + - name: Check paths for Windows + run: node ./tools/scripts/check-for-invalid-windows-paths.mjs diff --git a/README.md b/README.md index 8b694fdb4..e8584e9ea 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,9 @@ SPDX-License-Identifier: AGPL-3.0-only # Jayvee -Jayvee is a DSL to model and execute automated data pipelines, e.g., for data engineering. +Jayvee is a domain-specific language (DSL) for automated processing of data pipelines. +The Jayvee interpreter allows executing such data pipelines on local machines. +Data engineers can use Jayvee and its interpreter to clean and preprocess data for later activities like data science or machine learning. [![Official Docs](assets/docs-banner.png)](https://jvalue.github.io/jayvee) @@ -26,23 +28,20 @@ In case you would like to contribute to Jayvee, please have a look at our [contr In case you run into problems, make sure to use the current LTS version of Node.js and npm. - ## Projects overview -| Name | Description | NPM package | -|-------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------| ------------------------------------------ | -| [`language-server`](./libs/language-server) | Jayvee language definition and language server implementation | [@jvalue/jayvee-language-server](https://www.npmjs.com/package/@jvalue/jayvee-language-server) | -| [`interpreter`](./apps/interpreter) | Command line tool for interpreting Jayvee files | [@jvalue/jayvee-interpreter](https://www.npmjs.com/package/@jvalue/jayvee-interpreter) | +| Name | Description | NPM package | +| ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | +| [`language-server`](./libs/language-server) | Jayvee language definition and language server implementation | [@jvalue/jayvee-language-server](https://www.npmjs.com/package/@jvalue/jayvee-language-server) | +| [`interpreter`](./apps/interpreter) | Command line tool for interpreting Jayvee files | [@jvalue/jayvee-interpreter](https://www.npmjs.com/package/@jvalue/jayvee-interpreter) | | [`language-server-web-worker`](./apps/language-server-web-worker) | Ready-to-use Jayvee language server, bundled as a [Web Worker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API) | [@jvalue/jayvee-language-server-web-worker](https://www.npmjs.com/package/@jvalue/jayvee-language-server-web-worker) | -| [`vs-code-extension`](./apps/vs-code-extension) | Visual Studio Code extension for editing Jayvee files | - | -| [`docs`](./apps/docs) | Website for Jayvee user documentation | - | -| [`monaco-editor`](./libs/monaco-editor) | React component for editing Jayvee files | [@jvalue/jayvee-monaco](https://www.npmjs.com/package/@jvalue/jayvee-monaco) | -| [`execution`](./libs/execution) | Shared code for Jayvee extensions and the interpreter | - | -| [`extensions/std`](./libs/extensions/std) | Standard Jayvee extension consisting of the extensions below | - | -| [`extensions/rdbms`](./libs/extensions/rdbms) | Jayvee extension for relational databases | - | -| [`extensions/tabular`](./libs/extensions/tabular) | Jayvee extension for tabular data | - | - - +| [`vs-code-extension`](./apps/vs-code-extension) | Visual Studio Code extension for editing Jayvee files | - | +| [`docs`](./apps/docs) | Website for Jayvee user documentation | - | +| [`monaco-editor`](./libs/monaco-editor) | React component for editing Jayvee files | [@jvalue/jayvee-monaco](https://www.npmjs.com/package/@jvalue/jayvee-monaco) | +| [`execution`](./libs/execution) | Shared code for Jayvee extensions and the interpreter | - | +| [`extensions/std`](./libs/extensions/std) | Standard Jayvee extension consisting of the extensions below | - | +| [`extensions/rdbms`](./libs/extensions/rdbms) | Jayvee extension for relational databases | - | +| [`extensions/tabular`](./libs/extensions/tabular) | Jayvee extension for tabular data | - | ## Scripts diff --git a/apps/docs/docs/dev/03-architecture-overview.md b/apps/docs/docs/dev/03-architecture-overview.md index cb4576fa2..b609a6135 100644 --- a/apps/docs/docs/dev/03-architecture-overview.md +++ b/apps/docs/docs/dev/03-architecture-overview.md @@ -12,6 +12,14 @@ On the pure language side, the central project is the [language server](https:// It contains the syntax definition (i.e. the grammar) and is capable of performing static semantic analysis on models, so invalid models can be rejected and errors are reported to the user. It uses the [Language Server Protocol](https://microsoft.github.io/language-server-protocol/) (LSP) for communicating with IDEs in order to provide common features such as diagnostics, auto completion and much more. +**Note:** The [Langium framework](https://langium.org/) generate TypeScript files for the abstract syntax tree (AST), based on the grammar specification. +The following locations might be especially helpful to understand the grammar and its AST: + +- The Langium grammar files (see [here](https://github.com/jvalue/jayvee/tree/main/libs/language-server/src/grammar) or locally at `libs/language-server/src/grammar`; with `.langium` file ending). These files define the **syntax of the language**. +- The generated TypeScript AST files (execute `npm run generate` to generate them at `libs/language-server/src/lib/ast/generated` in your local repository). These files include **TypeScript interfaces for AST nodes** (e.g., `BlockDefinition`) and **guard methods** (e.g., `isBlockDefinition`). + They reflect the input of the grammar files regarding naming. +- The remaining source files of the language server implement the language server protocol (LSP) and the additional validations beyond the syntax of Jayvee. + ## Interpreter The Jayvee [interpreter](https://github.com/jvalue/jayvee/tree/main/apps/interpreter) on the other hand is capable of running Jayvee models. diff --git a/apps/docs/docs/dev/04-guides/02-working-with-the-ast.md b/apps/docs/docs/dev/04-guides/02-working-with-the-ast.md index ca02dc335..895cae12c 100644 --- a/apps/docs/docs/dev/04-guides/02-working-with-the-ast.md +++ b/apps/docs/docs/dev/04-guides/02-working-with-the-ast.md @@ -141,7 +141,7 @@ assert(referenced !== undefined); ## AST wrapper classes The generated interfaces for AST nodes in `ast.ts` are only meant to represent the AST structurally, they don't define any behavior. -Also, in case of syntactic sugar, there may be different kinds of AST nodes representing the same semantic language concept (e.g. single pipes with a verbose syntax or chained pipes). +Also, in case of syntactic sugar, there may be different kinds of AST nodes representing the same semantic language concept. To cope with this problem, there is the concept of an `AstNodeWrapper`. An AST node wrapper is capable of wrapping AST nodes that represent the same semantic language concept and adding behavior to them via custom methods. diff --git a/apps/docs/docs/user/composite-blocks.md b/apps/docs/docs/user/composite-blocks.md index f5c9bdb57..3db9e78f1 100644 --- a/apps/docs/docs/user/composite-blocks.md +++ b/apps/docs/docs/user/composite-blocks.md @@ -102,7 +102,7 @@ If the CSVExtractor is available in the scope of the `CarsPipeline` from before ```jayvee pipeline CarsPipeline { // HttpExtractor, TextFileInterpreter and CSVInterpreter have been replaced by CSVExtractor - CSVExtractor + CarsExtractor -> CarsTableInterpreter -> CarsLoader; diff --git a/apps/docs/docs/user/core-concepts.md b/apps/docs/docs/user/core-concepts.md index eba148319..add0a01f5 100644 --- a/apps/docs/docs/user/core-concepts.md +++ b/apps/docs/docs/user/core-concepts.md @@ -24,28 +24,6 @@ pipeline CarsPipeline { } ``` -Alternatively, you can use a slightly longer syntax for pipes: - -```jayvee -pipeline CarsPipeline { - // Assumption: blocks "GasReserveHttpExtractor", "GasReserveCSVInterpreter", "GasReserveTableInterpreter", and "GasReserveLoader" are defined - - pipe { - from: GasReserveHttpExtractor; - to: GasReserveTextFileInterpreter; - - } - - pipe { - from: GasReserveTextFileInterpreter; - to: GasReserveCSVInterpreter; - - } - - // etc. -} -``` - ## Blocks A `Block` is a processing step within a `Pipeline`. diff --git a/apps/docs/docs/user/examples/.gitignore b/apps/docs/docs/user/examples/.gitignore index 277e9bb17..4a27c98c8 100644 --- a/apps/docs/docs/user/examples/.gitignore +++ b/apps/docs/docs/user/examples/.gitignore @@ -2,4 +2,5 @@ # # SPDX-License-Identifier: AGPL-3.0-only +!README.mdx *.md \ No newline at end of file diff --git a/apps/docs/docs/user/examples/README.mdx b/apps/docs/docs/user/examples/README.mdx new file mode 100644 index 000000000..3b8b94487 --- /dev/null +++ b/apps/docs/docs/user/examples/README.mdx @@ -0,0 +1,15 @@ +--- +sidebar_position: 10 +--- + +# Jayvee Examples + +Examples of Jayvee models. +Copy them to your local file system and execute them with the `jv` command on your command line (see [usage](/docs/user/intro#usage)). +You can [find all examples on Github](https://github.com/jvalue/jayvee/tree/main/example). + +```mdx-code-block +import DocCardList from '@theme/DocCardList'; + + +``` diff --git a/apps/docs/docs/user/examples/_category_.json.license b/apps/docs/docs/user/examples/README.mdx.license similarity index 100% rename from apps/docs/docs/user/examples/_category_.json.license rename to apps/docs/docs/user/examples/README.mdx.license diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/01-intro.md b/apps/docs/versioned_docs/version-0.3.0/dev/01-intro.md new file mode 100644 index 000000000..fef3c08e1 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/01-intro.md @@ -0,0 +1,52 @@ +--- +sidebar_position: 1 +--- + +# Introduction for Jayvee Developers + +## How to contribute + +In order to contribute to the Jayvee project, please have a look at our [contribution guide](https://github.com/jvalue/jayvee/blob/main/CONTRIBUTING.md). Before planning a contribution, please read the [design principles](./05-design-principles.md) and consider if your changes fit the vision expressed there. + +The overall project is licensed under the `AGPL-3.0-only` license and is compliant to the [REUSE Specification](https://reuse.software/spec/) by the [Free Software Foundation Europe](https://fsfe.org/). +Because of this, contributions are required to adhere to the license and follow that specification. +More details on this topic can be found [here](./02-dev-processes/03-licensing-and-copyright.md). + +And last but not least, please read and follow our [code of conduct](https://github.com/jvalue/jayvee/blob/main/CODE_OF_CONDUCT.md). + +## Project overview + +The Jayvee repository is located [here](https://github.com/jvalue/jayvee) on GitHub. +It uses an [Nx mono-repository](https://nx.dev/) setup and contains all related projects, most notably the Jayvee language server and interpreter. +Please have a look at the [architecture overview](./03-architecture-overview.md) for more details. + +## Prerequisites + +Node.js and npm are required for development. +It is recommended to use their LTS version to avoid any potential compatibility issues. +Also, refer to this [quick start guide](https://github.com/jvalue/jayvee#development-quickstart) for developers. + +In order to run the Jayvee VS Code extension during development, VS Code is required as IDE. +Using VS Code also allows installing the [Langium VS Code extension](https://marketplace.visualstudio.com/items?itemName=langium.langium-vscode) for better IDE support when working with Langium grammar files. + +## Resources for getting started + +### Langium + +- [**Langium documentation**](https://langium.org/docs/) +- Practical examples using Langium: + - [Langium examples](https://github.com/langium/langium/tree/main/examples): Official example languages by Langium + - [Langium's grammar language](https://github.com/langium/langium/tree/main/packages/langium): The implementation of Langium's own grammar language + - [Language server for SQL](https://github.com/langium/langium-sql): An implementation of a language server for SQL + - [MiniLogo DSL](https://github.com/langium/langium-minilogo): A domain-specific language for drawing pictures + - [Lox language](https://github.com/langium/langium-lox): An implementation of the Lox scripting language (also see the "Crafting Interpreters" book below) + - [SimpleUI DSL](https://github.com/TypeFox/langium-ui-framework): A domain-specific language for generating user interfaces + - [STPA-DSL](https://github.com/kieler/stpa): A domain-specific language for System-Theoretic Process Analysis +- [Langium playground](https://langium.org/playground/): A tool for testing Langium grammars and visualizing resulting ASTs +- [Typefox blog](https://www.typefox.io/blog/): A blog by the company behind Langium, mostly posting Langium-related content. + +### Building compilers / interpreters / DSLs + +- [Crafting Interpreters](https://craftinginterpreters.com/contents.html): A book by Robert Nystrom on implementing an interpreter for the Lox scripting language +- [DSL Engineering](https://voelter.de/dslbook/markusvoelter-dslengineering-1.0.pdf): A book by Markus Voelter on designing, implementing and using domain-specific languages +- [Awesome Compilers](https://github.com/aalhour/awesome-compilers#readme): A list featuring many resources on compilers and interpreters diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/01-intro.md.license b/apps/docs/versioned_docs/version-0.3.0/dev/01-intro.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/01-intro.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/01-rfc-process.md b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/01-rfc-process.md new file mode 100644 index 000000000..074440850 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/01-rfc-process.md @@ -0,0 +1,13 @@ +--- +title: Language Design Process (RFCs) +sidebar_position: 1 +--- + +We use RFCs to discuss changes to the language before implementing them. You can have a look at all closed (accepted / rejected) RFCs [here](https://github.com/jvalue/jayvee/pulls?q=is%3Apr+is%3Aclosed+RFC+), and all RFCs under discussion [here](https://github.com/jvalue/jayvee/pulls?q=is%3Apr+is%3Aopen+RFC). + +If you want to contribute an RFC please follow these steps: +1. Make a copy of the **template** at `rfc/0000-rfc-template.md` and place it into the `rfc` folder. +2. Create a draft for the RFC on a new branch. Follow the `TODOs` in template to do so. +3. Open a pull request with prefix `RFC ` in the title. +4. Address the reviews. Consider opening a new PR if major things need to be addressed and the discussion log becomes too confusing. +5. Once accepted, create an issue with the necessary steps to implement the RFC. \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/01-rfc-process.md.license b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/01-rfc-process.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/01-rfc-process.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/02-debug-vs-code-extension.md b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/02-debug-vs-code-extension.md new file mode 100644 index 000000000..5fbcf07cd --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/02-debug-vs-code-extension.md @@ -0,0 +1,25 @@ +--- +title: Debugging via the VS Code extension +sidebar_position: 2 +--- + +The VS Code extension of Jayvee can be used to interactively debug the language server. +During development, when using VS Code as IDE, another instance of VS Code can be opened which has the most recent, locally built VS Code extension loaded. +This can be achieved using the launch configuration `Run extension` the `Run and Debug` side-menu of VS Code or by pressing the `F5` key if that launch configuration is already selected there. + +Note that there is no file watching mechanism involved. +So in order to reflect changes to the source code, the additional VS Code instance has to be **closed and reopened** as described above. + +## How to attach a debugger + +1. Use the launch configuration `Run extension` to open the additional VS Code instance with the extension loaded +2. Use the launch configuration `Attach to Language Server` to attach the debugger + +Any set breakpoints should now be marked as active and pause the execution once they are reached. + +## How to view logs of the language server + +In the additional VS Code instance, it is possible to view the logs of the language server. +They might also be helpful for debugging since any uncaught errors are logged with their stack trace by the Langium framework. + +To view the logs, open the bottom panel called `Output` and select `Jayvee` in the dropdown menu. diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/02-debug-vs-code-extension.md.license b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/02-debug-vs-code-extension.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/02-debug-vs-code-extension.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/03-licensing-and-copyright.md b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/03-licensing-and-copyright.md new file mode 100644 index 000000000..201e21ca7 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/03-licensing-and-copyright.md @@ -0,0 +1,69 @@ +--- +title: Licensing and copyright +sidebar_position: 3 +--- + +The [Jayvee repository](https://github.com/jvalue/jayvee) is REUSE compliant, meaning that it fulfills the +[REUSE Specification](https://reuse.software/spec/) by the [Free Software Foundation Europe](https://fsfe.org/) (FSFE). +This makes clear under which license each project file is licensed under and who owns the copyright, not only for +humans but also for machines. + +This is done by explicitly adding copyright and licensing information to each file of the project. This is achieved +by either using a comment header or a separate `*.license` file in case comments are not possible. + +See more information. + +## What license is used in this project and who is the copyright holder? + +The entire project is licensed under [AGPL-3.0-only](https://spdx.org/licenses/AGPL-3.0-only.html), the +license file can be found [here](https://github.com/jvalue/jayvee/blob/main/LICENSES/AGPL-3.0-only.txt). +The copyright holder is the [Friedrich-Alexander-Universität Erlangen-Nürnberg](https://www.fau.eu/). + +## How to submit a REUSE compliant contribution? + +In case you want to contribute to the project, you will need to ensure that all of your contributed files are REUSE +compliant. In order to achieve this, you need to add a key-value pair for both copyright and licensing information +following this schema: + +``` +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only +``` + +In case a file allows comments, use single-line comments to add the copyright and licensing information at the top +of the file. Otherwise, create a corresponding `*.license` file with the text above as its content. You can have a +look at some existing project files to get an impression on it is done in practice. + +For files with common file extensions, you can use the [reuse CLI tool](https://github.com/fsfe/reuse-tool) to add +licensing and copyright information automatically. + +For more details, you can have a look at the [Getting Started tutorial](https://reuse.software/tutorial/) on the REUSE +website. + +## How to validate REUSE compliance? + +When you make a contribution and open a new pull request, the CI checks whether your contribution is REUSE compliant +using the [reuse CLI tool](https://github.com/fsfe/reuse-tool). + +In order to validate REUSE compliance in your local development environment, you have to install the +[reuse CLI tool](https://github.com/fsfe/reuse-tool) and run the following command in the projects' root folder: + +```bash +reuse lint +``` + +You can also set up a pre-commit hook, so the above command is run before each commit. +See [here](https://reuse.readthedocs.io/en/latest/readme.html#run-as-pre-commit-hook) for details on how to set it up. + +## How to hide `*.license` files in IDEs + +During development, the file explorer of your IDE may be cluttered due to the numerous `*.license` files in the +project. Luckily, most IDEs allow hiding certain files, e.g. by specifying a pattern to exclude them from the +explorer. + +Below, you can find instructions on how to hide `*.license` files in commonly used IDEs: +- **Visual Studio Code**: Add `**/*.license` to the +[`files.exclude` setting](https://code.visualstudio.com/docs/getstarted/userinterface#_explorer) +- **WebStorm**: Add `*.license` to the +[excluded files setting](https://www.jetbrains.com/help/webstorm/configuring-project-structure.html#exclude-by-pattern) diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/03-licensing-and-copyright.md.license b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/03-licensing-and-copyright.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/03-licensing-and-copyright.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/04-release-jayvee-version.md b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/04-release-jayvee-version.md new file mode 100644 index 000000000..374f3468b --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/04-release-jayvee-version.md @@ -0,0 +1,22 @@ +--- +title: Releasing a new Jayvee version +sidebar_position: 4 +--- + +## Version Numbers + +In this early stage of the project we do not yet follow [semantic versioning](https://semver.org/) since we expect the introduction of breaking changes frequently. +To indicate that, we only release alpha versions where the `version` is incremented with every release. +- For the npm packages, we use the version `0.0.`. +- For the GitHub releases, we use the git tag `v0.0.-alpha`. + +## Jayvee Release Procedure + +For releasing a new version of Jayvee, you need to complete the following steps: + +1. Increment the version in the `package.json` file. +2. Run `npm i` to update the `package-lock.json`. +3. Run `npx nx run docs:version-snapshot` to generate a snapshot of the docs for the previous version. +4. If you are on a feature or dev branch, merge into main. +5. Create a GitHub release on the main branch. Attach a changelog. +6. The CI/CD will deal with the rest. diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/04-release-jayvee-version.md.license b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/04-release-jayvee-version.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/04-release-jayvee-version.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/_category_.json b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/_category_.json new file mode 100644 index 000000000..c8f512450 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Development Process", + "position": 2, + "link": { + "type": "generated-index", + "description": "Here you can find general processes around developing Jayvee." + } +} diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/_category_.json.license b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/_category_.json.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/02-dev-processes/_category_.json.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/03-architecture-overview.md b/apps/docs/versioned_docs/version-0.3.0/dev/03-architecture-overview.md new file mode 100644 index 000000000..cb4576fa2 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/03-architecture-overview.md @@ -0,0 +1,43 @@ +--- +title: Architecture overview +sidebar_position: 4 +--- + +Jayvee has a clear separation between the language itself and its execution. +This guide gives an overview of the overall architecture. + +## Language Server + +On the pure language side, the central project is the [language server](https://github.com/jvalue/jayvee/tree/main/libs/language-server) which is heavily built upon the [Langium framework](https://langium.org/). +It contains the syntax definition (i.e. the grammar) and is capable of performing static semantic analysis on models, so invalid models can be rejected and errors are reported to the user. +It uses the [Language Server Protocol](https://microsoft.github.io/language-server-protocol/) (LSP) for communicating with IDEs in order to provide common features such as diagnostics, auto completion and much more. + +## Interpreter + +The Jayvee [interpreter](https://github.com/jvalue/jayvee/tree/main/apps/interpreter) on the other hand is capable of running Jayvee models. +Therefore, it uses the language server as a library to perform lexing, parsing and semantic analysis on given models. +In case no errors were found during these phases, it executes the model by interpreting it. +This means that models are not compiled to any other language, like a compiler does, but rather directly executed on the fly without any code generation involved. +The interpreter comes with a command-line interface (CLI) for users, so they are able to execute Jayvee models easily. + +The following diagram visualizes the architecture described so far: + +```mermaid +graph LR +model[Jayvee Model] --> lexical(Lexical Analysis) +subgraph Jayvee Interpreter + subgraph Jayvee Language Server + subgraph Langium Framework + lexical --> syntax(Syntax Analysis) + end + syntax --> semantic(Semantic Analysis) + end + semantic --> interpretation(Interpretation) +end +``` + +## Jayvee Extensions + +Lastly, there are Jayvee extensions for adding additional features to Jayvee. +See [here](./04-guides/06-jayvee-extensions.md) for more details. +It is worth pointing out that extensions also follow the separation between language and execution described above. diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/03-architecture-overview.md.license b/apps/docs/versioned_docs/version-0.3.0/dev/03-architecture-overview.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/03-architecture-overview.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/01-jayvee-grammar.md b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/01-jayvee-grammar.md new file mode 100644 index 000000000..356fb09de --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/01-jayvee-grammar.md @@ -0,0 +1,34 @@ +--- +title: The Jayvee grammar +sidebar_position: 1 +--- + +The grammar of Jayvee describes the syntax and structure of the language. +It is located in the language server project. +The grammar itself is written in [Langium's own grammar language](https://langium.org/docs/grammar-language/) which is similar to [Xtext](https://www.eclipse.org/Xtext/) grammars. +Such grammar files are easily identifiable via their `.langium` file extension. + +The grammar is used to generate TypeScript interfaces that represent the Abstract Syntax Tree (AST) and different, semantically equivalent files to define syntax highlighting for different applications. +For instance, a [TextMate](https://macromates.com/manual/en/language_grammars) file is generated for the syntax highlighting in the Jayvee VS Code extension whereas a [Monarch](https://microsoft.github.io/monaco-editor/monarch.html) file is generated for the syntax highlighting in the Monaco editor. + +For further information on the grammar language of Langium, visit the corresponding [Langium documentation](https://langium.org/docs/grammar-language/). + +## Working with the grammar + +To run the code generation, either use `npm run generate` for solely the code generation or `npm run build` for an entire build. The code generation also generates further code, like the standard library. + +Whenever the grammar is changed and the code generation is run during development, it is advisory to **close and reopen the IDE**, so the changes are noticed and the file indexing is updated. + +## How to rename AST nodes + +Renaming AST nodes is not as straight forward as one might initially assume. +When renaming individual rules in the grammar, just the generated TypeScript code in `ast.ts` will reflect the renaming, but not the rest of the codebase. + +The following steps explain how to rename an AST node, so the change is reflected in the entire codebase. As an example, an AST node called `A` is supposed to be renamed to `B`: + +- Open the `ast.ts` file in the language server project +- Locate the `A` interface / type and the `isA` typeguard +- Use the rename feature by the IDE to perform the renaming from `A` to `B` and from `isA` to `isB` +- Open the grammar and locate the `A` rule +- Use the rename feature by the Langium VS Code extension to perform the renaming from `A` to `B` +- Run `npm run generate` diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/01-jayvee-grammar.md.license b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/01-jayvee-grammar.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/01-jayvee-grammar.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/02-working-with-the-ast.md b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/02-working-with-the-ast.md new file mode 100644 index 000000000..895cae12c --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/02-working-with-the-ast.md @@ -0,0 +1,148 @@ +--- +title: Working with the AST +sidebar_position: 2 +--- + +The nodes of the Abstract Syntax Tree (AST) consist of types and interfaces generated from the language grammar. +See [here](./01-jayvee-grammar.md) for more information on that topic. + +The following sections provide practical guides and tips for working with nodes of the AST. + +## Dealing with potentially incomplete AST nodes + +According to the generated interfaces, properties of AST nodes are never `undefined`. +In practice however, this is not always the case. + +For example, consider the language server being confronted with an incomplete Jayvee model with syntax errors. +In such cases, properties of AST nodes are in fact `undefined`, despite their interface definition. +In the interpreter however, after lexical and syntactic analysis succeeded, it can be assumed that all AST nodes are complete (i.e. they have no undefined properties) and even that all references are resolved. + +In order to avoid accessing properties of AST nodes that are potentially undefined, it is recommended to access them via the [`?.` operator](https://www.typescriptlang.org/docs/handbook/release-notes/typescript-3-7.html#optional-chaining) rather than the regular `.` operator. +Note that this might result in a conflict with the ESLint rule [`@typescript-eslint/no-unnecessary-condition`](https://typescript-eslint.io/rules/no-unnecessary-condition/), so it has to be disabled manually for such cases. + +For disabling the rule in an entire file, place this comment below the copyright and licensing header: + +```typescript +/* eslint-disable @typescript-eslint/no-unnecessary-condition */ +``` + +For just a single line, place this comment above that particular line: + +```typescript +// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition +``` + +
+ +Full example + +Consider an exemplary AST node `A` with a property `x` of type `string`. To access that property safely: + +```typescript +import { A } from './ast' + +const astNode: A; + +// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition +const property: string | undefined = astNode?.x; +``` + +
+ +## Usage of `assertUnreachable` + +Most times, it is beneficial to make case distinctions exhaustive, especially when working with AST nodes, properties with a [union literal type](https://www.typescriptlang.org/docs/handbook/2/everyday-types.html#literal-types) or enums. +Exhaustiveness in this context means, that the TypeScript compiler is supposed to yield an error if a case distinction does not cover all possibilities. + +Langium offers a function called `assertUnreachable` which is capable of enforcing exhaustiveness and producing compiler errors in case of violations. See the following examples to get an idea on how to use it in practice: + +
+ +Example for an exhaustive switch statement on a union literal type + +```typescript +import { assertUnreachable } from 'langium'; + +const operator: '+' | '-'; + +switch(operator) { + case '+': { + // ... + break; + } + case '-': { + // ... + break; + } + default: { + // To ensure the switch being exhaustive on `operator`: + assertUnreachable(operator); + } +} +``` + +
+ +
+ +Example for an exhaustive if-elseif-else cascade using typeguards + +Consider the exemplary AST nodes `A`, `B` and `C` and that `A = B | C`: + +```typescript +import { assertUnreachable } from 'langium'; +import { A, B, isB, C, isC } from './ast' + +const astNode: A; + +if (isB(astNode)) { + // `astNode` has type `B` here +} else if (isC(astNode)) { + // `astNode` has type `C` here +} else { + // To ensure the if-elseif-else cascade being exhaustive on `astNode`: + assertUnreachable(astNode); +} +``` + +
+ +## Usage of `assert` for expressing runtime expectations + +During development, it may occur that a certain condition is expected to **be always true** at runtime. +For example, an AST node being of a certain type or a property being defined. +The type system of TypeScript is not always able to infer such facts, so developers may have to express these expectations explicitly in their code. +Examples are [type assertions](https://www.typescriptlang.org/docs/handbook/advanced-types.html) or the [non-null assertion operator](https://www.typescriptlang.org/docs/handbook/release-notes/typescript-2-0.html#non-null-assertion-operator). +Their usage may be problematic in case the condition does not always hold, e.g. due to a bug in the program or a wrong expectation by the programmer. +In such cases, it is hard to locate the origin and debug the program because such operations are erased in the compiled JavaScript code. + +To avoid these issues, it is recommended to express such expectations as boolean expressions that are actually validated at runtime. +This can be easily achieved by using the `assert` function. +It evaluates a given boolean expression and throws an `AssertionError` in case it evaluates to `false`. +After calling `assert`, the type system of TypeScript assumes the condition to be `true` and afterwards narrows types accordingly. + +Here is an example of how to use it in practice: + +```typescript +// Import the `assert` function like this: +import { strict as assert } from 'assert'; + +import { A, B, isB } from './ast'; + +const astNode: A; +assert(isB(astNode)); +// Here `astNode` has type `B` + +const referenced = astNode?.reference?.ref; +assert(referenced !== undefined); +// Here `referred` is not `undefined` +``` + +## AST wrapper classes + +The generated interfaces for AST nodes in `ast.ts` are only meant to represent the AST structurally, they don't define any behavior. +Also, in case of syntactic sugar, there may be different kinds of AST nodes representing the same semantic language concept. + +To cope with this problem, there is the concept of an `AstNodeWrapper`. +An AST node wrapper is capable of wrapping AST nodes that represent the same semantic language concept and adding behavior to them via custom methods. +To get an impression on how this can be done in practice, please have a look at the [`PipeWrapper` class](https://github.com/jvalue/jayvee/blob/main/libs/language-server/src/lib/ast/wrappers/pipe-wrapper.ts) and the [`AstNodeWrapper` interface](https://github.com/jvalue/jayvee/blob/main/libs/language-server/src/lib/ast/wrappers/ast-node-wrapper.ts). diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/02-working-with-the-ast.md.license b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/02-working-with-the-ast.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/02-working-with-the-ast.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/03-validation-and-diagnostics.md b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/03-validation-and-diagnostics.md new file mode 100644 index 000000000..328d57285 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/03-validation-and-diagnostics.md @@ -0,0 +1,64 @@ +--- +title: Validation and diagnostics +sidebar_position: 3 +--- + +Validation in the context of Jayvee can be understood as semantic analysis of Jayvee models. +The purpose is to uncover errors in models besides lexing, parsing and linking errors that the Langium framework already detects out of the box. +In case such errors are found, the language server makes the IDE display a diagnostic. +This means that the location of the error is underlined and an error message is shown. + +For example, each pipeline is expected to contain at least one starting block that requires no input. +Therefore, the language server analyzes every pipeline in a Jayvee model and checks for the presence of such a starting block. +In case no such block is found, the IDE underlines the pipeline definition in a red color and displays an error message. + +## How to implement validation + +### For arbitrary AST nodes + +In the file [`validation-registry.ts`](https://github.com/jvalue/jayvee/blob/main/libs/language-server/src/lib/validation/validation-registry.ts) of the language server, there is a registry containing validation functions for various AST nodes. +A validation function provided for a certain kind of AST node is called for every occurrence of such a node in a Jayvee model. +E.g. when registering a validation function for `PipelineDefinition` nodes, that function gets called once for each pipeline. + +A validation function always operates on a single, concrete AST node. +Besides the AST node, a `ValidationContext` object is passed to the function for reporting diagnostics via its `accept` method. + +The overall validation for a specific kind of AST node is usually subdivided into smaller checks that are called sequentially. +This potentially allows aborting the validation early, i.e. before all checks were run. +Aborting early may be useful if errors were reported during previous checks. +This can be determined via the `ValidationContext` which keeps track whether any errors occurred so far. + +A practical example where this plays a role are blocks: +When the type of a block is unknown to the language server, it makes no sense to validate inputs / outputs and properties of that block. +So, in case an error was reported during the check of the block type, the validation of that block is discontinued at that point. + +### For properties of blocks and constraints + +Validating property values of blocks / constraints works a bit differently because their individual validation is already incorporated into the overall validation framework. + +In general, the validation logic for a property values is located in the meta information of the block type / constraint type. +There, properties are defined using the `PropertySpecification` interface. +In order to add validation logic to a certain property, a validation function needs to be added to that property. + +For more complex validations that need to take multiple property values into account, it is also possible to provide a more general validation function which operates on the entire `PropertyBody` AST node. + +See [`text-range-selector-meta-inf.ts`](https://github.com/jvalue/jayvee/blob/main/libs/extensions/std/lang/src/text-range-selector-meta-inf.ts) for an example which includes both cases described above. + +## Reporting diagnostics + +During validation, diagnostics can be reported by calling the `accept` method of the given `ValidationContext` object. + +The call requires the severity of the diagnostic (`error`, `warning`, `info` or `hint`), the error message and its location. +The location is described by a `DiagnosticInfo` object which contains the affected AST node and optionally some further restrictions. +For more details, have a look at the [`DiagnosticInfo` interface](https://github.com/langium/langium/blob/main/packages/langium/src/validation/validation-registry.ts) by Langium. + +## Common issues + +When working on validations, a diagnostic with the following message may unexpectedly occur: `An error occurred during validation: ...` + +Such a diagnostic is generated by the Langium framework if an error object is thrown within the validation. +In most cases, the reason is the access of an `undefined` AST node property or an `AssertionError`. +For more details on such errors and how to avoid them, have a look at the documentation on [how to work with the AST](./02-working-with-the-ast.md). + +One way locate the origin of such errors is to debug the Jayvee VS Code extension, see [here](../02-dev-processes/02-debug-vs-code-extension.md) for more details. +Another possibility is to debug the interpreter and set appropriate breakpoints in the language server codebase. diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/03-validation-and-diagnostics.md.license b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/03-validation-and-diagnostics.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/03-validation-and-diagnostics.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/04-expressions-and-operators.md b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/04-expressions-and-operators.md new file mode 100644 index 000000000..91b51bd1a --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/04-expressions-and-operators.md @@ -0,0 +1,151 @@ +--- +title: Expressions and operators +sidebar_position: 4 +--- + +Jayvee supports arbitrarily nested expressions and offers a variety of different operators. +Such expressions are similar to those used in programming languages, and they are intended to be read and understood intuitively. + +The following sections explain the definition of expressions in the language grammar, their type inference mechanism, and how the evaluation of expressions works. +As a small practical example, you may also have a look at the [arithmetics example by Langium](https://github.com/langium/langium/tree/main/examples/arithmetics) which has a heavy focus on mathematical expressions and functions. + +## Expressions in the grammar + +Expressions in the Jayvee grammar are defined in [`expression.langium`](https://github.com/jvalue/jayvee/blob/main/libs/language-server/src/grammar/expression.langium) and consist of operators (unary / binary) and literals. +Unary operators only have a single operand (e.g. the `not` operator) whereas binary operators require two operands (e.g. the `*` operator). + +The grammar is written in a way that literals end up in the leaves of the resulting AST and the nodes above represent the operators. + +As an example, have a look at the following AST structure of the expression `(-3 + 5) * 7`: + +```mermaid +classDiagram + +class `:BinaryOperator` { + operator = '*' +} + +%% Uses spaces in the name so it is unique +class ` :BinaryOperator` { + operator = '+' +} + +class `:UnaryOperator` { + operator = '-' +} + +%% Uses spaces in the name so it is unique +class ` :NumericLiteral` { + value = 3 +} + +%% Uses spaces in the name so it is unique +class ` :NumericLiteral` { + value = 5 +} + +class `:NumericLiteral` { + value = 7 +} + +`:BinaryOperator` --> ` :BinaryOperator` : leftOperand +`:BinaryOperator` --> `:NumericLiteral` : rightOperand +` :BinaryOperator` --> `:UnaryOperator` : leftOperand +`:UnaryOperator` --> ` :NumericLiteral` : operand +` :BinaryOperator` --> ` :NumericLiteral` : rightOperand +``` + +The AST is constructed in a way that ensures an unambiguous evaluation order when using depth-first search. +This implies that the AST structure already reflects the precedence and associativity of the operators, and that parentheses do not need to be explicitly represented. + +For more details on how such a grammar for expressions can be realized, see the [official Langium documentation](https://langium.org/docs/grammar-language/#tree-rewriting-actions), [this blog post](https://www.typefox.io/blog/parsing-expressions-with-xtext) and the following two sections which discuss the precedence and associativity of operators. + +### Precedence of operators + +The grammar implicitly defines the precedence of operators. +The operator precedence defines the order in which operators are evaluated within an expression. +For example, multiplication has a higher precedence than addition, which leads to multiplications being performed before additions. +In order to manually override such precedence conventions, parentheses can be used in expressions. + +The diagram below shows a more extensive precedence hierarchy, focused on common arithmetic operators. +Note that the hierarchy is arranged in ascending order, according to the operator precedence. +Such an order is similar to how operators in the actual Jayvee grammar are arranged: + +```mermaid +graph TD + subgraph BinaryOperators + add/sub(Addition Operator / Subtraction operator) --> mul/div(Multiplication Operator / Division Operator) + mul/div --> exp/root(Exponential Operator / Root Operator) + end + subgraph UnaryOperators + exp/root --> plus/minus(Plus Operator / Minus Operator) + end + plus/minus --> literal(Numeric Literal) +``` + +In the Jayvee grammar, every level of precedence is represented by a single grammar rule. +Within each such rule, the grammar rule which defines the operators with the next higher precedence is referenced. +E.g. the `AdditiveExpression` rule refers to the `MultiplicativeExpression` rule because multiplicative operators are supposed to have the next higher precedence. + +In order to alter the precedence of operators, their hierarchy of grammar rules has to be adjusted accordingly in [`expression.langium`](https://github.com/jvalue/jayvee/blob/main/libs/language-server/src/grammar/expression.langium). + +### Associativity of operators + +The associativity of operators defines the order in which operators with the same precedence are evaluated when they appear in succession without parentheses. +Operators may be either **left-associative**, **right-associative** or **non-associative**. +For example, depending on the associativity of the binary `+` operator, the expression `a + b + c` has different semantics: + +- Left-associative: `(a + b) + c` (evaluation from left to right) +- Right-associative: `a + (b + c)` (evaluation from right to left) +- Non-associative: _syntax error_, the operator cannot be chained + +The associativity of operators is also defined in the Jayvee grammar, more specifically by the structure of the grammar rules that define operators. +For details on how to encode the different kinds of associativity in grammar rules, have a look at the "Associativity" section near the end of [this blog post](https://www.typefox.io/blog/parsing-expressions-with-xtext). +The patterns described in the linked article can be found in [`expression.langium`](https://github.com/jvalue/jayvee/blob/main/libs/language-server/src/grammar/expression.langium), so each operator grammar rule encodes its own associativity. + +## Typing of expressions + +Jayvee has a mechanism for inferring and validating the type of a given expression. +This is achieved using a recursive algorithm which performs depth-first search on a given expression: + +The base case for the algorithm are literals, i.e. the tree leaves of an expression. +Depending on the type of literal, their type can be inferred trivially (in case of value literals) or otherwise from the context where the expression is located. + +For operators, the types of their operands are first inferred via recursion, and then it is checked whether they are supported by the operator. +Next, given the operand types, the resulting type is computed. +Such behavior is defined in a _type computer class_ located [here](https://github.com/jvalue/jayvee/tree/main/libs/language-server/src/lib/ast/expressions/type-computers). +Additionally, in [`operator-registry.ts`](https://github.com/jvalue/jayvee/blob/main/libs/language-server/src/lib/ast/expressions/operator-registry.ts), a type computer is registered for each kind of operator. + +In case the algorithm fails to infer a type, e.g. due to unsupported operand types or unresolved references in literals, the resulting type is `undefined`. +In order to report diagnostics in such cases, a `ValidationContext` object can be supplied when calling the type inference. + +## Evaluation of expressions + +The evaluation has the purpose of computing a single value out of an expression. +For a successful evaluation, it is a **precondition** that **a type could successfully be inferred** from the respective expression. +Also, the **value for each free variable** in that expression (i.e. literals resembling placeholders for values) needs to be **known beforehand**. +Therefore, an `ExecutionContext` object is passed to the evaluation which holds values for free variables in the current context. + +### Algorithm + +The algorithm for evaluating expressions is also based on a recursive depth-first search, similar to how the type inference works: + +Again, literals are the base case. A value literal trivially evaluates to its own value. +Other literals that resemble free variables evaluate to their value provided by the given `ExecutionContext` object. + +Regarding operators, their operands first are evaluated recursively. +Next, using the concrete operand values, the operator computes its resulting value. +This is defined in operator evaluator classes located [here](https://github.com/jvalue/jayvee/tree/main/libs/language-server/src/lib/ast/expressions/evaluators). +The classes are registered in [operator-registry.ts](https://github.com/jvalue/jayvee/blob/main/libs/language-server/src/lib/ast/expressions/operator-registry.ts) for every operator, similar to the previously mentioned type computers. + +The result of an evaluation may be `undefined` in case any errors occurred. +If the preconditions were all met, such errors are most likely arithmetic errors (like division by zero). +It is possible to provide a `ValidationContext` object to the evaluation for reporting such errors as diagnostics. + +### Evaluation strategies + +There are different evaluation strategies to choose from. +They affect the way, expressions are evaluated and thus have an impact on their semantics: + +- `exhaustive`: A full depth-first search is performed, all parts of an expression are evaluated. +- `lazy`: An evaluation with the least effort is performed. Logical operators use [short circuit semantics](https://en.wikipedia.org/wiki/Short-circuit_evaluation) and binary operators don't evaluate their right operand if the left one evaluated to `undefined`. diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/04-expressions-and-operators.md.license b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/04-expressions-and-operators.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/04-expressions-and-operators.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/05-standard-library.md b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/05-standard-library.md new file mode 100644 index 000000000..6db17bd74 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/05-standard-library.md @@ -0,0 +1,48 @@ +--- +title: Working with the Standard Library +sidebar_position: 5 +--- + +Jayvee ships with its own standard library on board, including the most often used valuetypes, transformations, and so on. +The standard library itself is written in `.jv` files [here](https://github.com/jvalue/jayvee/tree/main/libs/language-server/src/stdlib/). + +## Standard Library Contents + +The following elements are part of the standard library: + +## Builtin Contents + +The implementations of builtin contents are not expressed in Jayvee itself but on the TypeScript layer. Examples: + +- **Builtin valuetypes**: These valuetypes are the base for defining user-defined valuetypes in Jayvee, e.g., `text`, `integer`, `decimal`, `boolean`. +- **Builtin iotypes**: These iotypes are used to describe in inputs and outputs of blocktypes, e.g., `Sheet`, `File`. +- **Builtin blocktypes**: These blocktypes are the very basic building blocks in Jayvee, e.g., `HttpExtractor`, `SqliteLoader`. +- **Builtin constraint types**: These constraint types are constraints with custom logic, e.g., `LengthConstraint`, `RegexConstraint`. + +Builtin definitions are usually generated and added to the standard library from the internal representations of the concepts. + +### User-defined Contents + +The implementations of user-defined contents are expressed in Jayvee itself. Examples: + +- **User-defined valuetypes**: These valuetypes are based on builtin or other user-defined valuetypes. Their definition is expressed natively in Jayvee, e.g., `Percent`. +- **User-defined blocktypes**: These blocktypes are based on builtin or other user-defined blocktypes. Their definition is expressed natively in Jayvee. + +We use `jv` files to add user-defined valuetypes to the standard library (see below). + +## Extending the Standard Library + +Just add `jv` files to the directory [here](https://github.com/jvalue/jayvee/tree/main/libs/language-server/src/stdlib/). It is crawled hierarchically, meaning that you can also organize files in folders. + +## Implementation + +### 1. Code generation + +We use code generation to transform these `.jv` files into TypeScript files that the language server can used. The [generation script](https://github.com/jvalue/jayvee/tree/main/tools/scripts/language-server/generate-stdlib.mjs) is run via `npm run generate` next to the AST generation. + +### 2. Builtin libraries + +The solution we chose to implement the standard library mechanism is close to the [builtin library tutorial](https://langium.org/guides/builtin-library/) by Langium. The following components are of interest: + +- [JayveeWorkspaceManager](https://github.com/jvalue/jayvee/tree/main/libs/language-server/src/lib/builtin-library/jayvee-workspace-manager.ts) in the `language-server` that registers all libraries with the langium framework. +- [StandardLibraryFileSystemProvider](https://github.com/jvalue/jayvee/tree/main/apps/vs-code-extension/src/standard-library-file-system-provider.ts) in the `vs-code-extension` that registers all libraries with the vscode plugin framework. diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/05-standard-library.md.license b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/05-standard-library.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/05-standard-library.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/06-jayvee-extensions.md b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/06-jayvee-extensions.md new file mode 100644 index 000000000..107569d26 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/06-jayvee-extensions.md @@ -0,0 +1,190 @@ +--- +title: Jayvee Extensions +sidebar_position: 6 +--- + +## Concepts + +### Jayvee extension + +A Jayvee extension is used to add new block types to the language without having to modify the actual grammar of the language. Such a Jayvee extension usually consists of two parts: a language extension and an execution extension which are described in the upcoming two sections. + +Jayvee extensions that shall be used by default are bundled into the so-called [standard extension](https://github.com/jvalue/jayvee/tree/main/libs/extensions/std). That way, the [language server](https://github.com/jvalue/jayvee/tree/main/libs/language-server) and the [interpreter](https://github.com/jvalue/jayvee/tree/main/apps/interpreter) are able to load them out of the box. + +### Language extension + +A language extension defines meta information of block types which are required by the +[language server](https://github.com/jvalue/jayvee/tree/main/libs/language-server). +Such meta information describes properties of +block types such as their names, input / output types and their properties. + +Note that language extensions don't define any behavior. Instead, this is done by the corresponding execution extension. + +### Execution extension + +An execution extension defines behavior for block types. They build on the meta information from the corresponding +language extension, e.g. input / output types of the block need to match the signature of the execution method and +properties are accessed by their specified name. + +Execution extensions are only required by the [interpreter](https://github.com/jvalue/jayvee/tree/main/apps/interpreter) and not necessarily by the [language server](https://github.com/jvalue/jayvee/tree/main/libs/language-server) as they solely define behavior. + +## Recipes + +### Add a new Jayvee execution extension + +#### 1. Generate an execution libraries + +```bash +npx nx g @nrwl/node:library --name="extensions//exec" +``` + +#### 2. Create extension classes + +In `libs/extensions//exec/src/extension.ts`: + +```typescript +import { + BlockExecutorClass, + JayveeExecExtension, +} from '@jvalue/jayvee-execution'; + +export class MyExecExtension implements JayveeExecExtension { + getBlockExecutors(): BlockExecutorClass[] { + return []; + } +} +``` + +#### 3. Export extension classes + +In `libs/extensions//exec/src/index.ts`: + +```typescript +export * from './extension'; +``` + +#### 4. Register new extension classes in the standard extension + +In `libs/extensions/std/exec/src/extension.ts`: + +```typescript +// ... + +import { MyExecExtension } from '@jvalue/jayvee-extensions//exec'; + +export class StdExecExtension implements JayveeExecExtension { + private readonly wrappedExtensions: JayveeExecExtension[] = [ + // ... + // Register your execution extension here: + new MyExecExtension(), + // ... + ]; + + // ... +} +``` + +### Add a new block type in a Jayvee extension + +#### 1. Create a builtin blocktype + +Define the syntax of the new blocktype in the [language server's builtin blocktypes](https://github.com/jvalue/jayvee/tree/main/libs/language-server/src/stdlib/builtin-blocktypes). + +The following example defines a block type `MyExtractor` with a text property called `url` and a property `retries` with a default value: + +```jayvee +builtin blocktype MyExtractor { + input default oftype None; + output default oftype Sheet; + + property url oftype text; + property retries oftype interger: 10; +} +``` + +The new block type will be automatically registered on the language server startup. + +#### 2. Add custom validation logic (if required) + +If the block type and/or its properties requires custom validation logic, you can implement it in the [language server's block type specific checks](https://github.com/jvalue/jayvee/tree/main/libs/language-server/src/lib/validation/checks/blocktype-specific). + +#### 3. Implement `BlockExecutor` + +The following example implements an executor for the previously defined block type `MyExtractor`. + +The `execute` method defines the behavior when a block is executed. Its signature matches the input and output types defined in `MyExtractor.jv` file. + +In `libs/extensions//exec/src/lib/my-extractor-executor.ts`: + +```typescript +import * as R from '@jvalue/jayvee-execution'; +import { + AbstractBlockExecutor, + BlockExecutorClass, + ExecutionContext, + Sheet, + implementsStatic, +} from '@jvalue/jayvee-execution'; +import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server'; + +@implementsStatic() +export class MyExtractorExecutor + extends AbstractBlockExecutor +{ + // Needs to match the type in meta information: + public static readonly type = 'MyExtractor'; + + public readonly inputType = IOType.NONE; + public readonly outputType = IOType.SHEET; + + async doExecute( + input: None, + context: ExecutionContext, + ): Promise> { + // Accessing property values by their name: + const url = context.getPropertyValue( + 'url', + PrimitiveValuetypes.Text, + ); + const limit = context.getPropertyValue( + 'limit', + PrimitiveValuetypes.Integer, + ); + + // ... + + if (error) { + return R.err(...); + } + + return R.ok(...); + } +} +``` + +> **Info** +> The interface `BlockExecutor` is used as an API for block executors. The abstract class `AbstractBlockExecutor` gives some further functionality for free, e.g., debug logging. + +> **Warning** +> The generic types of `AbstractBlockExecutor` need to match the input and output types of the corresponding `blocktype` definition. + +#### 4. Register the new `BlockExecutor` in the execution extension + +In `libs/extensions//exec/src/extension.ts`: + +```typescript +// ... + +import { MyExtractorExecutor } from './lib/my-extractor-executor'; + +export class MyExecExtension implements JayveeExecExtension { + getBlockExecutors(): BlockExecutorClass[] { + return [ + // ... + // Register your block executor here: + MyExtractorExecutor, + // ... + ]; + } +} +``` diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/06-jayvee-extensions.md.license b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/06-jayvee-extensions.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/06-jayvee-extensions.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/_category_.json b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/_category_.json new file mode 100644 index 000000000..80f6fce8f --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Guides", + "position": 4, + "link": { + "type": "generated-index", + "description": "Here you can find guides that will help you developing certain aspects of Jayvee." + } +} diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/_category_.json.license b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/_category_.json.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/04-guides/_category_.json.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/05-design-principles.md b/apps/docs/versioned_docs/version-0.3.0/dev/05-design-principles.md new file mode 100644 index 000000000..8b15dd8c3 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/05-design-principles.md @@ -0,0 +1,24 @@ +--- +title: Design Principles +sidebar_position: 5 +--- + +When deciding on new features for the domain-specific language itself, we try to adhere to the following high level guidelines. Of course, none of these statements is set in stone and every decision is a tradeoff. + +## Jayvee Manifesto +_Inspired by the [Agile Manifesto](https://agilemanifesto.org/)._ + +We are uncovering better ways of _modeling data pipelines by providing a domain-specific language for data engineering and making it easy for everyone to participate in it_. + +Through this work we have come to value: + +1. **Describing a goal state** over how to get there. +2. **Explicit modeling** over hidden magic. +3. **Composition** over inheritance. +4. **Flat structures** over deep nesting. + +That is, while there is value in the items on the right, we value the items on the left more. + +Through this work, we also have come to explore: + +5. **Libraries** over language features. \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/05-design-principles.md.license b/apps/docs/versioned_docs/version-0.3.0/dev/05-design-principles.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/05-design-principles.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/12-jayvee-testing.md b/apps/docs/versioned_docs/version-0.3.0/dev/12-jayvee-testing.md new file mode 100644 index 000000000..1121c1ef9 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/12-jayvee-testing.md @@ -0,0 +1,254 @@ +--- +title: Writing tests for Jayvee +--- + +In order to ensure that Jayvee works as intended and to catch breaking changes, we have implemented the following components for regression testing: +- Testing utils: utils to create Langium Typescript objects from *.jv test assets (see [here](#testing-utils)) as well as mocks for execution testing (see [here](#testing-utils-1)) +- [Grammar tests](#grammar-tests): test the grammar parsing and validation +- [Execution tests](#execution-tests): test the execution of blocks + +## Conventions +All of the existing tests follow these conventions: +1. The `.spec.ts` file is located next to the `.ts` file itself. +2. The `*.jv` assets are located inside a `test/assets/` folder. +Take a look at one of the exisiting tests for more details. + +## Grammar tests +These kind of tests are mainly located inside the [language-server](https://github.com/jvalue/jayvee/tree/main/libs/language-server) as well as the language parts of each extension (for example [std/lang](https://github.com/jvalue/jayvee/tree/main/libs/extensions/std/lang)). + +### Testing utils +The testing utils are located inside the `language-server` in a dedicated [test folder](https://github.com/jvalue/jayvee/tree/main/libs/language-server/src/test). +These utils can be imported using `@jvalue/jayvee-language-server/test` and contain the following parts: + +[**langium-utils.ts**](https://github.com/jvalue/jayvee/blob/main/libs/language-server/src/test/langium-utils.ts): +This utils file contains two functions: +- `parseHelper` to simplify parsing the input (content of a *.jv file) and returning the corresponding `LangiumDocument`, and +- `validationHelper` parse and validate the created document. +They are kept in a separate file due to being copied from the Langium repository and thus subject to a different code license and copyright. + +[**utils.ts**](https://github.com/jvalue/jayvee/blob/main/libs/language-server/src/test/utils.ts): +This file contains custom testing utility utils functions, like `readJvTestAssetHelper` for reading jv test assets. +Example: +``` ts +import * as path from 'path'; + +import { createJayveeServices } from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, LangiumDocument } from 'langium'; + +describe('My example test', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/', // localized path to test assets folder + ); + + beforeAll(() => { + // [...] register extensions etc + const services = createJayveeServices(NodeFileSystem).Jayvee; // Or retrieve them if services already exist + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + // [...] + + it('My dummy test', () => { + const text = readJvTestAsset('/.jv'); + + const document = await parse(text); + expectNoParserAndLexerErrors(document); + // Rest of test + }); +}); +``` +If you want to simply validate the test assets, simply replace `parseHelper` with `validationHelper` (and adjust the types). +You can find detailed documentation of all the utility functions directly in the code. + +[**extension/**](https://github.com/jvalue/jayvee/tree/main/libs/language-server/src/test/extension): +This folder contains a Jayvee extension for testing. +If there are certain blocks required for testing a certain feature, they can be defined here. +One such example is the already defined `TestProperty` block which has a multitude of different properties, each with a different type. +This block is used for testing properties and property-assignments. +The extension provides loader and extractor blocks for all IOTypes without any properties. +These blocks are automatically generated at runtime with the following naming scheme: +`Test${ioType}${io === 'input' ? 'Loader' : 'Extractor'}` (Example: `TestFileExtractor`). +This allows for easy (grammar) testing of non loader/extractor blocks: +``` jv +pipeline Pipeline { + + TestExtractor -> BlockUnderTest -> TestLoader; + + block BlockUnderTest oftype CellWriter { + at: range A1:A3; + write: ['values', 'to', 'write']; + } + + block TestExtractor oftype TestSheetExtractor { } + block TestLoader oftype TestSheetLoader { } +} +``` + +### Existing tests +Currently there are already tests for the following parts: +- Language-server validation checks (located [here](https://github.com/jvalue/jayvee/tree/main/libs/language-server/src/lib/validation)) +- Language-server constraint validation (located [here](https://github.com/jvalue/jayvee/tree/dev/libs/language-server/src/lib/constraint)) +- Custom block (property) validation of the three existing extensions (std extension located [here](https://github.com/jvalue/jayvee/blob/dev/libs/extensions/std/lang/src)) +- Grammar validation tests for all official full examples from the [/example](https://github.com/jvalue/jayvee/tree/main/example) folder (located [here](https://github.com/jvalue/jayvee/blob/dev/libs/extensions/std/lang/src/example-validation.spec.ts)) +- Grammar validation tests for all block examples of the std extension (located [here](https://github.com/jvalue/jayvee/blob/dev/libs/extensions/std/lang/src/meta-inf-example-validation.spec.ts)) + +## Execution tests +These kind of tests are mainly located inside the [interpreter](https://github.com/jvalue/jayvee/tree/main/libs/language-server), the [interpreter-lib](https://github.com/jvalue/jayvee/tree/dev/libs/interpreter-lib), the [execution lib](https://github.com/jvalue/jayvee/tree/dev/libs/execution) as well as the execution parts of each extension (for example [std/exec](https://github.com/jvalue/jayvee/tree/main/libs/extensions/std/exec)). + +### Testing utils +The testing utils for execution tests are spread between the extensions, with the interfaces and base utils located inside the [execution lib](https://github.com/jvalue/jayvee/tree/dev/libs/execution). +They can be imported using `@jvalue/jayvee-extensions/rdbms/test`, `@jvalue/jayvee-extensions/std/test` and `@jvalue/jayvee-execution/test`. + +[**utils.ts**](https://github.com/jvalue/jayvee/blob/dev/libs/execution/test/utils.ts): +At the moment this only contains two functions: +- `clearBlockExecutorRegistry` for clearing the registry containing all `BlockExecutor`s, and +- `clearConstraintExecutorRegistry` clearing the corresponding `ConstraintExecutor`s registry. +They are required in case the tested method initializes Jayvee itself (see [smoke test](#existing-tests-1)). + +[**test-logger.ts**](https://github.com/jvalue/jayvee/blob/dev/libs/execution/test/test-logger.ts): +This contains a subclass of the [`DefaultLogger`](https://github.com/jvalue/jayvee/blob/dev/libs/execution/src/lib/logging/default-logger.ts) used for tests which require a `Logger` implementation. The `TestLogger` contains the following tests functionality: +- `getLogs`: retrieve the cached logs that the logger received. +- `clearLogs`: clear the cached logs. + +[**block-executor-mocks.ts**](https://github.com/jvalue/jayvee/blob/dev/libs/execution/test/block-executor-mock.ts): +`BlockExecutorMock` interface for defining mocks for `AbstractBlockExecutor`. Generally only loader and executor blocks require mocks, because they interact with "the outside world" (i.e. `HttpExtractor` making http calls). +Due to how vastly different each `BlockExecutor` can be, this interface is very simple, containing only a `setup(...args: unknown[])` and a `restore()` method. See below for existing implementations. + +[**rdbms/exec/test**](https://github.com/jvalue/jayvee/tree/dev/libs/extensions/rdbms/exec/test): +Contains the implementation of `BlockExecutorMock` for `PostgresLoaderExecutor` and `SQLiteLoaderExecutor`. +Both of these executors are mocked using `jest.mock` to mock the corresponding libraries (`pg` and `sqlite3`) +**Usage:** +``` ts +import { + PostgresLoaderExecutorMock, + SQLiteLoaderExecutorMock, +} from '@jvalue/jayvee-extensions/rdbms/test'; + +// Global mocking of external library at the top of test file required, +// even though the mocking is encapsulated in helper classes +jest.mock('pg', () => { + const mClient = { + connect: jest.fn(), + query: jest.fn(), + end: jest.fn(), + }; + return { Client: jest.fn(() => mClient) }; +}); +jest.mock('sqlite3', () => { + const mockDB = { + close: jest.fn(), + run: jest.fn(), + }; + return { Database: jest.fn(() => mockDB) }; +}); + +describe('Dummy describe', () => { + // [...] + + let postgresLoaderMock: PostgresLoaderExecutorMock; + let sqliteLoaderMock: SQLiteLoaderExecutorMock; + + beforeAll(() => { + postgresLoaderMock = new PostgresLoaderExecutorMock(); + sqliteLoaderMock = new SQLiteLoaderExecutorMock(); + }); + + afterEach(() => { + postgresLoaderMock.restore(); + sqliteLoaderMock.restore(); + }); + + it('Dummy test', async () => { + // Prepare mocks + postgresLoaderMock.setup(); + sqliteLoaderMock.setup(); + + // [...] execute test + + expect(postgresLoaderMock.pgClient.connect).toBeCalledTimes(1); + expect(postgresLoaderMock.pgClient.query).toBeCalledTimes(1); + expect(postgresLoaderMock.pgClient.end).toBeCalledTimes(1); + expect(sqliteLoaderMock.sqliteClient.run).toBeCalledTimes(1); + expect(sqliteLoaderMock.sqliteClient.close).toBeCalledTimes(1); + }); +}); +``` + +[**std/exec/test/mocks**](https://github.com/jvalue/jayvee/tree/dev/libs/extensions/std/exec/test): +Contains the implementation of `BlockExecutorMock` for `HttpExtractorExecutorMock`. +This implementation uses [nock](https://www.npmjs.com/package/nock) for mocking HTTP(S) responses. +The `setup` method is further specified requiring one parameter `registerMocks: () => Array`, which returns all used `nock.Scope` (i.e. the return value of `nock('')`), see usage below: +**Usage:** +``` ts +import * as path from 'path'; + +import { HttpExtractorExecutorMock } from '@jvalue/jayvee-extensions/std/test'; + +describe('Dummy describe', () => { + // [...] + + let httpExtractorMock: HttpExtractorExecutorMock; + + beforeAll(() => { + httpExtractorMock = new HttpExtractorExecutorMock(); + }); + + afterEach(() => { + httpExtractorMock.restore(); + }); + + it('should have no errors when executing gtfs-static-and-rt.jv example', async () => { + // Prepare mocks + httpExtractorMock.setup(() => { + return [ + nock( + '', + ) + .get('') + .replyWithFile( + 200, + path.resolve(__dirname, '../test/assets/file1.zip'), + { + 'Content-Type': 'application/octet-stream', + }, + ), + nock('') + .get('') + .replyWithFile( + 200, + path.resolve( + __dirname, + '../test/assets/file2', + ), + { + 'Content-Type': 'application/octet-stream', + }, + ) + .get('') + .reply(200, { content: "My dummy json reply." }), + ]; + }) + + // [...] execute test + + expect(httpExtractorMock.nockScopes.every((scope) => scope.isDone())); + }); +}); +``` + +### Existing tests +Currently there are already tests for the following parts: +- Smoke test for official examples (located [here](https://github.com/jvalue/jayvee/blob/dev/apps/interpreter/src/examples-smoke-test.spec.ts)) diff --git a/apps/docs/versioned_docs/version-0.3.0/dev/12-jayvee-testing.md.license b/apps/docs/versioned_docs/version-0.3.0/dev/12-jayvee-testing.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/dev/12-jayvee-testing.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/ArchiveInterpreter.md b/apps/docs/versioned_docs/version-0.3.0/user/block-types/ArchiveInterpreter.md new file mode 100644 index 000000000..2a622e276 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/ArchiveInterpreter.md @@ -0,0 +1,33 @@ +--- +title: ArchiveInterpreter +--- + + + +Input type: `File` + +Output type: `FileSystem` + +## Description + +Interprets a `File` as an archive file and converts it to a `FileSystem`. The archive file root is considered the root of the `FileSystem`. + +## Example 1 + +```jayvee + block ZipArchiveInterpreter oftype ArchiveInterpreter { + archiveType: "zip"; + } +``` + +Interprets a `File` as a ZIP-archive and creates a `FileSystem` of its extracted contents. + +## Properties + +### `archiveType` + +Type `text` + +#### Description + +The archive type to be interpreted, e.g., "zip" or "gz". diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/ArchiveInterpreter.md.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/ArchiveInterpreter.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/ArchiveInterpreter.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/CSVInterpreter.md b/apps/docs/versioned_docs/version-0.3.0/user/block-types/CSVInterpreter.md new file mode 100644 index 000000000..9242e0459 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/CSVInterpreter.md @@ -0,0 +1,55 @@ +--- +title: CSVInterpreter +--- + + + +Input type: `TextFile` + +Output type: `Sheet` + +## Description + +Interprets an input file as a csv-file containing string-values delimited by `delimiter` and outputs a `Sheet`. + +## Example 1 + +```jayvee + block AgencyCSVInterpreter oftype CSVInterpreter { + delimiter: ";"; + } +``` + +Interprets an input file as a csv-file containing string-values delimited by `;` and outputs `Sheet`. + +## Properties + +### `delimiter` + +Type `text` + +Default: `","` + +#### Description + +The delimiter for values in the CSV file. + +### `enclosing` + +Type `text` + +Default: `""` + +#### Description + +The enclosing character that may be used for values in the CSV file. + +### `enclosingEscape` + +Type `text` + +Default: `""` + +#### Description + +The character to escape enclosing characters in values. diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/CSVInterpreter.md.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/CSVInterpreter.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/CSVInterpreter.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/CellRangeSelector.md b/apps/docs/versioned_docs/version-0.3.0/user/block-types/CellRangeSelector.md new file mode 100644 index 000000000..2e55f64cd --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/CellRangeSelector.md @@ -0,0 +1,33 @@ +--- +title: CellRangeSelector +--- + + + +Input type: `Sheet` + +Output type: `Sheet` + +## Description + +Selects a subset of a `Sheet` to produce a new `Sheet`. + +## Example 1 + +```jayvee + block CarsCoreDataSelector oftype CellRangeSelector { + select: range A1:E*; + } +``` + +Selects the cells in the given range and produces a new `Sheet` containing only the selected cells. + +## Properties + +### `select` + +Type `CellRange` + +#### Description + +The cell range to select. diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/CellRangeSelector.md.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/CellRangeSelector.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/CellRangeSelector.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/CellWriter.md b/apps/docs/versioned_docs/version-0.3.0/user/block-types/CellWriter.md new file mode 100644 index 000000000..e397dccf6 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/CellWriter.md @@ -0,0 +1,53 @@ +--- +title: CellWriter +--- + + + +Input type: `Sheet` + +Output type: `Sheet` + +## Description + +Writes textual values into cells of a `Sheet`. The number of text values needs to match the number of cells to write into. + +## Example 1 + +```jayvee + block NameHeaderWriter oftype CellWriter { + at: cell A1; + write: ["Name"]; + } +``` + +Write the value "Name" into cell `A1`. + +## Example 2 + +```jayvee + block HeaderSequenceWriter oftype CellWriter { + at: range A1:A2; + write: ["Name", "Age"]; + } +``` + +Write the values "Name", "Age" into cells `A1` and `A2`. + +## Properties + +### `write` + +Type `Collection` + +#### Description + +The values to write. + +### `at` + +Type `CellRange` + +#### Description + +The cells to write into. diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/CellWriter.md.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/CellWriter.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/CellWriter.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/ColumnDeleter.md b/apps/docs/versioned_docs/version-0.3.0/user/block-types/ColumnDeleter.md new file mode 100644 index 000000000..affce2760 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/ColumnDeleter.md @@ -0,0 +1,33 @@ +--- +title: ColumnDeleter +--- + + + +Input type: `Sheet` + +Output type: `Sheet` + +## Description + +Deletes columns from a `Sheet`. Column IDs of subsequent columns will be shifted accordingly, so there will be no gaps. + +## Example 1 + +```jayvee + block MpgColumnDeleter oftype ColumnDeleter { + delete: [column B]; + } +``` + +Deletes column B (i.e. the second column). + +## Properties + +### `delete` + +Type `Collection` + +#### Description + +The columns to delete. Has to be a full column. diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/ColumnDeleter.md.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/ColumnDeleter.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/ColumnDeleter.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/FilePicker.md b/apps/docs/versioned_docs/version-0.3.0/user/block-types/FilePicker.md new file mode 100644 index 000000000..9fd0d5ad4 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/FilePicker.md @@ -0,0 +1,33 @@ +--- +title: FilePicker +--- + + + +Input type: `FileSystem` + +Output type: `File` + +## Description + +Selects one `File` from a `FileSystem` based on its relative path to the root of the `FileSystem`. If no file matches the relative path, no output is created and the execution of the pipeline is aborted. + +## Example 1 + +```jayvee + block AgencyFilePicker oftype FilePicker { + path: "./agency.txt"; + } +``` + +Tries to pick the file `agency.txt` from the root of the provided `FileSystem`. If `agency.txt` exists it is passed on as `File`, if it does not exist the execution of the pipeline is aborted. + +## Properties + +### `path` + +Type `text` + +#### Description + +The path of the file to select, relative to the root of the provided `FileSystem`. diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/FilePicker.md.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/FilePicker.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/FilePicker.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/GtfsRTInterpreter.md b/apps/docs/versioned_docs/version-0.3.0/user/block-types/GtfsRTInterpreter.md new file mode 100644 index 000000000..3da451b25 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/GtfsRTInterpreter.md @@ -0,0 +1,76 @@ +--- +title: GtfsRTInterpreter +--- + + + +Input type: `File` + +Output type: `Sheet` + +## Description + +Interprets an protobuf file (binary) of type `File` by decoding the file according to `gtfs-realtime.proto`. Outputs the extracted entity defined by `entity` as a `Sheet` + +## Example 1 + +```jayvee + block GtfsRTTripUpdateInterpreter oftype GtfsRTInterpreter{ + entity: "trip_update"; + } +``` + +A file is interpretet as an GTFS-RT file, which contains TripUpdate. + +## Properties + +### `entity` + +Type `text` + +#### Description + +Entity to process from GTFS-RT-feed (`trip_update`, `alert` or `vehicle`). + We currently support following Output-Sheets, each are an equivalent to the flattened Element Index defined in (just required fields are included): + Entity TripUpdate: + ``` + [ + 'header.gtfs_realtime_version', + 'header.timestamp', + 'header.incrementality', + 'entity.id', + 'entity.trip_update.trip.trip_id', + 'entity.trip_update.trip.route_id', + 'entity.trip_update.stop_time_update.stop_sequence', + 'entity.trip_update.stop_time_update.stop_id', + 'entity.trip_update.stop_time_update.arrival.time', + 'entity.trip_update.stop_time_update.departure.time', + ]; + ``` + Entity VehiclePosition: + ``` + [ + 'header.gtfs_realtime_version', + 'header.timestamp', + 'header.incrementality', + 'entity.id', + 'entity.vehicle_position.vehicle_descriptor.id', + 'entity.vehicle_position.trip.trip_id', + 'entity.vehicle_position.trip.route_id', + 'entity.vehicle_position.position.latitude', + 'entity.vehicle_position.position.longitude', + 'entity.vehicle_position.timestamp', + ]; + ``` + Entity Alert: + ``` + [ + 'header.gtfs_realtime_version', + 'header.timestamp', + 'header.incrementality', + 'entity.id', + 'entity.alert.informed_entity.route_id', + 'entity.alert.header_text', + 'entity.alert.description_text', + ]; + ``` diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/GtfsRTInterpreter.md.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/GtfsRTInterpreter.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/GtfsRTInterpreter.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/HttpExtractor.md b/apps/docs/versioned_docs/version-0.3.0/user/block-types/HttpExtractor.md new file mode 100644 index 000000000..523e4eb39 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/HttpExtractor.md @@ -0,0 +1,73 @@ +--- +title: HttpExtractor +--- + + + +Input type: `None` + +Output type: `File` + +## Description + +Extracts a `File` from the web. + +## Example 1 + +```jayvee + block CarsFileExtractor oftype HttpExtractor { + url: "tinyurl.com/4ub9spwz"; + } +``` + +Fetches a file from the given URL. + +## Properties + +### `url` + +Type `text` + +#### Description + +The URL to the file in the web to extract. + +### `retries` + +Type `integer` + +Default: `0` + +#### Description + +Configures how many retries should be executed after a failure fetching the data. + +### `retryBackoffMilliseconds` + +Type `integer` + +Default: `1000` + +#### Description + +Configures the wait time in milliseconds before executing a retry. + +### `retryBackoffStrategy` + +Type `text` + +Default: `"exponential"` + +#### Description + +Configures the wait strategy before executing a retry. Can have values "exponential" or "linear". + +### `followRedirects` + +Type `boolean` + +Default: `true` + +#### Description + +Indicates, whether to follow redirects on get requests. If `false`, redirects are not followed. Default `true` diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/HttpExtractor.md.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/HttpExtractor.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/HttpExtractor.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/PostgresLoader.md b/apps/docs/versioned_docs/version-0.3.0/user/block-types/PostgresLoader.md new file mode 100644 index 000000000..91782ab5b --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/PostgresLoader.md @@ -0,0 +1,78 @@ +--- +title: PostgresLoader +--- + + + +Input type: `Table` + +Output type: `None` + +## Description + +Loads a `Table` into a PostgreSQL database sink. + +## Example 1 + +```jayvee + block CarsLoader oftype PostgresLoader { + host: "localhost"; + port: 5432; + username: "postgres"; + password: "postgres"; + database: "CarsDB"; + table: "Cars"; + } +``` + +A local Postgres instance is filled with table data about cars. + +## Properties + +### `host` + +Type `text` + +#### Description + +The hostname or IP address of the Postgres database. + +### `port` + +Type `integer` + +#### Description + +The port of the Postgres database. + +### `username` + +Type `text` + +#### Description + +The username to login to the Postgres database. + +### `password` + +Type `text` + +#### Description + +The password to login to the Postgres database. + +### `database` + +Type `text` + +#### Description + +The database to use. + +### `table` + +Type `text` + +#### Description + +The name of the table to write into. diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/PostgresLoader.md.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/PostgresLoader.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/PostgresLoader.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/RowDeleter.md b/apps/docs/versioned_docs/version-0.3.0/user/block-types/RowDeleter.md new file mode 100644 index 000000000..20093b186 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/RowDeleter.md @@ -0,0 +1,33 @@ +--- +title: RowDeleter +--- + + + +Input type: `Sheet` + +Output type: `Sheet` + +## Description + +Deletes one or more rows from a `Sheet`. Row IDs of subsequent rows will be shifted accordingly, so there will be no gaps. + +## Example 1 + +```jayvee + block SecondRowDeleter oftype RowDeleter { + delete: [row 2]; + } +``` + +Deletes row 2 (i.e. the second row). + +## Properties + +### `delete` + +Type `Collection` + +#### Description + +The rows to delete. diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/RowDeleter.md.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/RowDeleter.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/RowDeleter.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/SQLiteLoader.md b/apps/docs/versioned_docs/version-0.3.0/user/block-types/SQLiteLoader.md new file mode 100644 index 000000000..d48e6f0ff --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/SQLiteLoader.md @@ -0,0 +1,52 @@ +--- +title: SQLiteLoader +--- + + + +Input type: `Table` + +Output type: `None` + +## Description + +Loads a `Table` into a SQLite database sink. + +## Example 1 + +```jayvee + block CarsLoader oftype SQLiteLoader { + table: "cars"; + file: "./cars.db"; + } +``` + +A SQLite file `cars.db` is created in the working directory. Incoming data is written to the table `cars`. + +## Properties + +### `table` + +Type `text` + +#### Description + +The name of the table to write into. + +### `file` + +Type `text` + +#### Description + +The path to a SQLite file that will be created if it does not exist. Usual file extensions are `.sqlite` and `.db`. + +### `dropTable` + +Type `boolean` + +Default: `true` + +#### Description + +Indicates, whether to drop the table before loading data into it. If `false`, data is appended to the table instead of dropping it. diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/SQLiteLoader.md.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/SQLiteLoader.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/SQLiteLoader.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/SheetPicker.md b/apps/docs/versioned_docs/version-0.3.0/user/block-types/SheetPicker.md new file mode 100644 index 000000000..26710170b --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/SheetPicker.md @@ -0,0 +1,33 @@ +--- +title: SheetPicker +--- + + + +Input type: `Workbook` + +Output type: `Sheet` + +## Description + +Selects one `Sheet` from a `Workbook` based on its `sheetName`. If no sheet matches the name, no output is created and the execution of the pipeline is aborted. + +## Example 1 + +```jayvee + block AgencySheetPicker oftype SheetPicker { + sheetName: "AgencyNames"; + } +``` + +Tries to pick the sheet `AgencyNames` from the provided `Workbook`. If `AgencyNames` exists it is passed on as `Sheet`, if it does not exist the execution of the pipeline is aborted. + +## Properties + +### `sheetName` + +Type `text` + +#### Description + +The name of the sheet to select. diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/SheetPicker.md.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/SheetPicker.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/SheetPicker.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/TableInterpreter.md b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TableInterpreter.md new file mode 100644 index 000000000..7d81aae3b --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TableInterpreter.md @@ -0,0 +1,63 @@ +--- +title: TableInterpreter +--- + + + +Input type: `Sheet` + +Output type: `Table` + +## Description + +Interprets a `Sheet` as a `Table`. In case a header row is present in the sheet, its names can be matched with the provided column names. Otherwise, the provided column names are assigned in order. + +## Example 1 + +```jayvee + block CarsTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "name" oftype text, + "mpg" oftype decimal, + "cyl" oftype integer, + ]; + } +``` + +Interprets a `Sheet` about cars with a topmost header row and interprets it as a `Table` by assigning a primitive valuetype to each column. The column names are matched to the header, so the order of the type assignments does not matter. + +## Example 2 + +```jayvee + block CarsTableInterpreter oftype TableInterpreter { + header: false; + columns: [ + "name" oftype text, + "mpg" oftype decimal, + "cyl" oftype integer, + ]; + } +``` + +Interprets a `Sheet` about cars without a topmost header row and interprets it as a `Table` by sequentially assigning a name and a primitive valuetype to each column of the sheet. Note that the order of columns matters here. The first column (column `A`) will be named "name", the second column (column `B`) will be named "mpg" etc. + +## Properties + +### `header` + +Type `boolean` + +Default: `true` + +#### Description + +Whether the first row should be interpreted as header row. + +### `columns` + +Type `Collection` + +#### Description + +Collection of valuetype assignments. Uses column names (potentially matched with the header or by sequence depending on the `header` property) to assign a primitive valuetype to each column. diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/TableInterpreter.md.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TableInterpreter.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TableInterpreter.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/TableTransformer.md b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TableTransformer.md new file mode 100644 index 000000000..eb52dc905 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TableTransformer.md @@ -0,0 +1,73 @@ +--- +title: TableTransformer +--- + + + +Input type: `Table` + +Output type: `Table` + +## Description + +Applies a transform on each value of a column. The input port type of the used transform has to match the type of the input column. + +## Example 1 + +```jayvee + transform CelsiusToFahrenheit { + from Celsius oftype decimal; + to Fahrenheit oftype decimal; + Fahrenheit: (Celsius * 9/5) + 32; + } + block CelsiusToFahrenheitTransformer oftype TableTransformer { + inputColumns: ['temperature']; + outputColumn: 'temperature'; + use: CelsiusToFahrenheit; + } +``` + +Given a column "temperature" with temperature values in Celsius, it overwrites the column with computed values in Fahrenheit by using the `CelsiusToFahrenheit` transform. The transform itself is defined elsewhere in the model. + +## Example 2 + +```jayvee + transform CelsiusToFahrenheit { + from Celsius oftype decimal; + to Fahrenheit oftype decimal; + Fahrenheit: (Celsius * 9/5) + 32; + } + block CelsiusToFahrenheitTransformer oftype TableTransformer { + inputColumns: ['temperatureCelsius']; + outputColumn: 'temperatureFahrenheit'; + use: CelsiusToFahrenheit; + } +``` + +Given a column "temperatureCelsius" with temperature values in Celsius, it adds a new column "temperatureFahrenheit" with computed values in Fahrenheit by using the `CelsiusToFahrenheit` transform. The transform itself is defined elsewhere in the model. + +## Properties + +### `inputColumns` + +Type `Collection` + +#### Description + +The names of the input columns. The columns have to be present in the table and match with the transform's input port types. + +### `outputColumn` + +Type `text` + +#### Description + +The name of the output column. Overwrites the column if it already exists, or otherwise creates a new one. + +### `use` + +Type `Transform` + +#### Description + +Reference to the transform that is applied to the column. diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/TableTransformer.md.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TableTransformer.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TableTransformer.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/TextFileInterpreter.md b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TextFileInterpreter.md new file mode 100644 index 000000000..0fefdcc24 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TextFileInterpreter.md @@ -0,0 +1,35 @@ +--- +title: TextFileInterpreter +--- + + + +Input type: `File` + +Output type: `TextFile` + +## Description + +Interprets a `File` as a `TextFile`. + +## Properties + +### `encoding` + +Type `text` + +Default: `"utf-8"` + +#### Description + +The encoding used for decoding the file contents. + +### `lineBreak` + +Type `Regex` + +Default: `{}` + +#### Description + +The regex for identifying line breaks. diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/TextFileInterpreter.md.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TextFileInterpreter.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TextFileInterpreter.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/TextLineDeleter.md b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TextLineDeleter.md new file mode 100644 index 000000000..b36a46c48 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TextLineDeleter.md @@ -0,0 +1,23 @@ +--- +title: TextLineDeleter +--- + + + +Input type: `TextFile` + +Output type: `TextFile` + +## Description + +Deletes individual lines from a `TextFile`. + +## Properties + +### `lines` + +Type `Collection` + +#### Description + +The line numbers to delete. diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/TextLineDeleter.md.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TextLineDeleter.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TextLineDeleter.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/TextRangeSelector.md b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TextRangeSelector.md new file mode 100644 index 000000000..75f5e68e4 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TextRangeSelector.md @@ -0,0 +1,36 @@ +--- +title: TextRangeSelector +--- + + + +Input type: `TextFile` + +Output type: `TextFile` + +## Description + +Selects a range of lines from a `TextFile`. + +## Properties + +### `lineFrom` + +Type `integer` + +Default: `1` + +#### Description + +Inclusive beginning line number for the selection. + +### `lineTo` + +Type `integer` + +Default: `9007199254740991` + +#### Description + +Inclusive ending line number for the selection. + The default value is the biggest usable integer. diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/TextRangeSelector.md.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TextRangeSelector.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/TextRangeSelector.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/XLSXInterpreter.md b/apps/docs/versioned_docs/version-0.3.0/user/block-types/XLSXInterpreter.md new file mode 100644 index 000000000..3dd9a3c91 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/XLSXInterpreter.md @@ -0,0 +1,23 @@ +--- +title: XLSXInterpreter +--- + + + +Input type: `File` + +Output type: `Workbook` + +## Description + +Interprets an input file as a XLSX-file and outputs a `Workbook` containing `Sheet`s. + +## Example 1 + +```jayvee + block AgencyXLSXInterpreter oftype XLSXInterpreter { } +``` + +Interprets an input file as a XLSX-file and outputs a `Workbook` containing `Sheet`s. + +## Properties diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/XLSXInterpreter.md.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/XLSXInterpreter.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/XLSXInterpreter.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/_category_.json b/apps/docs/versioned_docs/version-0.3.0/user/block-types/_category_.json new file mode 100644 index 000000000..740dd2f28 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Block Types", + "position": 3, + "link": { + "type": "generated-index", + "description": "These blocks are shipped with Jayvee and are available right out of the box." + } +} diff --git a/apps/docs/versioned_docs/version-0.3.0/user/block-types/_category_.json.license b/apps/docs/versioned_docs/version-0.3.0/user/block-types/_category_.json.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/block-types/_category_.json.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/user/composite-blocks.md b/apps/docs/versioned_docs/version-0.3.0/user/composite-blocks.md new file mode 100644 index 000000000..3db9e78f1 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/composite-blocks.md @@ -0,0 +1,115 @@ +--- +sidebar_position: 4 +--- + +# Composite Blocks + +Composite blocks are a way to create new blocktypes in Jayvee by combining the functionality of existing blocks and pipes. By relying on composite blocks instead of implementing more builtin blocks in a language interpreter, Jayvee supports easy extension by users. + +Composite blocks define: +- with the `property` keyword: properties with a name and [value type](./core-concepts.md#valuetypes), optionally a default value +- with the `input` keyword: one input with a name and io type (that can be None) +- with the `output` keyword: one output with a name and io type (that can be None) +- one pipeline definition, starting from the input (using its name) and ending in the output (again using its name) +- all blocks that are used in the pipeline definition (either builtin or other composite blocks) + +## Example +As an example, the common use-case of extracting a CSV file from a webserver using HTTP. With builtin blocks, a pipeline would start with a HttpExtractor source that downloads a file from the internet and outputs a binary file. This file must be interpreted as text (using a TextFileInterpreter) and finally as Sheet (using a CSVInterpreter). + +### Implementation with builtin blocks +```mermaid +flowchart LR + A[HttpExtractor] --> B(TextFileInterpreter) + B --> C(CSVInterpreter) + C --> D(TableInterpreter) + D --> E[SQLiteSink] +``` + +A pipeline with builtin blocks is very verbose: + +```jayvee +pipeline CarsPipeline { + CarsExtractor + -> CarsTextFileInterpreter + -> CarsCSVInterpreter + -> CarsTableInterpreter + -> CarsLoader; + + block CarsExtractor oftype HttpExtractor { + url: "https://example.com/cars.csv"; + } + + block CarsTextFileInterpreter oftype TextFileInterpreter { } + + block CarsCSVInterpreter oftype CSVInterpreter { + enclosing: '"'; + } + // ... further block definitions +} +``` + +### Refactoring using composite blocks + +The common use-case of downloading a CSV file using HTTP can be refactored into a composite block. Note that we define all properties of the builtin blocks that are used as properties of the new CSVExtractor blocktype (but add fallback values). If some internal configuration is always the same, we could also not expose it as a property of the new blocktype. + +```jayvee +// Define a new blocktype named CSVExtractor outside of the pipeline +composite blocktype CSVExtractor { + // Properties of the CSVExtractor, some with default values + property url oftype text; + property delimiter oftype text: ','; + property enclosing oftype text: ''; + property enclosingEscape oftype text: ''; + + // Input and outputs + input inputName oftype None; + output outputName oftype Sheet; + + // Pipeline definition from input, over blocks defined later, to output + inputName + ->FileExtractor + ->FileTextInterpreter + ->FileCSVInterpreter + ->outputName; + + // Block definitions using values from properties by name + block FileExtractor oftype HttpExtractor { url: url; } + block FileTextInterpreter oftype TextFileInterpreter {} + + block FileCSVInterpreter oftype CSVInterpreter { + delimiter: delimiter; + enclosing: enclosing; + enclosingEscape: enclosingEscape; + } +} +``` + +With the new CSVExtractor composite blocktype, the pipeline now looks like this. + +```mermaid +flowchart LR + CSVExtractor --> D(TableInterpreter) + D --> E[SQLiteSink] + + subgraph CSVExtractor + A[HttpExtractor] --> B(TextFileInterpreter) + B --> C(CSVInterpreter) +end +``` + +If the CSVExtractor is available in the scope of the `CarsPipeline` from before (e.g., by defining it above the pipeline), it can then be used to shorten the actual pipeline code. + +```jayvee +pipeline CarsPipeline { + // HttpExtractor, TextFileInterpreter and CSVInterpreter have been replaced by CSVExtractor + CarsExtractor + -> CarsTableInterpreter + -> CarsLoader; + + block CarsExtractor oftype CSVExtractor { + url: "https://example.com/cars.csv"; + } + + // ... further block definitions +} +``` \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/composite-blocks.md.license b/apps/docs/versioned_docs/version-0.3.0/user/composite-blocks.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/composite-blocks.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/AllowlistConstraint.md b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/AllowlistConstraint.md new file mode 100644 index 000000000..f4710e006 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/AllowlistConstraint.md @@ -0,0 +1,27 @@ +--- +title: AllowlistConstraint +--- + + + +Compatible ValueType: text + +## Description + +Limits the values to a defined a set of allowed values. Only values in the list are valid. + +## Example 1 + +```jayvee + constraint TimeUnitString oftype AllowlistConstraint { + allowlist: ["ms", "s", "min", "h", "d", "m", "y"]; + } +``` + +Only allows the common abbreviations for millisecond, second, minute, etc.. + +## Properties + +### `allowlist` + +Type `Collection` diff --git a/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/AllowlistConstraint.md.license b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/AllowlistConstraint.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/AllowlistConstraint.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/DenylistConstraint.md b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/DenylistConstraint.md new file mode 100644 index 000000000..91a3da328 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/DenylistConstraint.md @@ -0,0 +1,27 @@ +--- +title: DenylistConstraint +--- + + + +Compatible ValueType: text + +## Description + +Defines a set of forbidden values. All values in the list are considered invalid. + +## Example 1 + +```jayvee + constraint NoPrimaryColors oftype DenylistConstraint { + denylist: ["red", "blue", "yellow"]; + } +``` + +Denies all primary colors. + +## Properties + +### `denylist` + +Type `Collection` diff --git a/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/DenylistConstraint.md.license b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/DenylistConstraint.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/DenylistConstraint.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/LengthConstraint.md b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/LengthConstraint.md new file mode 100644 index 000000000..8c4d37291 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/LengthConstraint.md @@ -0,0 +1,37 @@ +--- +title: LengthConstraint +--- + + + +Compatible ValueType: text + +## Description + +Limits the length of a string with an upper and/or lower boundary. + Only values with a length within the given range are valid. + +## Example 1 + +```jayvee + constraint ShortAnswerConstraint oftype LengthConstraint { + minLength: 0; + maxLength: 20; + } +``` + +A text constraint with 0 to 20 characters. + +## Properties + +### `minLength` + +Type `integer` + +Default: `0` + +### `maxLength` + +Type `integer` + +Default: `9007199254740991` diff --git a/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/LengthConstraint.md.license b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/LengthConstraint.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/LengthConstraint.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/RangeConstraint.md b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/RangeConstraint.md new file mode 100644 index 000000000..8b61b1e0c --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/RangeConstraint.md @@ -0,0 +1,60 @@ +--- +title: RangeConstraint +--- + + + +Compatible ValueType: decimal + +## Description + +Limits the range of a number value with an upper and/or lower boundary which can be inclusive or exclusive. Only values within the given range are considered valid. + +## Example 1 + +```jayvee + constraint HundredScale oftype RangeConstraint { + lowerBound: 1; + upperBound: 100; + } +``` + +A scale between (and including) 1 and 100. + +## Example 2 + +```jayvee + constraint HundredScale oftype RangeConstraint { + lowerBound: 1; + lowerBoundInclusive: false; + upperBound: 100; + } +``` + +A scale for numbers strictly larger than 1 and less or equal to 100. + +## Properties + +### `lowerBound` + +Type `decimal` + +Default: `-9007199254740991` + +### `lowerBoundInclusive` + +Type `boolean` + +Default: `true` + +### `upperBound` + +Type `decimal` + +Default: `9007199254740991` + +### `upperBoundInclusive` + +Type `boolean` + +Default: `true` diff --git a/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/RangeConstraint.md.license b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/RangeConstraint.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/RangeConstraint.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/RegexConstraint.md b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/RegexConstraint.md new file mode 100644 index 000000000..b9b5c7948 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/RegexConstraint.md @@ -0,0 +1,28 @@ +--- +title: RegexConstraint +--- + + + +Compatible ValueType: text + +## Description + +Limits the values complying with a regex. + Only values that comply with the regex are considered valid. + +## Example 1 + +```jayvee + constraint IPv4Format oftype RegexConstraint { + regex: /^((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\b){4}$/; + } +``` + +Text that complies with the IPv4 address format. + +## Properties + +### `regex` + +Type `Regex` diff --git a/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/RegexConstraint.md.license b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/RegexConstraint.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/RegexConstraint.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/_category_.json b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/_category_.json new file mode 100644 index 000000000..cd8b13006 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Constraint Types", + "position": 6, + "link": { + "type": "generated-index", + "description": "These constraints are shipped with Jayvee and are available right out of the box." + } +} diff --git a/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/_category_.json.license b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/_category_.json.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/constraint-types/_category_.json.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/user/core-concepts.md b/apps/docs/versioned_docs/version-0.3.0/user/core-concepts.md new file mode 100644 index 000000000..add0a01f5 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/core-concepts.md @@ -0,0 +1,89 @@ +--- +sidebar_position: 2 +--- + +# Core Concepts + +The core concepts of Jayvee are `Pipelines`, `Blocks`, and `ValueTypes`. + +## Pipelines + +A `Pipeline` is a sequence of different computing steps, the `Blocks`. +The default output of a block becomes the default input of the next block, building a chain of computing steps. +In the scope of a `Pipeline`, you can connect these blocks via the `pipe` syntax: + +```jayvee +pipeline CarsPipeline { + // Assumption: blocks "GasReserveHttpExtractor", "GasReserveCSVInterpreter", "GasReserveTableInterpreter", and "GasReserveLoader" are defined + + GasReserveHttpExtractor + -> GasReserveTextFileInterpreter + -> GasReserveCSVInterpreter + -> GasReserveTableInterpreter + -> GasReserveLoader; +} +``` + +## Blocks + +A `Block` is a processing step within a `Pipeline`. +It can have a default input and a default output. +We differentiate the following types of `Blocks`: +- `ExtractorBlocks` do not have a default input but only a default output. They model a **data source**. +- `TransformatorBlocks` have a default input and a default output. They model a **transformation**. +- `LoaderBlocks` do have a default input but nor a default output. They model a **data sink**. + +The general structure of a `Pipeline` consisting of different blocks is the following: + +```mermaid +flowchart LR + A[ExtractorBlock] --> B(TransformatorBlock) + B --> C(TransformatorBlock) + C --> D(LoaderBlock) +``` + +The common syntax of blocks is at its core a key-value map to provide configuration to the block. +The availability of property keys and their respective `ValueTypes` is determined by the type of the `Block` - indicated by the identifier after the keyword `oftype`: + +```jayvee +block GasReserveHttpExtractor oftype HttpExtractor { + // key: value + url: "https://www.bundesnetzagentur.de/_tools/SVG/js2/_functions/csv_export.html?view=renderCSV&id=1089590"; +} +``` + +In the example above, the `url` property of type `text` is defined by the corresponding `HttpExtractor` block type. + +Blocks can be either defined as part of the language, called `builtin` or defined as composition of existing blocks by users in Jayvee, called `composite`. See the documentation for [Composite Blocks](./composite-blocks.md). + +## ValueTypes + +A `ValueType` is the definition of a data type of the processed data. +Some `Blocks` use `ValueTypes` to define logic (like filtering or assessing the data type in a data sink). +We differentiate the following types of `ValueTypes`: +- `Built-in ValueTypes` come with the basic version of Jayvee. See [Built-in Valuetypes](./valuetypes/builtin-valuetypes). +- `Primitive ValueTypes` can be defined by the user to model domain-specific data types and represent a single value. + `Constraints` can be added to a `Primitive ValueType`. +See [Primitive Valuetypes](./valuetypes/primitive-valuetypes). +- `Compound ValueTypes`: UPCOMING. + +```jayvee +valuetype GasFillLevel oftype integer { + constraints: [ GasFillLevelRange ]; +} + +constraint GasFillLevelRange on decimal: + value >= 0 and value <= 100; +``` + +## Transforms +`Transforms` are used to transform data from one `ValueType` to a different one. For more details, see [Transforms](./transforms.md). + +```jayvee +transform CelsiusToKelvin { + from tempCelsius oftype decimal; + to tempKelvin oftype decimal; + + tempKelvin: tempCelsius + 273.15; +} +``` \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/core-concepts.md.license b/apps/docs/versioned_docs/version-0.3.0/user/core-concepts.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/core-concepts.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/docs/user/examples/_category_.json b/apps/docs/versioned_docs/version-0.3.0/user/examples/_category_.json similarity index 100% rename from apps/docs/docs/user/examples/_category_.json rename to apps/docs/versioned_docs/version-0.3.0/user/examples/_category_.json diff --git a/apps/docs/versioned_docs/version-0.3.0/user/examples/_category_.json.license b/apps/docs/versioned_docs/version-0.3.0/user/examples/_category_.json.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/examples/_category_.json.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/user/examples/cars.md b/apps/docs/versioned_docs/version-0.3.0/user/examples/cars.md new file mode 100644 index 000000000..ea96dbfa0 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/examples/cars.md @@ -0,0 +1,110 @@ +--- +title: cars +--- + +```jayvee +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +// Example 1: Cars +// Learning goals: +// - Understand the core concepts pipeline, block, and pipe +// - Understand the general structure of a pipeline + +// 1. This Jayvee model describes a pipeline +// from a CSV file in the web +// to a SQLite file sink. +pipeline CarsPipeline { + + // 2. We describe the structure of the pipeline, + // usually at the top of the pipeline. + // by connecting blocks via pipes. + + // 3. Syntax of a pipe + // connecting the block CarsExtractor + // with the block CarsTextFileInterpreter. + CarsExtractor -> CarsTextFileInterpreter; + + // 4. The output of the preceding block is hereby used + // as input for the succeeding block. + + // 5. Pipes can be further chained, + // leading to an overview of the pipeline. + CarsTextFileInterpreter + -> CarsCSVInterpreter + -> NameHeaderWriter + -> CarsTableInterpreter + -> CarsLoader; + + + // 6. Below the pipes, we usually define the blocks + // that are connected by the pipes. + + // 7. Blocks instantiate a blocktype by using the oftype keyword. + // The blocktype defines the available properties that the block + // can use to specify the intended behavior of the block + block CarsExtractor oftype HttpExtractor { + + // 8. Properties are assigned to concrete values. + // Here, we specify the URL where the file shall be downloaded from. + url: "https://gist.githubusercontent.com/noamross/e5d3e859aa0c794be10b/raw/b999fb4425b54c63cab088c0ce2c0d6ce961a563/cars.csv"; + } + + // 9. The HttpExtractor requires no input and produces a binary file as output. + // This file has to be interpreted, e.g., as text file. + block CarsTextFileInterpreter oftype TextFileInterpreter { } + + // 10. Next, we interpret the text file as sheet. + // A sheet only contains text cells and is useful for manipulating the shape of data before assigning more strict value types to cells. + block CarsCSVInterpreter oftype CSVInterpreter { + enclosing: '"'; + } + + // 11. We can write into cells of a sheet using the CellWriter blocktype. + block NameHeaderWriter oftype CellWriter { + // 12. We utilize a syntax similar to spreadsheet programs. + // Cell ranges can be described using the keywords "cell", "row", "column", or "range" that indicate which + // cells are selected for the write action. + at: cell A1; + + // 13. For each cell we selected with the "at" property above, + // we can specify what value shall be written into the cell. + write: ["name"]; + } + + // 14. As a next step, we interpret the sheet as a table by adding structure. + // We define a valuetype per column that specifies the data type of the column. + // Rows that include values that are not valid according to the their valuetypes are dropped automatically. + block CarsTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "name" oftype text, + "mpg" oftype decimal, + "cyl" oftype integer, + "disp" oftype decimal, + "hp" oftype integer, + "drat" oftype decimal, + "wt" oftype decimal, + "qsec" oftype decimal, + "vs" oftype integer, + "am" oftype integer, + "gear" oftype integer, + "carb" oftype integer + ]; + } + + // 15. As a last step, we load the table into a sink, + // here into a sqlite file. + // The structural information of the table is used + // to generate the correct table. + block CarsLoader oftype SQLiteLoader { + table: "Cars"; + file: "./cars.sqlite"; + } + + // 16. Congratulations! + // You can now use the sink for your data analysis, app, + // or whatever you want to do with the cleaned data. +} +``` \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/examples/cars.md.license b/apps/docs/versioned_docs/version-0.3.0/user/examples/cars.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/examples/cars.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/examples/electric-vehicles.md b/apps/docs/versioned_docs/version-0.3.0/user/examples/electric-vehicles.md new file mode 100644 index 000000000..99398633c --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/examples/electric-vehicles.md @@ -0,0 +1,150 @@ +--- +title: electric-vehicles +--- + +```jayvee +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +// Example 2: Electric Vehicles +// Learning goals: +// - Understand further core concepts transforms and valuetypes +// - Understand how to construct a pipeline with multiple sinks +// - Understand the use of runtime parameters + +// 1. This Jayvee model describes a pipeline +// from a CSV file in the web +// to a SQLite file and a PostgreSQL db sink. +pipeline ElectricVehiclesPipeline { + // See here for meta-data of the data source + // https://catalog.data.gov/dataset/electric-vehicle-population-data/resource/fa51be35-691f-45d2-9f3e-535877965e69 + + // 2. At the top of a pipeline, we describe the + // structure of the pipeline. The first part until + // the ElectricRangeTransformer is a linear sequence + // of blocks. From there we can see a split into two + // parallel sequences that load the data in to two + // different sinks. + ElectricVehiclesHttpExtractor + -> ElectricVehiclesTextFileInterpreter + -> ElectricVehiclesCSVInterpreter + -> ElectricVehiclesTableInterpreter + -> ElectricRangeTransformer; + + ElectricRangeTransformer + -> ElectricVehiclesSQLiteLoader; + + ElectricRangeTransformer + -> ElectricVehiclesPostgresLoader; + + // 3. After the pipeline structure, we define the blocks used. + block ElectricVehiclesHttpExtractor oftype HttpExtractor { + url: "https://data.wa.gov/api/views/f6w7-q2d2/rows.csv?accessType=DOWNLOAD"; + } + + block ElectricVehiclesTextFileInterpreter oftype TextFileInterpreter { } + + block ElectricVehiclesCSVInterpreter oftype CSVInterpreter { } + + block ElectricVehiclesTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + // 4. Here, a user-deifned valuetype is used to describe this column. + // The capital letter indicates that the valuetype is not builtin + // by convention. The valuetype itself is defined further below. + "VIN (1-10)" oftype VehicleIdentificationNumber10, + "County" oftype text, + "City" oftype text, + "State" oftype UsStateCode, + "Postal Code" oftype text, + "Model Year" oftype integer, + "Make" oftype text, + "Model" oftype text, + "Electric Vehicle Type" oftype text, + "Clean Alternative Fuel Vehicle (CAFV) Eligibility" oftype text, + "Electric Range" oftype integer, + "Base MSRP" oftype integer, + "Legislative District" oftype text, + "DOL Vehicle ID" oftype integer, + "Vehicle Location" oftype text, + "Electric Utility" oftype text, + "2020 Census Tract" oftype text, + ]; + } + + // 5. This block describes the application of a transform function + // taking a column as input and adding another computed column. + // The applied transform function is defined below and referenced + // by the "use" property. + block ElectricRangeTransformer oftype TableTransformer { + inputColumns: ["Electric Range"]; + outputColumn: "Electric Range (km)"; + use: MilesToKilometers; + } + + // 6. Here, we define a transform function, taking parameters + // as input ("from" keyword), and producing an output ("to" keyword). + // Inputs and outputs have to be further described by a valuetype. + transform MilesToKilometers { + from miles oftype decimal; + to kilometers oftype integer; + + // 7. In order to express what the transform function does, + // we assign an expression to the output. Values from the input and output of the transform can be referred to by name. + kilometers: round (miles * 1.609344); + } + + block ElectricVehiclesSQLiteLoader oftype SQLiteLoader { + table: "ElectricVehiclePopulationData"; + file: "./electric-vehicles.sqlite"; + } + + block ElectricVehiclesPostgresLoader oftype PostgresLoader { + // 8. The requires keyword allows us to define runtime parameters. + // These values have to be provided as environment variables when interpreting the Jayvee model. + host: requires DB_HOST; + port: requires DB_PORT; + username: requires DB_USERNAME; + password: requires DB_PASSWORD; + database: requires DB_DATABASE; + table: "ElectricVehiclePopulationData"; + } +} + +// 9. Below the pipeline, we model user-define valuetypes. +// We give them a speaking name and provide a base valuetype +// that this valuetype builts on. User-defined valuetypes always place additional constraints on existing valuetypes. +valuetype VehicleIdentificationNumber10 oftype text { + // 10. Valuetypes can be further refined by providing constraints. + constraints: [ + OnlyCapitalLettersAndDigits, + ExactlyTenCharacters, + ]; +} + +// 11. This constraint works on text valuetypes and requires values +// to match a given regular expression in order to be valid. +constraint OnlyCapitalLettersAndDigits on text: + value matches /^[A-Z0-9]*$/; + +constraint ExactlyTenCharacters on text: + value.length == 10; + +valuetype UsStateCode oftype text { + constraints: [ + UsStateCodeAllowlist, + ]; +} + +constraint UsStateCodeAllowlist on text: + value in [ + "AL", "AK", "AZ", "AR", "AS", "CA", "CO", "CT", "DE", "DC", + "FL", "GA", "GU", "HI", "ID", "IL", "IN", "IA", "KS", "KY", + "LA", "ME", "MD", "MA", "MI", "MN", "MS", "MO", "MT", "NE", + "NV", "NH", "NJ", "NM", "NY", "NC", "ND", "MP", "OH", "OK", + "OR", "PA", "PR", "RI", "SC", "SD", "TN", "TX", "TT", "UT", + "VT", "VA", "VI", "WA", "WV", "WI", "WY", + ]; + +``` \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/examples/electric-vehicles.md.license b/apps/docs/versioned_docs/version-0.3.0/user/examples/electric-vehicles.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/examples/electric-vehicles.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/examples/gtfs-rt.md b/apps/docs/versioned_docs/version-0.3.0/user/examples/gtfs-rt.md new file mode 100644 index 000000000..7610d8c10 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/examples/gtfs-rt.md @@ -0,0 +1,133 @@ +--- +title: gtfs-rt +--- + +```jayvee +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +// Example 3: GTFS Realtime Data +// Learning goals: +// - Understand the construction of a csv file with multiple tables +// - Understand how to work with live data + +// 1. This Jayvee model describes a pipeline +// from a GTFS RT data source in the web +// to a SQLite file with multiple tables. +pipeline GtfsRTSimplePipeline { + + // 2. As you can see here, we have three independent + // sequences of pipes in this pipeline. + GTFSRTTripUpdateFeedExtractor + ->GtfsRTTripUpdateInterpreter + ->TripUpdateTableInterpreter + ->TripUpdateLoader; + + GTFSRTVehiclePositionFeedExtractor + ->GtfsRTVehiclePositionInterpreter + ->VehiclePositionTableInterpreter + ->VehicleLoader; + + GTFSRTAlertFeedExtractor + ->GtfsRTAlertInterpreter + ->AlertTableInterpreter + ->AlertLoader; + + // 3. We define a series of HttpExtractors that each pull data + // from an HTTP endpoint + block GTFSRTTripUpdateFeedExtractor oftype HttpExtractor { + url: "https://proxy.transport.data.gouv.fr/resource/bibus-brest-gtfs-rt-trip-update"; + } + + block GTFSRTVehiclePositionFeedExtractor oftype HttpExtractor { + url: "https://proxy.transport.data.gouv.fr/resource/bibus-brest-gtfs-rt-vehicle-position"; + } + + block GTFSRTAlertFeedExtractor oftype HttpExtractor { + url: "https://proxy.transport.data.gouv.fr/resource/bibus-brest-gtfs-rt-alerts"; + } + + // 4. In the next step, we use the domain-specific GtfsRTInterpreter + // to interpret the fetched files as sheets + block GtfsRTTripUpdateInterpreter oftype GtfsRTInterpreter { + entity: "trip_update"; + } + + block GtfsRTAlertInterpreter oftype GtfsRTInterpreter { + entity: "alert"; + } + + block GtfsRTVehiclePositionInterpreter oftype GtfsRTInterpreter { + entity: "vehicle"; + } + + // 5. Next, we interpret the sheets as tables + block TripUpdateTableInterpreter oftype TableInterpreter { + header: true; + columns:[ + "header.gtfs_realtime_version" oftype text, + "header.timestamp" oftype text, + "header.incrementality" oftype text, + "entity.id" oftype text, + "entity.trip_update.trip.trip_id" oftype text, + "entity.trip_update.trip.route_id" oftype text, + "entity.trip_update.stop_time_update.stop_sequence" oftype text, + "entity.trip_update.stop_time_update.stop_id" oftype text, + "entity.trip_update.stop_time_update.arrival.time" oftype text, + "entity.trip_update.stop_time_update.departure.time" oftype text, + ]; + } + + block VehiclePositionTableInterpreter oftype TableInterpreter { + header: true; + columns:[ + "header.gtfs_realtime_version" oftype text, + "header.timestamp" oftype text, + "header.incrementality" oftype text, + "entity.id" oftype text, + "entity.vehicle_position.vehicle_descriptor.id" oftype text, + "entity.vehicle_position.trip.trip_id" oftype text, + "entity.vehicle_position.trip.route_id" oftype text, + "entity.vehicle_position.position.latitude" oftype text, + "entity.vehicle_position.position.longitude" oftype text, + "entity.vehicle_position.timestamp" oftype text + ]; + } + + block AlertTableInterpreter oftype TableInterpreter { + header: true; + columns:[ + 'header.gtfs_realtime_version' oftype text, + 'header.timestamp' oftype text, + 'header.incrementality' oftype text, + 'entity.id' oftype text, + 'entity.alert.informed_entity.route_id' oftype text, + 'entity.alert.header_text' oftype text, + 'entity.alert.description_text' oftype text, + ]; + } + + // 6. Last, we load the tables into the same SQLite file. + // Each loader has to define a different table name. + // For working with live data, we use the property "dropTable: false" + // to append data instead of deleting the previous data. + block TripUpdateLoader oftype SQLiteLoader { + table: "gtfs-rt-trip_update"; + file: "./gtfs.sqlite"; + dropTable: false; + } + + block VehicleLoader oftype SQLiteLoader { + table: "gtfs-rt-vehicle_position"; + file: "./gtfs.sqlite"; + dropTable: false; + } + + block AlertLoader oftype SQLiteLoader { + table: "gtfs-rt-alert"; + file: "./gtfs.sqlite"; + dropTable: false; + } +} +``` \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/examples/gtfs-rt.md.license b/apps/docs/versioned_docs/version-0.3.0/user/examples/gtfs-rt.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/examples/gtfs-rt.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/examples/gtfs-static.md b/apps/docs/versioned_docs/version-0.3.0/user/examples/gtfs-static.md new file mode 100644 index 000000000..ca6434df6 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/examples/gtfs-static.md @@ -0,0 +1,370 @@ +--- +title: gtfs-static +--- + +```jayvee +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +// Example 4: GTFS Static Data +// Learning goals: +// - Understand how to work with file systems + +// 1. This Jayvee model describes a pipeline +// from a zip file in the GTFS format in the web +// to a joint SQLite file with multiple tables. +pipeline GtfsPipeline { + + // 2. The origin for multiple pipe sequences is a zip + // file. Each csv file in this zip is further processed + // by its own sequence of blocks and pipes. + GTFSSampleFeedExtractor -> ZipArchiveInterpreter; + + ZipArchiveInterpreter + -> AgencyFilePicker + -> AgencyTextFileInterpreter + -> AgencyCSVInterpreter + -> AgencyTableInterpreter + -> AgencyLoader; + + ZipArchiveInterpreter + -> CalendarDatesFilePicker + -> CalendarDatesTextFileInterpreter + -> CalendarDatesCSVInterpreter + -> CalendarDatesTableInterpreter + -> CalendarDatesLoader; + + ZipArchiveInterpreter + -> CalendarFilePicker + -> CalendarTextFileInterpreter + -> CalendarCSVInterpreter + -> CalendarTableInterpreter + -> CalendarLoader; + + ZipArchiveInterpreter + -> FareAttributesFilePicker + -> FareAttributesTextFileInterpreter + -> FareAttributesCSVInterpreter + -> FareAttributesTableInterpreter + -> FareAttributesLoader; + + ZipArchiveInterpreter + -> FareRulesFilePicker + -> FareRulesTextFileInterpreter + -> FareRulesCSVInterpreter + -> FareRulesTableInterpreter + -> FareRulesLoader; + + ZipArchiveInterpreter + -> FrequenciesFilePicker + -> FrequenciesTextFileInterpreter + -> FrequenciesCSVInterpreter + -> FrequenciesTableInterpreter + -> FrequenciesLoader; + + ZipArchiveInterpreter + -> RoutesFilePicker + -> RoutesTextFileInterpreter + -> RoutesCSVInterpreter + -> RoutesTableInterpreter + -> RoutesLoader; + + ZipArchiveInterpreter + -> ShapesFilePicker + -> ShapesTextFileInterpreter + -> ShapesCSVInterpreter + -> ShapesTableInterpreter + -> ShapesLoader; + + ZipArchiveInterpreter + -> StopTimesFilePicker + -> StopTimesTextFileInterpreter + -> StopTimesCSVInterpreter + -> StopTimesTableInterpreter + -> StopTimesLoader; + + ZipArchiveInterpreter + -> StopsFilePicker + -> StopsTextFileInterpreter + -> StopsCSVInterpreter + -> StopsTableInterpreter + -> StopsLoader; + + ZipArchiveInterpreter + -> TripsFilePicker + -> TripsTextFileInterpreter + -> TripsCSVInterpreter + -> TripsTableInterpreter + -> TripsLoader; + + // 3. As a first step, we download the zip file and interpret it. + block GTFSSampleFeedExtractor oftype HttpExtractor { + url: "https://developers.google.com/static/transit/gtfs/examples/sample-feed.zip"; + } + + block ZipArchiveInterpreter oftype ArchiveInterpreter { + archiveType: "zip"; + } + + // 4. Next, we pick several csv files (with the file extension ".txt") + // for further processing . + block AgencyFilePicker oftype FilePicker { + path: "/agency.txt"; + } + + block CalendarDatesFilePicker oftype FilePicker { + path: "/calendar_dates.txt"; + } + + block CalendarFilePicker oftype FilePicker { + path: "/calendar.txt"; + } + + block FareAttributesFilePicker oftype FilePicker { + path: "/fare_attributes.txt"; + } + + block FareRulesFilePicker oftype FilePicker { + path: "/fare_rules.txt"; + } + + block FrequenciesFilePicker oftype FilePicker { + path: "/frequencies.txt"; + } + + block RoutesFilePicker oftype FilePicker { + path: "/routes.txt"; + } + + block ShapesFilePicker oftype FilePicker { + path: "/shapes.txt"; + } + + block StopTimesFilePicker oftype FilePicker { + path: "/stop_times.txt"; + } + + block StopsFilePicker oftype FilePicker { + path: "/stops.txt"; + } + + block TripsFilePicker oftype FilePicker { + path: "/trips.txt"; + } + + // 5. The rest of the pipeline follows the usual pattern. + block AgencyTextFileInterpreter oftype TextFileInterpreter { } + block CalendarDatesTextFileInterpreter oftype TextFileInterpreter { } + block CalendarTextFileInterpreter oftype TextFileInterpreter { } + block FareAttributesTextFileInterpreter oftype TextFileInterpreter { } + block FareRulesTextFileInterpreter oftype TextFileInterpreter { } + block FrequenciesTextFileInterpreter oftype TextFileInterpreter { } + block RoutesTextFileInterpreter oftype TextFileInterpreter { } + block ShapesTextFileInterpreter oftype TextFileInterpreter { } + block StopTimesTextFileInterpreter oftype TextFileInterpreter { } + block StopsTextFileInterpreter oftype TextFileInterpreter { } + block TripsTextFileInterpreter oftype TextFileInterpreter { } + block AgencyCSVInterpreter oftype CSVInterpreter { } + block CalendarDatesCSVInterpreter oftype CSVInterpreter { } + block CalendarCSVInterpreter oftype CSVInterpreter { } + block FareAttributesCSVInterpreter oftype CSVInterpreter { } + block FareRulesCSVInterpreter oftype CSVInterpreter { } + block FrequenciesCSVInterpreter oftype CSVInterpreter { } + block RoutesCSVInterpreter oftype CSVInterpreter { } + block ShapesCSVInterpreter oftype CSVInterpreter { } + block StopTimesCSVInterpreter oftype CSVInterpreter { } + block StopsCSVInterpreter oftype CSVInterpreter { } + block TripsCSVInterpreter oftype CSVInterpreter { } + + block AgencyTableInterpreter oftype TableInterpreter { + header: true; + columns:[ + "agency_id" oftype text, //Conditional columns are considered as required + "agency_name" oftype text, + "agency_url" oftype text, + "agency_timezone" oftype text + ]; + } + + block CalendarDatesTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "service_id" oftype text, + "date" oftype text, + "exception_type" oftype text + ]; + } + + block CalendarTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "service_id" oftype text, + "monday" oftype text, + "tuesday" oftype text, + "wednesday" oftype text, + "thursday" oftype text, + "friday" oftype text, + "saturday" oftype text, + "sunday" oftype text, + "start_date" oftype text, + "end_date" oftype text + ]; + } + + block FareAttributesTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "fare_id" oftype text, + "price" oftype text, + "currency_type" oftype text, + "payment_method" oftype text, + "transfers" oftype text, + "transfer_duration" oftype text + ]; + } + + block FareRulesTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "fare_id" oftype text, + "route_id" oftype text, + "origin_id" oftype text, + "destination_id" oftype text, + "contains_id" oftype text + ]; + } + + block FrequenciesTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "trip_id" oftype text, + "start_time" oftype text, + "end_time" oftype text, + "headway_secs" oftype text + ]; + } + + block RoutesTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "route_id" oftype text, + "agency_id" oftype text, + "route_short_name" oftype text, + "route_long_name" oftype text, + "route_desc" oftype text, + "route_type" oftype text, + "route_url" oftype text, + "route_color" oftype text, + "route_text_color" oftype text + ]; + } + + block ShapesTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "shape_id" oftype text, + "shape_pt_lat" oftype text, + "shape_pt_lon" oftype text, + "shape_pt_sequence" oftype text, + "shape_dist_traveled" oftype text + ]; + } + + block StopTimesTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "trip_id" oftype text, + "arrival_time" oftype text, + "departure_time" oftype text, + "stop_id" oftype text, + "stop_sequence" oftype text, + "stop_headsign" oftype text, + "pickup_type" oftype text, + "drop_off_time" oftype text, + "shape_dist_traveled" oftype text + ]; + } + + block StopsTableInterpreter oftype TableInterpreter { + header: true; + columns:[ + "stop_id" oftype text, + "stop_name" oftype text, + "stop_desc" oftype text, + "stop_lat" oftype text, + "stop_lon" oftype text, + "zone_id" oftype text, + "stop_url" oftype text + ]; + } + + block TripsTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "route_id" oftype text, + "service_id" oftype text, + "trip_id" oftype text, + "trip_headsign" oftype text, + "direction_id" oftype text, + "block_id" oftype text, + "shape_id" oftype text + ]; + } + + block AgencyLoader oftype SQLiteLoader { + table: "agency"; + file: "./gtfs.sqlite"; + } + + block CalendarDatesLoader oftype SQLiteLoader { + table: "calendar_dates"; + file: "./gtfs.sqlite"; + } + + block CalendarLoader oftype SQLiteLoader { + table: "calendar"; + file: "./gtfs.sqlite"; + } + + block FareAttributesLoader oftype SQLiteLoader { + table: "fare_attributes"; + file: "./gtfs.sqlite"; + } + + block FareRulesLoader oftype SQLiteLoader { + table: "fare_rules"; + file: "./gtfs.sqlite"; + } + + block FrequenciesLoader oftype SQLiteLoader { + table: "frequencies"; + file: "./gtfs.sqlite"; + } + + block RoutesLoader oftype SQLiteLoader { + table: "routes"; + file: "./gtfs.sqlite"; + } + + block ShapesLoader oftype SQLiteLoader { + table: "shapes"; + file: "./gtfs.sqlite"; + } + + block StopTimesLoader oftype SQLiteLoader { + table: "stop_times"; + file: "./gtfs.sqlite"; + } + + block StopsLoader oftype SQLiteLoader { + table: "stops"; + file: "./gtfs.sqlite"; + } + + block TripsLoader oftype SQLiteLoader { + table: "trips"; + file: "./gtfs.sqlite"; + } +} +``` \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/examples/gtfs-static.md.license b/apps/docs/versioned_docs/version-0.3.0/user/examples/gtfs-static.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/examples/gtfs-static.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/examples/workbooks-xlsx.md b/apps/docs/versioned_docs/version-0.3.0/user/examples/workbooks-xlsx.md new file mode 100644 index 000000000..a46e886a0 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/examples/workbooks-xlsx.md @@ -0,0 +1,97 @@ +--- +title: workbooks-xlsx +--- + +```jayvee +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +// Example 1: LightTrapping +// Learning goals: +// - Understand how to work with XLSX files and workbooks + +// 1. This Jayvee model describes a pipeline +// from a XLSX file with multiple Sheets in the web +// to a SQLite file sink. +pipeline LightTrappingSiliconSolarCellsPipeline { + // 2. We directly get the xlsx file from the web via the HttpExtractor + // The data is provided under CC BY-SA 4.0 + // Saive, Rebecca (2023). Data supporting the publication: + // Light trapping in thin silicon solar cells: a review on fundamentals and technologies. + // 4TU.ResearchData. Dataset. https://doi.org/10.4121/14554815.v1 + block LightTrappingSiliconSolarCellsExtractor oftype HttpExtractor { + url: "https://figshare.com/ndownloader/files/27923598"; + } + + // 3. The incoming file is interpreted as a XLSX file and transformed into a Workbook + // Workbooks contain at least 1 Sheet. Every sheet has a unique name. + block LightTrappingSiliconSolarCellsTextXLSXInterpreter oftype XLSXInterpreter { + + } + + // 4.1 Here, we pick one sheet with the name 'RefractiveIndexSi GaAs' from the Workbook to use within our pipeline. + // The output type from SheetPicker is Sheet, which was already introduced in the cars example + block LightTrappingSiliconSolarCellsSheetpicker oftype SheetPicker { + sheetName: 'RefractiveIndexSi GaAs'; + } + + block NameHeaderWriter oftype CellWriter { + at: range F1:L1; + write: ["F","G","nm","wl","n2", "k2", "alpha (cm-1)2"]; + } + + block LightTrappingSiliconSolarCellsTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "Wavelength" oftype integer, + "Wavelength (µm)" oftype decimal, + "n" oftype decimal, + "k" oftype text, + "alpha (cm-1)" oftype text, + "nm" oftype decimal, + "n2" oftype text, + "k2" oftype decimal, + "alpha (cm-1)2" oftype decimal + ]; + } + + block LightTrappingSiliconSolarCellsLoader oftype SQLiteLoader { + table: "LightTrappingSiliconSolarCells"; + file: "./LightTrappingSiliconSolarCells.sqlite"; + } + + // 4.2 Here, we pick another sheet named 'Wavelength thickness trapping' from the Workbook + block SecondLightTrappingSiliconSolarCellsSheetpicker oftype SheetPicker { + sheetName: 'Wavelength thickness trapping'; + } + + block SecondLightTrappingSiliconSolarCellsTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "n" oftype decimal, + "Wavelength (µm)" oftype decimal, + ]; + } + + block SecondLightTrappingSiliconSolarCellsLoader oftype SQLiteLoader { + + table: "SecondLightTrappingSiliconSolarCells"; + file: "./LightTrappingSiliconSolarCells.sqlite"; + } + + LightTrappingSiliconSolarCellsExtractor + -> LightTrappingSiliconSolarCellsTextXLSXInterpreter + -> LightTrappingSiliconSolarCellsSheetpicker + -> NameHeaderWriter + -> LightTrappingSiliconSolarCellsTableInterpreter + -> LightTrappingSiliconSolarCellsLoader; + + // 5. Once the XLSX file is interpreted, we can split the pipeline and + // work separately on the different sheets from our input file + LightTrappingSiliconSolarCellsTextXLSXInterpreter + -> SecondLightTrappingSiliconSolarCellsSheetpicker + -> SecondLightTrappingSiliconSolarCellsTableInterpreter + -> SecondLightTrappingSiliconSolarCellsLoader; +} +``` \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/examples/workbooks-xlsx.md.license b/apps/docs/versioned_docs/version-0.3.0/user/examples/workbooks-xlsx.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/examples/workbooks-xlsx.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/expressions.md b/apps/docs/versioned_docs/version-0.3.0/user/expressions.md new file mode 100644 index 000000000..99e612a5f --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/expressions.md @@ -0,0 +1,80 @@ +--- +sidebar_position: 7 +--- + +# Expressions + +Expressions in Jayvee are arbitrarily nested statements. They consist of: +- literals (e.g., numbers `5` or strings `"Example"`) +- variables (e.g., declared by `from` properties in [Transforms](./transforms.md)) +- operators (e.g., `*` or `sqrt`) + +Expressions get evaluated at runtime by the interpreter to a [Built-in ValueType](./valuetypes/builtin-valuetypes). + +### Example + +The following expression is evaluated to the `integer` `10`: `(2 + 3) * 2` + +The following expression is evaluated to the `integer` `3`: `floor (3.14)` + +The following expression is evaluated to the `boolean` `true`: `"Example" == "Example"` + +### List of Operators + +#### Arithmetics (binary operators) +- `+` for addition, e.g., `5 + 3` evaluates to `8` +- `-` for subtraction, e.g., `5 - 3` evaluates to `2` +- `*` for multiplication, e.g., `5 * 3` evaluates to `15` +- `/` for division, e.g., `6 / 3` evaluates to `2` +- `%` for modulo, e.g., `5 % 3` evaluates to `2` +- `pow` for power, e.g., `2 pow 3` evaluates to `8` +- `root` for root, e.g., `27 root 3` evaluates to `3` + +#### Arithmetics (unary operators) +- `+` for positive signing, e.g., `+5` evaluates to `5` +- `-` for negative signing, e.g., `-5` evaluates to `-5` +- `sqrt` for square root, e.g., `sqrt 9` evaluates to `3` +- `foor` for flooring a number, e.g., `floor 5.3` evaluates to `5` +- `ceil` for ceiling a number, e.g., `floor 5.3` evaluates to `6` +- `round` for rounding a number, e.g., `floor 5.3` evaluates to `5` + +#### Relational (binary operators) +- `<` for smaller, e.g., `3 < 3` evaluates to `false` +- `<=` for smaller or equal, e.g., `3 <= 3` evaluates to `true` +- `>` for greater, e.g., `3 > 3` evaluates to `false` +- `>=` for greater or equal, e.g., `3 >= 3` evaluates to `true` +- `==` for equal, e.g., `3 == 3` evaluates to `true` +- `!=` for not equal, e.g., `3 != 3` evaluates to `false` + +#### Logical (binary operators) +- `and` for a logical and (both need to be true to evaluate to true) +- `or` for a logical or (at least left or right needs to be true to evaluate to true) +- `xor` for a logical xor (either left or right needs to be true to evaluate to true) + +#### Logical (unary operators) +- `not` for logical negation, `not true` evaluates to `false` + +#### Others (binary operators) +- `matches` for a regex match, e.g., `"A07" matches /^[A-Z0-9]*$/` evaluates to `true` +- `in` for inclusion in an array, e.g., `"a" in ["a", "b", "c"]` evaluates to `true` + +### Operator Details + +#### `in` Operator + +The `in` operator checks whether a value is included in a collection of values. For example: + +```jayvee +4.5 in [3, 6.5] // evaluates to false +3 in [3.0, 6.5] // evaluates to true +"a" in ["a", "b", "c"] // evaluates to true +``` + +The operator supports `text`, `integer` and `decimal` values as operands. The compatibility of left and right operand types follows these rules: +- For the `in` operator we have a type for the needle (left operand) and a type for the elements in the haystack (right operand). +- There is an automated type conversion as long as it is lossless and clearly defined (integer to decimal as of now). +- We allow any combination of operands that has either: (i) An automated type conversion from needle type (left operand) to the type of the elements in the haystack (right operand), or (ii) the other way around. + + +### Further reading +For a deeper documentation of how expressions and operators work internally, refer to the [developer docs](../dev/04-guides/04-expressions-and-operators.md). \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/expressions.md.license b/apps/docs/versioned_docs/version-0.3.0/user/expressions.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/expressions.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/user/intro.md b/apps/docs/versioned_docs/version-0.3.0/user/intro.md new file mode 100644 index 000000000..33e4bb6ad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/intro.md @@ -0,0 +1,99 @@ +--- +sidebar_position: 1 +--- + +# Introduction to Jayvee + +Jayvee is a domain-specific language (DSL) for automated processing of data pipelines. +The Jayvee interpreter allows executing such data pipelines on local machines. +Data engineers can use Jayvee and its interpreter to clean and preprocess data for later activities like data science or machine learning. + +## Installation + +Install the interpreter via `npm`. You will need a **nodejs version >= 17.0.0**. + +```bash +npm install -g @jvalue/jayvee-interpreter +``` + +You can install a specific version using the `@`-syntax, e.g., version `0.0.17`: + +```bash +npm install -g @jvalue/jayvee-interpreter@0.0.17 +``` + +## Update + +Updating the interpreter is done by reinstalling it using `npm`. Make sure to also update the [VSCode plugin](#vscode-plugin) to match the installed interpreter if you use it. + +```bash +npm install -g @jvalue/jayvee-interpreter +``` + +## Usage + +### Show help + +```console +jv -h +``` + +### Run a `.jv` file + +```console +jv +``` + +Run with **additional debug output**: + +```console +jv -d +``` + +With **runtime parameters**: + +```console +jv -e = -e = ... +``` + +### Debug a `.jv` file + +Print debugging is further configured by the parameters `--debug-granularity` and `--debug-target`. + +```console +jv -d -dg peek +``` + +The value of the parameter `--debug-granularity` (short `-dg`) can have the following values: + +- `peek` to log a short summary, including a small subset of data +- `exhaustive` to log a summary, including the full data +- `minimal` to log a summary, including no additional data (default). + To see logs, debugging has to be enabled using the `-d` flag. + +```console +jv -d --debug-granularity peek +``` + +The parameter `--debug-target` (short `-dt`) allows to specify which blocks should be logged for debugging. Separate block names by comma if multiple blocks are targeted. All blocks are logged if the parameter is omitted. + +```console +jv -d --debug-granularity peek --debug-target MyExtractorBlock,MySinkBlock +``` + +## Examples + +You can find multiple examples [here](https://github.com/jvalue/jayvee/tree/main/example). Copy them to your local file system and execute them with the `jv` command on your command line (see [usage](#usage)). + +## VSCode Plugin + +To set up Jayvee locally in VS Code, you need to install the latest Jayvee VS Code extension. +To install the most recent extension, go to our [latest release](https://github.com/jvalue/jayvee/releases/latest) +and download the `jayvee.vsix` file from the release assets. +Next, go to [this page](https://code.visualstudio.com/docs/editor/extension-marketplace#_install-from-a-vsix) and +follow the instructions for installing the downloaded extension. + +## Troubleshooting + +1. Error `structuredClone is not defined` + - Please make sure you use node version 17+. diff --git a/apps/docs/versioned_docs/version-0.3.0/user/intro.md.license b/apps/docs/versioned_docs/version-0.3.0/user/intro.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/intro.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/user/runtime-parameters.md b/apps/docs/versioned_docs/version-0.3.0/user/runtime-parameters.md new file mode 100644 index 000000000..bbdab1ed4 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/runtime-parameters.md @@ -0,0 +1,27 @@ +--- +sidebar_position: 9 +--- + +# Runtime Parameters + +Property values in Jayvee can be assigned to `values` or left open for later configuration via `runtime parameters`. + +## Syntax + +Runtime parameters are indicated by the `requires` keyword, followed by the identifier of the parameter. Example + +```jayvee +block CarsLoader oftype SQLiteLoader { + table: "Cars"; + file: requires CARS_SQLITE_FILE; +} +``` + +## CLI + +The interpreter CLI has to define all existing runtime parameters for execution. +Use the CLI flag `-e` to to define them as key-value pairs of identifier and value. + +```console +jv -e = -e = ... +``` diff --git a/apps/docs/versioned_docs/version-0.3.0/user/runtime-parameters.md.license b/apps/docs/versioned_docs/version-0.3.0/user/runtime-parameters.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/runtime-parameters.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/user/transforms.md b/apps/docs/versioned_docs/version-0.3.0/user/transforms.md new file mode 100644 index 000000000..bf3dfd7d9 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/transforms.md @@ -0,0 +1,77 @@ +--- +sidebar_position: 8 +--- + +# Transforms + +Transforms are a concept in Jayvee to define the transformation of individual values. +They are similar to functions in programming languages, i.e. they perform computations on some input values and produce output values. Transform work by mapping input values to outputs using [expressions](./expressions.md). + +:::info Important + +Up to version `0.0.16`, we only supported a single input for transformers! + +::: + +:::info Important + +In its current state, Jayvee only supports a arbitrary numbers of inputs and a single output for transforms. +For the future, it is planned to support arbitrary numbers for outputs as well. + +::: + + +## Syntax + +The general syntax of transforms looks like this: + +```jayvee +transform { + from oftype ; + to oftype ; + + : ; +} +``` + +The `transform` keyword is used to define a transform and give it a name. +The curly braces denote the body of the transform. + +The body first contains the definitions of input and output ports. +Input ports are defined using the `from` keyword whereas output ports use the `to` keyword. +Next, they are given a name and, after the `oftype` keyword, typed with a valuetype. + +Below, there needs to be an output assignment for each output port. +The output assignment defines how a particular output value is computed. +They consist of the name of an output port, followed by a `:`. +Next, an [expression](./expressions.md) specifies how the output value shall be computed. +Names of input ports can be used in such an expression to refer to input values. + +### Example + +The following transform converts temperature values from degree Celsius to Kelvin: + +```jayvee +transform CelsiusToKelvin { + from tempCelsius oftype decimal; + to tempKelvin oftype decimal; + + tempKelvin: tempCelsius + 273.15; +} +``` + +The following transform converts a text based status into a boolean value, `true` if the text is `Active`, `false` for any other value: + +```jayvee +transform StatusToBoolean { + from statusText oftype text; + to statusBoolean oftype boolean; + + statusBoolean: statusText == "Active"; +} +``` + +## Applying transforms to table columns + +Transforms can be applied to columns of a table. +Please refer to the documentation of the [`TableTransformer` block type](./block-types/TableTransformer.md) to find out how. diff --git a/apps/docs/versioned_docs/version-0.3.0/user/transforms.md.license b/apps/docs/versioned_docs/version-0.3.0/user/transforms.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/transforms.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/user/valuetypes/_category_.json b/apps/docs/versioned_docs/version-0.3.0/user/valuetypes/_category_.json new file mode 100644 index 000000000..ed1ce95f3 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/valuetypes/_category_.json @@ -0,0 +1,8 @@ +{ + "label": "Valuetypes", + "position": 5, + "link": { + "type": "generated-index", + "description": "Jayvee supports these different kinds of valuetypes." + } +} diff --git a/apps/docs/versioned_docs/version-0.3.0/user/valuetypes/_category_.json.license b/apps/docs/versioned_docs/version-0.3.0/user/valuetypes/_category_.json.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/valuetypes/_category_.json.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_docs/version-0.3.0/user/valuetypes/builtin-valuetypes.md b/apps/docs/versioned_docs/version-0.3.0/user/valuetypes/builtin-valuetypes.md new file mode 100644 index 000000000..0bc36dcbd --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/valuetypes/builtin-valuetypes.md @@ -0,0 +1,98 @@ +--- +title: Built-in Valuetypes +--- + + + +# Description + +For an introduction to valuetypes, see the [Core Concepts](../core-concepts). +Built-in valuetypes come with the basic version of Jayvee. +They are the basis for more restricted [Primitive Valuetypes](./primitive-valuetypes) +that fullfil [Constraints](./primitive-valuetypes#constraints). + +# Available built-in valuetypes + +## Boolean + +### Description + +A boolean value. +Examples: true, false + +### Example 1 + +```jayvee +block ExampleTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "columnName" oftype boolean + ]; +} +``` + +A block of type `TableInterpreter` that + interprets data in the column `columnName` as `boolean`. + +## Decimal + +### Description + +A decimal value. +Example: 3.14 + +### Example 1 + +```jayvee +block ExampleTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "columnName" oftype decimal + ]; +} +``` + +A block of type `TableInterpreter` that + interprets data in the column `columnName` as `decimal`. + +## Integer + +### Description + +An integer value. +Example: 3 + +### Example 1 + +```jayvee +block ExampleTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "columnName" oftype integer + ]; +} +``` + +A block of type `TableInterpreter` that + interprets data in the column `columnName` as `integer`. + +## Text + +### Description + +A text value. +Example: "Hello World" + +### Example 1 + +```jayvee +block ExampleTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "columnName" oftype text + ]; +} +``` + +A block of type `TableInterpreter` that + interprets data in the column `columnName` as `text`. diff --git a/apps/docs/versioned_docs/version-0.3.0/user/valuetypes/builtin-valuetypes.md.license b/apps/docs/versioned_docs/version-0.3.0/user/valuetypes/builtin-valuetypes.md.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/valuetypes/builtin-valuetypes.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/valuetypes/primitive-valuetypes.md b/apps/docs/versioned_docs/version-0.3.0/user/valuetypes/primitive-valuetypes.md new file mode 100644 index 000000000..92400a538 --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/valuetypes/primitive-valuetypes.md @@ -0,0 +1,48 @@ +--- +sidebar_position: 2 +--- +# Primitive ValueTypes + +`Primitive ValueTypes` are based on `Built-in ValueTypes` and use a collection of constraints to restrict the range of valid values. +Such constraints are implicitly connected via a logical `AND` relation. +Note that the `Constraints` need to be applicable to the base-type of the `ValueType` - indicated by the identifier after the keyword `oftype`: + +```jayvee +valuetype GasFillLevel oftype integer { + constraints: [ GasFillLevelRange ]; +} +``` + + +## Constraints + +`Constraints` for `ValueTypes` declare the validity criteria that each concrete value is checked against. + +### Syntax 1: Expression syntax + +The syntax of expression-based `Constraints` uses an expression that evaluates to `true` or `false` for the given `value`. The type of the values the expression is working in is indicated ofter the keyword `on`: + +```jayvee +constraint GasFillLevelRange on decimal: + value >= 0 and value <= 100; +``` + +Refer to the [Expression documentation](../expressions.md) for further reading on expressions. + + +### Syntax 2: Block-like syntax + +The syntax of `Constraints` is similar to the syntax of `Blocks`. +The availability of property keys and their respective `ValueTypes` is determined by the type of the `Constraint` - indicated by the identifier after the keyword `oftype`: + +```jayvee +constraint GasFillLevelRange oftype RangeConstraint { + lowerBound: 0; + lowerBoundInclusive: true; + upperBound: 100; + upperBoundInclusive: true; +} +``` + +Note that the type of `Constraint` also determines its applicability to `ValueTypes`. +For instance, a `RangeConstraint` can only be applied to the numerical types `integer` and `decimal`. \ No newline at end of file diff --git a/apps/docs/versioned_docs/version-0.3.0/user/valuetypes/primitive-valuetypes.md.license b/apps/docs/versioned_docs/version-0.3.0/user/valuetypes/primitive-valuetypes.md.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/apps/docs/versioned_docs/version-0.3.0/user/valuetypes/primitive-valuetypes.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/apps/docs/versioned_sidebars/version-0.3.0-sidebars.json b/apps/docs/versioned_sidebars/version-0.3.0-sidebars.json new file mode 100644 index 000000000..217726ac9 --- /dev/null +++ b/apps/docs/versioned_sidebars/version-0.3.0-sidebars.json @@ -0,0 +1,14 @@ +{ + "userDocsSidebar": [ + { + "type": "autogenerated", + "dirName": "user" + } + ], + "devDocsSidebar": [ + { + "type": "autogenerated", + "dirName": "dev" + } + ] +} diff --git a/apps/docs/versioned_sidebars/version-0.3.0-sidebars.json.license b/apps/docs/versioned_sidebars/version-0.3.0-sidebars.json.license new file mode 100644 index 000000000..42737858e --- /dev/null +++ b/apps/docs/versioned_sidebars/version-0.3.0-sidebars.json.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only \ No newline at end of file diff --git a/apps/docs/versions.json b/apps/docs/versions.json index 0e5a62585..b66c2f147 100644 --- a/apps/docs/versions.json +++ b/apps/docs/versions.json @@ -1,4 +1,5 @@ [ + "0.3.0", "0.2.0", "0.1.0", "0.0.18", diff --git a/apps/interpreter/src/index.ts b/apps/interpreter/src/index.ts index 5a7487b91..b5520eca5 100644 --- a/apps/interpreter/src/index.ts +++ b/apps/interpreter/src/index.ts @@ -57,6 +57,11 @@ program `Sets the target blocks of the of block debug logging, separated by comma. If not given, all blocks are targeted.`, undefined, ) + .option( + '-po, --parse-only', + 'Only parses the model without running it. Exits with 0 if the model is valid, with 1 otherwise.', + false, + ) .description('Run a Jayvee file') .action(runAction); diff --git a/apps/interpreter/src/parse-only.spec.ts b/apps/interpreter/src/parse-only.spec.ts new file mode 100644 index 000000000..eb5abe1b1 --- /dev/null +++ b/apps/interpreter/src/parse-only.spec.ts @@ -0,0 +1,86 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as fs from 'node:fs'; +import * as path from 'path'; +import * as process from 'process'; + +import { + clearBlockExecutorRegistry, + clearConstraintExecutorRegistry, +} from '@jvalue/jayvee-execution/test'; +import { + RunOptions, + interpretModel, + interpretString, +} from '@jvalue/jayvee-interpreter-lib'; + +import { runAction } from './run-action'; + +jest.mock('@jvalue/jayvee-interpreter-lib', () => { + const original: object = jest.requireActual('@jvalue/jayvee-interpreter-lib'); + return { + ...original, + interpretModel: jest.fn(), + interpretString: jest.fn(), + }; +}); + +describe('Parse Only', () => { + const pathToValidModel = path.resolve(__dirname, '../../../example/cars.jv'); + const pathToInvalidModel = path.resolve( + __dirname, + '../test/assets/broken-model.jv', + ); + + const defaultOptions: RunOptions = { + env: new Map(), + debug: false, + debugGranularity: 'minimal', + debugTarget: undefined, + }; + + afterEach(() => { + // Assert that model is not executed + expect(interpretString).not.toBeCalled(); + expect(interpretModel).not.toBeCalled(); + }); + + beforeEach(() => { + jest.clearAllMocks(); + jest.spyOn(process, 'exit').mockImplementation(() => { + throw new Error(); + }); + + // Reset jayvee specific stuff + clearBlockExecutorRegistry(); + clearConstraintExecutorRegistry(); + }); + + it('should exit with 0 on a valid option', async () => { + await expect( + runAction(pathToValidModel, { + ...defaultOptions, + parseOnly: true, + }), + ).rejects.toBeDefined(); + + expect(process.exit).toBeCalledTimes(1); + expect(process.exit).toHaveBeenCalledWith(0); + }); + + it('should exit with 1 on error', async () => { + expect(fs.existsSync(pathToInvalidModel)).toBe(true); + + await expect( + runAction(pathToInvalidModel, { + ...defaultOptions, + parseOnly: true, + }), + ).rejects.toBeDefined(); + + expect(process.exit).toBeCalledTimes(1); + expect(process.exit).toHaveBeenCalledWith(1); + }); +}); diff --git a/apps/interpreter/src/run-action.ts b/apps/interpreter/src/run-action.ts index a0bfa6faf..2d9274f49 100644 --- a/apps/interpreter/src/run-action.ts +++ b/apps/interpreter/src/run-action.ts @@ -2,11 +2,14 @@ // // SPDX-License-Identifier: AGPL-3.0-only +import * as process from 'process'; + import { LoggerFactory, RunOptions, extractAstNodeFromFile, interpretModel, + parseModel, } from '@jvalue/jayvee-interpreter-lib'; import { JayveeModel, JayveeServices } from '@jvalue/jayvee-language-server'; @@ -23,6 +26,11 @@ export async function runAction( services, loggerFactory.createLogger(), ); + if (options.parseOnly === true) { + const { model, services } = await parseModel(extractAstNodeFn, options); + const exitCode = model != null && services != null ? 0 : 1; + process.exit(exitCode); + } const exitCode = await interpretModel(extractAstNodeFn, options); process.exit(exitCode); } diff --git a/apps/interpreter/test/assets/broken-model.jv b/apps/interpreter/test/assets/broken-model.jv new file mode 100644 index 000000000..447eab7c2 --- /dev/null +++ b/apps/interpreter/test/assets/broken-model.jv @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: 2024 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline CarsPipeline { + // Try using a CoolCarsExtractor although we only have normal cars. + // This fill result in an error during parsing. + CoolCarsExtractor -> CarsTextFileInterpreter; + + CarsTextFileInterpreter + -> CarsCSVInterpreter + -> NameHeaderWriter + -> CarsTableInterpreter + -> CarsLoader; + + block CarsExtractor oftype HttpExtractor { + url: "https://gist.githubusercontent.com/noamross/e5d3e859aa0c794be10b/raw/b999fb4425b54c63cab088c0ce2c0d6ce961a563/cars.csv"; + } + + block CarsTextFileInterpreter oftype TextFileInterpreter { } + + block CarsCSVInterpreter oftype CSVInterpreter { + enclosing: '"'; + } + + block NameHeaderWriter oftype CellWriter { + at: cell A1; + write: ["name"]; + } + + block CarsTableInterpreter oftype TableInterpreter { + header: true; + columns: [ + "name" oftype text, + "mpg" oftype decimal, + "cyl" oftype integer, + "disp" oftype decimal, + "hp" oftype integer, + "drat" oftype decimal, + "wt" oftype decimal, + "qsec" oftype decimal, + "vs" oftype integer, + "am" oftype integer, + "gear" oftype integer, + "carb" oftype integer + ]; + } + + block CarsLoader oftype SQLiteLoader { + table: "Cars"; + file: "./cars.sqlite"; + } + +} \ No newline at end of file diff --git a/example/cars.jv b/example/cars.jv index 9ada5cb32..2100e65e7 100644 --- a/example/cars.jv +++ b/example/cars.jv @@ -16,64 +16,59 @@ pipeline CarsPipeline { // usually at the top of the pipeline. // by connecting blocks via pipes. - // 3. Verbose syntax of a pipe + // 3. Syntax of a pipe // connecting the block CarsExtractor // with the block CarsTextFileInterpreter. - pipe { - from: CarsExtractor; - to: CarsTextFileInterpreter; - } - - // 4. The output of the "from" block is hereby used - // as input for the "to" block. + CarsExtractor -> CarsTextFileInterpreter; - // 5. More convenient syntax of a pipe - CarsTextFileInterpreter -> CarsCSVInterpreter; + // 4. The output of the preceding block is hereby used + // as input for the succeeding block. - // 6. Pipes can be further chained, + // 5. Pipes can be further chained, // leading to an overview of the pipeline. - CarsCSVInterpreter + CarsTextFileInterpreter + -> CarsCSVInterpreter -> NameHeaderWriter -> CarsTableInterpreter -> CarsLoader; - // 7. Below the pipes, we usually define the blocks + // 6. Below the pipes, we usually define the blocks // that are connected by the pipes. - // 8. Blocks instantiate a blocktype by using the oftype keyword. + // 7. Blocks instantiate a blocktype by using the oftype keyword. // The blocktype defines the available properties that the block // can use to specify the intended behavior of the block block CarsExtractor oftype HttpExtractor { - // 9. Properties are assigned to concrete values. + // 8. Properties are assigned to concrete values. // Here, we specify the URL where the file shall be downloaded from. url: "https://gist.githubusercontent.com/noamross/e5d3e859aa0c794be10b/raw/b999fb4425b54c63cab088c0ce2c0d6ce961a563/cars.csv"; } - // 10. The HttpExtractor requires no input and produces a binary file as output. + // 9. The HttpExtractor requires no input and produces a binary file as output. // This file has to be interpreted, e.g., as text file. block CarsTextFileInterpreter oftype TextFileInterpreter { } - // 11. Next, we interpret the text file as sheet. + // 10. Next, we interpret the text file as sheet. // A sheet only contains text cells and is useful for manipulating the shape of data before assigning more strict value types to cells. block CarsCSVInterpreter oftype CSVInterpreter { enclosing: '"'; } - // 12. We can write into cells of a sheet using the CellWriter blocktype. + // 11. We can write into cells of a sheet using the CellWriter blocktype. block NameHeaderWriter oftype CellWriter { - // 13. We utilize a syntax similar to spreadsheet programs. + // 12. We utilize a syntax similar to spreadsheet programs. // Cell ranges can be described using the keywords "cell", "row", "column", or "range" that indicate which // cells are selected for the write action. at: cell A1; - // 14. For each cell we selected with the "at" property above, + // 13. For each cell we selected with the "at" property above, // we can specify what value shall be written into the cell. write: ["name"]; } - // 15. As a next step, we interpret the sheet as a table by adding structure. + // 14. As a next step, we interpret the sheet as a table by adding structure. // We define a valuetype per column that specifies the data type of the column. // Rows that include values that are not valid according to the their valuetypes are dropped automatically. block CarsTableInterpreter oftype TableInterpreter { @@ -94,7 +89,7 @@ pipeline CarsPipeline { ]; } - // 16. As a last step, we load the table into a sink, + // 15. As a last step, we load the table into a sink, // here into a sqlite file. // The structural information of the table is used // to generate the correct table. @@ -103,7 +98,7 @@ pipeline CarsPipeline { file: "./cars.sqlite"; } - // 17. Congratulations! + // 16. Congratulations! // You can now use the sink for your data analysis, app, // or whatever you want to do with the cleaned data. } \ No newline at end of file diff --git a/example/gtfs-static.jv b/example/gtfs-static.jv index e409abec1..7703ca1fe 100644 --- a/example/gtfs-static.jv +++ b/example/gtfs-static.jv @@ -14,79 +14,78 @@ pipeline GtfsPipeline { // 2. The origin for multiple pipe sequences is a zip // file. Each csv file in this zip is further processed // by its own sequence of blocks and pipes. - GTFSSampleFeedExtractor -> ZipArchiveInterpreter; - ZipArchiveInterpreter + GTFSSampleFeedExtractor -> AgencyFilePicker -> AgencyTextFileInterpreter -> AgencyCSVInterpreter -> AgencyTableInterpreter -> AgencyLoader; - ZipArchiveInterpreter + GTFSSampleFeedExtractor -> CalendarDatesFilePicker -> CalendarDatesTextFileInterpreter -> CalendarDatesCSVInterpreter -> CalendarDatesTableInterpreter -> CalendarDatesLoader; - ZipArchiveInterpreter + GTFSSampleFeedExtractor -> CalendarFilePicker -> CalendarTextFileInterpreter -> CalendarCSVInterpreter -> CalendarTableInterpreter -> CalendarLoader; - ZipArchiveInterpreter + GTFSSampleFeedExtractor -> FareAttributesFilePicker -> FareAttributesTextFileInterpreter -> FareAttributesCSVInterpreter -> FareAttributesTableInterpreter -> FareAttributesLoader; - ZipArchiveInterpreter + GTFSSampleFeedExtractor -> FareRulesFilePicker -> FareRulesTextFileInterpreter -> FareRulesCSVInterpreter -> FareRulesTableInterpreter -> FareRulesLoader; - ZipArchiveInterpreter + GTFSSampleFeedExtractor -> FrequenciesFilePicker -> FrequenciesTextFileInterpreter -> FrequenciesCSVInterpreter -> FrequenciesTableInterpreter -> FrequenciesLoader; - ZipArchiveInterpreter + GTFSSampleFeedExtractor -> RoutesFilePicker -> RoutesTextFileInterpreter -> RoutesCSVInterpreter -> RoutesTableInterpreter -> RoutesLoader; - ZipArchiveInterpreter + GTFSSampleFeedExtractor -> ShapesFilePicker -> ShapesTextFileInterpreter -> ShapesCSVInterpreter -> ShapesTableInterpreter -> ShapesLoader; - ZipArchiveInterpreter + GTFSSampleFeedExtractor -> StopTimesFilePicker -> StopTimesTextFileInterpreter -> StopTimesCSVInterpreter -> StopTimesTableInterpreter -> StopTimesLoader; - ZipArchiveInterpreter + GTFSSampleFeedExtractor -> StopsFilePicker -> StopsTextFileInterpreter -> StopsCSVInterpreter -> StopsTableInterpreter -> StopsLoader; - ZipArchiveInterpreter + GTFSSampleFeedExtractor -> TripsFilePicker -> TripsTextFileInterpreter -> TripsCSVInterpreter @@ -94,14 +93,10 @@ pipeline GtfsPipeline { -> TripsLoader; // 3. As a first step, we download the zip file and interpret it. - block GTFSSampleFeedExtractor oftype HttpExtractor { + block GTFSSampleFeedExtractor oftype GTFSExtractor { url: "https://developers.google.com/static/transit/gtfs/examples/sample-feed.zip"; } - block ZipArchiveInterpreter oftype ArchiveInterpreter { - archiveType: "zip"; - } - // 4. Next, we pick several csv files (with the file extension ".txt") // for further processing . block AgencyFilePicker oftype FilePicker { diff --git a/libs/execution/src/lib/blocks/block-execution-util.ts b/libs/execution/src/lib/blocks/block-execution-util.ts index ed17c157b..57e27972e 100644 --- a/libs/execution/src/lib/blocks/block-execution-util.ts +++ b/libs/execution/src/lib/blocks/block-execution-util.ts @@ -4,7 +4,9 @@ import { BlockDefinition, - collectParents, + CompositeBlocktypeDefinition, + PipelineDefinition, + PipelineWrapper, } from '@jvalue/jayvee-language-server'; import { ExecutionContext } from '../execution-context'; @@ -31,18 +33,27 @@ export interface ExecutionOrderItem { */ export async function executeBlocks( executionContext: ExecutionContext, - executionOrder: ExecutionOrderItem[], + pipesContainer: CompositeBlocktypeDefinition | PipelineDefinition, initialInputValue: IOTypeImplementation | undefined = undefined, ): Promise> { + const pipelineWrapper = new PipelineWrapper(pipesContainer); + const executionOrder: { + block: BlockDefinition; + value: IOTypeImplementation | null; + }[] = pipelineWrapper.getBlocksInTopologicalSorting().map((block) => { + return { block: block, value: NONE }; + }); + let isFirstBlock = true; for (const blockData of executionOrder) { const block = blockData.block; - const parentData = collectParents(block).map((parent) => - executionOrder.find((blockData) => parent === blockData.block), - ); - let inputValue = - parentData[0]?.value === undefined ? NONE : parentData[0]?.value; + const parentData = pipelineWrapper + .getParentBlocks(block) + .map((parent) => + executionOrder.find((blockData) => parent === blockData.block), + ); + let inputValue = parentData[0]?.value ?? NONE; const useExternalInputValueForFirstBlock = isFirstBlock && inputValue === NONE && initialInputValue !== undefined; diff --git a/libs/execution/src/lib/blocks/composite-block-executor.ts b/libs/execution/src/lib/blocks/composite-block-executor.ts index 8209eb015..f2fcf637c 100644 --- a/libs/execution/src/lib/blocks/composite-block-executor.ts +++ b/libs/execution/src/lib/blocks/composite-block-executor.ts @@ -16,13 +16,12 @@ import { createValuetype, evaluateExpression, evaluatePropertyValue, - getBlocksInTopologicalSorting, getIOType, isCompositeBlocktypeDefinition, } from '@jvalue/jayvee-language-server'; import { ExecutionContext } from '../execution-context'; -import { IOTypeImplementation, NONE } from '../types'; +import { IOTypeImplementation } from '../types'; // eslint-disable-next-line import/no-cycle import { executeBlocks } from './block-execution-util'; @@ -69,15 +68,9 @@ export function createCompositeBlockExecutor( this.addVariablesToContext(block, blockTypeReference.properties, context); - const executionOrder = getBlocksInTopologicalSorting( - blockTypeReference, - ).map((block) => { - return { block: block, value: NONE }; - }); - const executionResult = await executeBlocks( context, - executionOrder, + blockTypeReference, input, ); diff --git a/libs/execution/src/lib/types/valuetypes/internal-representation-parsing.ts b/libs/execution/src/lib/types/valuetypes/internal-representation-parsing.ts index 087b414a6..9220d51bb 100644 --- a/libs/execution/src/lib/types/valuetypes/internal-representation-parsing.ts +++ b/libs/execution/src/lib/types/valuetypes/internal-representation-parsing.ts @@ -11,10 +11,7 @@ import { ValuetypeVisitor, } from '@jvalue/jayvee-language-server'; -const DECIMAL_COMMA_SEPARATOR_REGEX = /^[+-]?([0-9]*[,])?[0-9]+$/; -const DECIMAL_DOT_SEPARATOR_REGEX = /^[+-]?([0-9]*[.])?[0-9]+$/; - -const INTEGER_REGEX = /^[+-]?[0-9]+$/; +const NUMBER_REGEX = /^[+-]?([0-9]*[,.])?[0-9]+([eE][+-]?\d+)?$/; const TRUE_REGEX = /^true$/i; const FALSE_REGEX = /^false$/i; @@ -47,24 +44,32 @@ class InternalRepresentationParserVisitor extends ValuetypeVisitor< } visitDecimal(): number | undefined { - let sanitizedValue: string; - if (DECIMAL_COMMA_SEPARATOR_REGEX.test(this.value)) { - sanitizedValue = this.value.replace(',', '.'); - } else if (DECIMAL_DOT_SEPARATOR_REGEX.test(this.value)) { - sanitizedValue = this.value; - } else { + if (!NUMBER_REGEX.test(this.value)) { return undefined; } - return Number.parseFloat(sanitizedValue); + return Number.parseFloat(this.value.replace(',', '.')); } visitInteger(): number | undefined { - if (!INTEGER_REGEX.test(this.value)) { + /** + * Reuse decimal number parsing to capture valid scientific notation + * of integers like 5.3e3 = 5300. In contrast to decimal, if the final number + * is not a valid integer, returns undefined. + */ + const decimalNumber = this.visitDecimal(); + + if (decimalNumber === undefined) { + return undefined; + } + + const integerNumber = Math.trunc(decimalNumber); + + if (decimalNumber !== integerNumber) { return undefined; } - return Number.parseInt(this.value, 10); + return integerNumber; } visitText(): string { diff --git a/libs/extensions/std/exec/src/file-util.spec.ts b/libs/execution/src/lib/util/file-util.spec.ts similarity index 97% rename from libs/extensions/std/exec/src/file-util.spec.ts rename to libs/execution/src/lib/util/file-util.spec.ts index decef2317..5d3074157 100644 --- a/libs/extensions/std/exec/src/file-util.spec.ts +++ b/libs/execution/src/lib/util/file-util.spec.ts @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: AGPL-3.0-only -import { FileExtension, MimeType } from '@jvalue/jayvee-execution'; +import { FileExtension, MimeType } from '../types'; import { inferFileExtensionFromContentTypeString, diff --git a/libs/extensions/std/exec/src/file-util.ts b/libs/execution/src/lib/util/file-util.ts similarity index 95% rename from libs/extensions/std/exec/src/file-util.ts rename to libs/execution/src/lib/util/file-util.ts index ff828156d..bab374acf 100644 --- a/libs/extensions/std/exec/src/file-util.ts +++ b/libs/execution/src/lib/util/file-util.ts @@ -2,9 +2,10 @@ // // SPDX-License-Identifier: AGPL-3.0-only -import { FileExtension, MimeType } from '@jvalue/jayvee-execution'; import * as mime from 'mime-types'; +import { FileExtension, MimeType } from '../types'; + export function inferMimeTypeFromFileExtensionString( fileExtension: string | undefined, ): MimeType | undefined { diff --git a/libs/execution/src/lib/util/index.ts b/libs/execution/src/lib/util/index.ts index 28697b9ea..9e67d5d9d 100644 --- a/libs/execution/src/lib/util/index.ts +++ b/libs/execution/src/lib/util/index.ts @@ -3,3 +3,5 @@ // SPDX-License-Identifier: AGPL-3.0-only export * from './implements-static-decorator'; +export * from './file-util'; +export * from './string-util'; diff --git a/libs/extensions/std/exec/src/string-util.ts b/libs/execution/src/lib/util/string-util.ts similarity index 100% rename from libs/extensions/std/exec/src/string-util.ts rename to libs/execution/src/lib/util/string-util.ts diff --git a/libs/extensions/std/exec/test/utils.ts b/libs/execution/test/utils/file-util.ts similarity index 91% rename from libs/extensions/std/exec/test/utils.ts rename to libs/execution/test/utils/file-util.ts index 7b7d43d5b..be8689515 100644 --- a/libs/extensions/std/exec/test/utils.ts +++ b/libs/execution/test/utils/file-util.ts @@ -10,13 +10,10 @@ import { FileExtension, MimeType, TextFile, -} from '@jvalue/jayvee-execution'; - -import { inferFileExtensionFromFileExtensionString, inferMimeTypeFromFileExtensionString, -} from '../src/file-util'; -import { splitLines } from '../src/string-util'; + splitLines, +} from '../../src'; export function createBinaryFileFromLocalFile(fileName: string): BinaryFile { const extName = path.extname(fileName); diff --git a/libs/language-server/src/test/assets/pipe-definition/single/valid-undefined-block.jv b/libs/execution/test/utils/index.ts similarity index 61% rename from libs/language-server/src/test/assets/pipe-definition/single/valid-undefined-block.jv rename to libs/execution/test/utils/index.ts index 56bf4b50d..144030b35 100644 --- a/libs/language-server/src/test/assets/pipe-definition/single/valid-undefined-block.jv +++ b/libs/execution/test/utils/index.ts @@ -2,9 +2,5 @@ // // SPDX-License-Identifier: AGPL-3.0-only -pipeline Pipeline { - pipe { - from: TestExtractor; - to: TestLoader; - } -} +export * from './test-infrastructure-util'; +export * from './file-util'; diff --git a/libs/execution/test/utils.ts b/libs/execution/test/utils/test-infrastructure-util.ts similarity index 99% rename from libs/execution/test/utils.ts rename to libs/execution/test/utils/test-infrastructure-util.ts index 671f10873..1565bf602 100644 --- a/libs/execution/test/utils.ts +++ b/libs/execution/test/utils/test-infrastructure-util.ts @@ -19,7 +19,7 @@ import { TableColumn, blockExecutorRegistry, constraintExecutorRegistry, -} from '../src'; +} from '../../src'; export function clearBlockExecutorRegistry() { blockExecutorRegistry.clear(); diff --git a/libs/extensions/rdbms/exec/src/lib/postgres-loader-executor.spec.ts b/libs/extensions/rdbms/exec/src/lib/postgres-loader-executor.spec.ts new file mode 100644 index 000000000..2b7c9a0c6 --- /dev/null +++ b/libs/extensions/rdbms/exec/src/lib/postgres-loader-executor.spec.ts @@ -0,0 +1,174 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { + constructTable, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + PrimitiveValuetypes, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { PostgresLoaderExecutor } from './postgres-loader-executor'; + +// eslint-disable-next-line no-var +var databaseConnectMock: jest.Mock; +// eslint-disable-next-line no-var +var databaseQueryMock: jest.Mock; +// eslint-disable-next-line no-var +var databaseEndMock: jest.Mock; +jest.mock('pg', () => { + databaseConnectMock = jest.fn(); + databaseQueryMock = jest.fn(); + databaseEndMock = jest.fn(); + const mClient = { + connect: databaseConnectMock, + query: databaseQueryMock, + end: databaseEndMock, + }; + return { Client: jest.fn(() => mClient) }; +}); + +describe('Validation of PostgresLoaderExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/postgres-loader-executor/', + ); + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.Table, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new PostgresLoaderExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should diagnose no error on valid loader config', async () => { + const text = readJvTestAsset('valid-postgres-loader.jv'); + + const inputTable = constructTable( + [ + { + columnName: 'Column1', + column: { + values: ['value 1'], + valuetype: PrimitiveValuetypes.Text, + }, + }, + { + columnName: 'Column2', + column: { + values: [20.2], + valuetype: PrimitiveValuetypes.Decimal, + }, + }, + ], + 1, + ); + const result = await parseAndExecuteExecutor(text, inputTable); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.NONE); + expect(databaseConnectMock).toBeCalledTimes(1); + expect(databaseQueryMock).nthCalledWith( + 1, + 'DROP TABLE IF EXISTS "Test";', + ); + expect(databaseQueryMock).nthCalledWith( + 2, + `CREATE TABLE IF NOT EXISTS "Test" ("Column1" text,"Column2" real);`, + ); + expect(databaseQueryMock).nthCalledWith( + 3, + `INSERT INTO "Test" ("Column1","Column2") VALUES ('value 1',20.2)`, + ); + expect(databaseEndMock).toBeCalledTimes(1); + } + }); + + it('should diagnose error on pg client connect error', async () => { + const text = readJvTestAsset('valid-postgres-loader.jv'); + + const inputTable = constructTable( + [ + { + columnName: 'Column1', + column: { + values: ['value 1'], + valuetype: PrimitiveValuetypes.Text, + }, + }, + { + columnName: 'Column2', + column: { + values: [20.2], + valuetype: PrimitiveValuetypes.Decimal, + }, + }, + ], + 1, + ); + databaseConnectMock.mockImplementation(() => { + throw new Error('Connection error'); + }); + const result = await parseAndExecuteExecutor(text, inputTable); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'Could not write to postgres database: Connection error', + ); + expect(databaseConnectMock).toBeCalledTimes(1); + expect(databaseQueryMock).toBeCalledTimes(0); + expect(databaseEndMock).toBeCalledTimes(1); + } + }); +}); diff --git a/libs/extensions/rdbms/exec/src/lib/sqlite-loader-executor.spec.ts b/libs/extensions/rdbms/exec/src/lib/sqlite-loader-executor.spec.ts new file mode 100644 index 000000000..26613a35c --- /dev/null +++ b/libs/extensions/rdbms/exec/src/lib/sqlite-loader-executor.spec.ts @@ -0,0 +1,200 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { + constructTable, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + PrimitiveValuetypes, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; +import * as sqlite3 from 'sqlite3'; + +import { SQLiteLoaderExecutor } from './sqlite-loader-executor'; + +type SqliteRunCallbackType = ( + result: sqlite3.RunResult, + err: Error | null, +) => void; +// eslint-disable-next-line no-var +var databaseMock: jest.Mock; +// eslint-disable-next-line no-var +var databaseRunMock: jest.Mock; +// eslint-disable-next-line no-var +var databaseCloseMock: jest.Mock; +jest.mock('sqlite3', () => { + databaseMock = jest.fn(); + databaseRunMock = jest.fn(); + databaseCloseMock = jest.fn(); + return { + Database: databaseMock, + }; +}); +function mockDatabaseDefault() { + const mockDB = { + close: databaseCloseMock, + run: databaseRunMock, + }; + databaseMock.mockImplementation(() => { + return mockDB; + }); +} + +describe('Validation of SQLiteLoaderExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/sqlite-loader-executor/', + ); + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.Table, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new SQLiteLoaderExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should diagnose no error on valid loader config', async () => { + mockDatabaseDefault(); + databaseRunMock.mockImplementation( + (sql: string, callback: SqliteRunCallbackType) => { + callback( + { + lastID: 0, + changes: 0, + } as sqlite3.RunResult, + null, + ); + return this; + }, + ); + const text = readJvTestAsset('valid-sqlite-loader.jv'); + + const inputTable = constructTable( + [ + { + columnName: 'Column1', + column: { + values: ['value 1'], + valuetype: PrimitiveValuetypes.Text, + }, + }, + { + columnName: 'Column2', + column: { + values: [20.2], + valuetype: PrimitiveValuetypes.Decimal, + }, + }, + ], + 1, + ); + const result = await parseAndExecuteExecutor(text, inputTable); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.NONE); + expect(databaseRunMock).toBeCalledTimes(3); + expect(databaseRunMock).nthCalledWith( + 1, + 'DROP TABLE IF EXISTS "Test";', + expect.any(Function), + ); + expect(databaseRunMock).nthCalledWith( + 2, + `CREATE TABLE IF NOT EXISTS "Test" ("Column1" text,"Column2" real);`, + expect.any(Function), + ); + expect(databaseRunMock).nthCalledWith( + 3, + `INSERT INTO "Test" ("Column1","Column2") VALUES ('value 1',20.2)`, + expect.any(Function), + ); + expect(databaseCloseMock).toBeCalledTimes(1); + } + }); + + it('should diagnose error on sqlite database open error', async () => { + databaseMock.mockImplementation(() => { + throw new Error('File not found'); + }); + const text = readJvTestAsset('valid-sqlite-loader.jv'); + + const inputTable = constructTable( + [ + { + columnName: 'Column1', + column: { + values: ['value 1'], + valuetype: PrimitiveValuetypes.Text, + }, + }, + { + columnName: 'Column2', + column: { + values: [20.2], + valuetype: PrimitiveValuetypes.Decimal, + }, + }, + ], + 1, + ); + const result = await parseAndExecuteExecutor(text, inputTable); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'Could not write to sqlite database: File not found', + ); + expect(databaseRunMock).toBeCalledTimes(0); + expect(databaseCloseMock).toBeCalledTimes(0); + } + }); +}); diff --git a/libs/extensions/rdbms/exec/test/assets/postgres-loader-executor/valid-postgres-loader.jv b/libs/extensions/rdbms/exec/test/assets/postgres-loader-executor/valid-postgres-loader.jv new file mode 100644 index 000000000..93b769bc9 --- /dev/null +++ b/libs/extensions/rdbms/exec/test/assets/postgres-loader-executor/valid-postgres-loader.jv @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestFileExtractor { + } + + block TestBlock oftype PostgresLoader { + host: "localhost"; + port: 5432; + username: "postgres"; + password: "postgres"; + database: "TestDB"; + table: "Test"; + } + + TestExtractor -> TestBlock; +} diff --git a/libs/extensions/rdbms/exec/test/assets/sqlite-loader-executor/valid-sqlite-loader.jv b/libs/extensions/rdbms/exec/test/assets/sqlite-loader-executor/valid-sqlite-loader.jv new file mode 100644 index 000000000..3b9354b53 --- /dev/null +++ b/libs/extensions/rdbms/exec/test/assets/sqlite-loader-executor/valid-sqlite-loader.jv @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestFileExtractor { + } + + block TestBlock oftype SQLiteLoader { + table: "Test"; + file: "./test.db"; + } + + TestExtractor -> TestBlock; +} diff --git a/libs/extensions/rdbms/exec/test/mocks/postgres-loader-executor-mock.ts b/libs/extensions/rdbms/exec/test/mocks/postgres-loader-executor-mock.ts index 6aba1c095..3c7d0cd77 100644 --- a/libs/extensions/rdbms/exec/test/mocks/postgres-loader-executor-mock.ts +++ b/libs/extensions/rdbms/exec/test/mocks/postgres-loader-executor-mock.ts @@ -7,7 +7,7 @@ import * as assert from 'assert'; import { BlockExecutorMock } from '@jvalue/jayvee-execution/test'; import { Client } from 'pg'; -type MockedPgClient = jest.Mocked>; +type MockedPgClient = jest.Mocked; export class PostgresLoaderExecutorMock implements BlockExecutorMock { private _pgClient: MockedPgClient | undefined; @@ -26,7 +26,7 @@ export class PostgresLoaderExecutorMock implements BlockExecutorMock { ) => void = defaultPostgresMockRegistration, ) { // setup pg mock - this._pgClient = new Client(); + this._pgClient = new Client() as MockedPgClient; registerMocks(this._pgClient); } restore() { diff --git a/libs/extensions/rdbms/exec/test/mocks/sqlite-loader-executor-mock.ts b/libs/extensions/rdbms/exec/test/mocks/sqlite-loader-executor-mock.ts index 888cce28b..566d379a7 100644 --- a/libs/extensions/rdbms/exec/test/mocks/sqlite-loader-executor-mock.ts +++ b/libs/extensions/rdbms/exec/test/mocks/sqlite-loader-executor-mock.ts @@ -7,7 +7,7 @@ import * as assert from 'assert'; import { BlockExecutorMock } from '@jvalue/jayvee-execution/test'; import * as sqlite3 from 'sqlite3'; -type MockedSqlite3Database = jest.Mocked>; +type MockedSqlite3Database = jest.Mocked; export class SQLiteLoaderExecutorMock implements BlockExecutorMock { private _sqliteClient: MockedSqlite3Database | undefined; @@ -26,7 +26,7 @@ export class SQLiteLoaderExecutorMock implements BlockExecutorMock { ) => void = defaultSQLiteMockRegistration, ) { // setup sqlite3 mock - this._sqliteClient = new sqlite3.Database('test'); + this._sqliteClient = new sqlite3.Database('test') as MockedSqlite3Database; registerMocks(this._sqliteClient); } restore() { diff --git a/libs/extensions/rdbms/exec/test/test-extension/TestBlockTypes.jv b/libs/extensions/rdbms/exec/test/test-extension/TestBlockTypes.jv new file mode 100644 index 000000000..e050777aa --- /dev/null +++ b/libs/extensions/rdbms/exec/test/test-extension/TestBlockTypes.jv @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +builtin blocktype TestTableExtractor { + input inPort oftype None; + output outPort oftype Table; +} diff --git a/libs/extensions/std/exec/src/archive-interpreter-executor.spec.ts b/libs/extensions/std/exec/src/archive-interpreter-executor.spec.ts index 7b174ae11..ae4eb2fcd 100644 --- a/libs/extensions/std/exec/src/archive-interpreter-executor.spec.ts +++ b/libs/extensions/std/exec/src/archive-interpreter-executor.spec.ts @@ -5,7 +5,10 @@ import * as path from 'path'; import * as R from '@jvalue/jayvee-execution'; -import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + createBinaryFileFromLocalFile, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; import { BlockDefinition, IOType, @@ -21,8 +24,6 @@ import { import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; import { NodeFileSystem } from 'langium/node'; -import { createBinaryFileFromLocalFile } from '../test'; - import { ArchiveInterpreterExecutor } from './archive-interpreter-executor'; describe('Validation of ArchiveInterpreterExecutor', () => { diff --git a/libs/extensions/std/exec/src/archive-interpreter-executor.ts b/libs/extensions/std/exec/src/archive-interpreter-executor.ts index ccd07d491..0fa207bed 100644 --- a/libs/extensions/std/exec/src/archive-interpreter-executor.ts +++ b/libs/extensions/std/exec/src/archive-interpreter-executor.ts @@ -18,15 +18,12 @@ import { MimeType, err, implementsStatic, + inferFileExtensionFromFileExtensionString, + inferMimeTypeFromFileExtensionString, } from '@jvalue/jayvee-execution'; import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server'; import * as JSZip from 'jszip'; -import { - inferFileExtensionFromFileExtensionString, - inferMimeTypeFromFileExtensionString, -} from './file-util'; - @implementsStatic() export class ArchiveInterpreterExecutor extends AbstractBlockExecutor< IOType.FILE, diff --git a/libs/extensions/std/exec/src/file-picker-executor.spec.ts b/libs/extensions/std/exec/src/file-picker-executor.spec.ts index 8fd8e3578..f89119a76 100644 --- a/libs/extensions/std/exec/src/file-picker-executor.spec.ts +++ b/libs/extensions/std/exec/src/file-picker-executor.spec.ts @@ -5,7 +5,10 @@ import * as path from 'path'; import * as R from '@jvalue/jayvee-execution'; -import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + createBinaryFileFromLocalFile, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; import { BlockDefinition, IOType, @@ -21,8 +24,6 @@ import { import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; import { NodeFileSystem } from 'langium/node'; -import { createBinaryFileFromLocalFile } from '../test'; - import { FilePickerExecutor } from './file-picker-executor'; describe('Validation of FilePickerExecutor', () => { diff --git a/libs/extensions/std/exec/src/gtfs-rt-interpreter-executor.spec.ts b/libs/extensions/std/exec/src/gtfs-rt-interpreter-executor.spec.ts index 75d8ae8d9..65cdcfb86 100644 --- a/libs/extensions/std/exec/src/gtfs-rt-interpreter-executor.spec.ts +++ b/libs/extensions/std/exec/src/gtfs-rt-interpreter-executor.spec.ts @@ -5,7 +5,10 @@ import * as path from 'path'; import * as R from '@jvalue/jayvee-execution'; -import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + createBinaryFileFromLocalFile, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; import { BlockDefinition, IOType, @@ -21,8 +24,6 @@ import { import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; import { NodeFileSystem } from 'langium/node'; -import { createBinaryFileFromLocalFile } from '../test'; - import { GtfsRTInterpreterExecutor } from './gtfs-rt-interpreter-executor'; describe('Validation of GtfsRTInterpreterExecutor', () => { diff --git a/libs/extensions/std/exec/src/http-extractor-executor.ts b/libs/extensions/std/exec/src/http-extractor-executor.ts index c9f052e85..61257b042 100644 --- a/libs/extensions/std/exec/src/http-extractor-executor.ts +++ b/libs/extensions/std/exec/src/http-extractor-executor.ts @@ -16,16 +16,14 @@ import { MimeType, None, implementsStatic, + inferFileExtensionFromContentTypeString, + inferFileExtensionFromFileExtensionString, + inferMimeTypeFromFileExtensionString, } from '@jvalue/jayvee-execution'; import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server'; import { http, https } from 'follow-redirects'; import { AstNode } from 'langium'; -import { - inferFileExtensionFromContentTypeString, - inferFileExtensionFromFileExtensionString, - inferMimeTypeFromFileExtensionString, -} from './file-util'; import { createBackoffStrategy, isBackoffStrategyHandle, diff --git a/libs/extensions/std/exec/src/text-file-interpreter-executor.spec.ts b/libs/extensions/std/exec/src/text-file-interpreter-executor.spec.ts index 342bceeab..0c9fd6462 100644 --- a/libs/extensions/std/exec/src/text-file-interpreter-executor.spec.ts +++ b/libs/extensions/std/exec/src/text-file-interpreter-executor.spec.ts @@ -5,7 +5,10 @@ import * as path from 'path'; import * as R from '@jvalue/jayvee-execution'; -import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + createBinaryFileFromLocalFile, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; import { BlockDefinition, IOType, @@ -21,8 +24,6 @@ import { import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; import { NodeFileSystem } from 'langium/node'; -import { createBinaryFileFromLocalFile } from '../test'; - import { TextFileInterpreterExecutor } from './text-file-interpreter-executor'; describe('Validation of TextFileInterpreterExecutor', () => { diff --git a/libs/extensions/std/exec/src/text-file-interpreter-executor.ts b/libs/extensions/std/exec/src/text-file-interpreter-executor.ts index c265aae3a..e3fb113e2 100644 --- a/libs/extensions/std/exec/src/text-file-interpreter-executor.ts +++ b/libs/extensions/std/exec/src/text-file-interpreter-executor.ts @@ -12,11 +12,10 @@ import { ExecutionContext, TextFile, implementsStatic, + splitLines, } from '@jvalue/jayvee-execution'; import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server'; -import { splitLines } from './string-util'; - @implementsStatic() export class TextFileInterpreterExecutor extends AbstractBlockExecutor< IOType.FILE, diff --git a/libs/extensions/std/exec/src/text-line-deleter-executor.spec.ts b/libs/extensions/std/exec/src/text-line-deleter-executor.spec.ts index 8ca8a191e..156151fce 100644 --- a/libs/extensions/std/exec/src/text-line-deleter-executor.spec.ts +++ b/libs/extensions/std/exec/src/text-line-deleter-executor.spec.ts @@ -5,7 +5,10 @@ import * as path from 'path'; import * as R from '@jvalue/jayvee-execution'; -import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + createTextFileFromLocalFile, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; import { BlockDefinition, IOType, @@ -21,8 +24,6 @@ import { import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; import { NodeFileSystem } from 'langium/node'; -import { createTextFileFromLocalFile } from '../test'; - import { TextLineDeleterExecutor } from './text-line-deleter-executor'; describe('Validation of TextLineDeleterExecutor', () => { diff --git a/libs/extensions/std/exec/src/text-range-selector-executor.spec.ts b/libs/extensions/std/exec/src/text-range-selector-executor.spec.ts index 73e333359..b9ad72c15 100644 --- a/libs/extensions/std/exec/src/text-range-selector-executor.spec.ts +++ b/libs/extensions/std/exec/src/text-range-selector-executor.spec.ts @@ -5,7 +5,10 @@ import * as path from 'path'; import * as R from '@jvalue/jayvee-execution'; -import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + createTextFileFromLocalFile, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; import { BlockDefinition, IOType, @@ -21,8 +24,6 @@ import { import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; import { NodeFileSystem } from 'langium/node'; -import { createTextFileFromLocalFile } from '../test'; - import { TextRangeSelectorExecutor } from './text-range-selector-executor'; describe('Validation of TextRangeSelectorExecutor', () => { diff --git a/libs/extensions/std/exec/test/index.ts b/libs/extensions/std/exec/test/index.ts index cfe12da6f..460aca6a3 100644 --- a/libs/extensions/std/exec/test/index.ts +++ b/libs/extensions/std/exec/test/index.ts @@ -3,4 +3,3 @@ // SPDX-License-Identifier: AGPL-3.0-only export * from './mocks'; -export * from './utils'; diff --git a/libs/extensions/tabular/exec/jest.config.ts b/libs/extensions/tabular/exec/jest.config.ts index e593333a4..a5a6a3c7d 100644 --- a/libs/extensions/tabular/exec/jest.config.ts +++ b/libs/extensions/tabular/exec/jest.config.ts @@ -5,6 +5,7 @@ export default { displayName: 'extensions-tabular-exec', preset: '../../../../jest.preset.js', + testEnvironment: './test/jsdom-environment-fix.ts', globals: { 'ts-jest': { tsconfig: '/tsconfig.spec.json', diff --git a/libs/extensions/tabular/exec/src/lib/cell-range-selector-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/cell-range-selector-executor.spec.ts new file mode 100644 index 000000000..5a16576a2 --- /dev/null +++ b/libs/extensions/tabular/exec/src/lib/cell-range-selector-executor.spec.ts @@ -0,0 +1,176 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { createWorkbookFromLocalExcelFile } from '../../test/util'; + +import { CellRangeSelectorExecutor } from './cell-range-selector-executor'; + +describe('Validation of CellRangeSelectorExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/cell-range-selector-executor/', + ); + + async function readTestWorkbook(fileName: string): Promise { + const absoluteFileName = path.resolve( + __dirname, + '../../test/assets/cell-range-selector-executor/', + fileName, + ); + return await createWorkbookFromLocalExcelFile(absoluteFileName); + } + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.Sheet, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new CellRangeSelectorExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + it('should diagnose no error on valid selector', async () => { + const text = readJvTestAsset('valid-A1-C.jv'); + + const testWorkbook = await readTestWorkbook('test-A1-C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getHeaderRow()).toEqual(['0', 'Test', 'true']); + expect(result.right.getData()).toEqual( + expect.arrayContaining([ + ['0', 'Test', 'true'], + ['1', 'Test', 'false'], + ['15', 'Test', 'true'], + ]), + ); + } + }); + + it('should diagnose no error on empty column', async () => { + const text = readJvTestAsset('valid-A1-C.jv'); + + const testWorkbook = await readTestWorkbook('test-B1-C2.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(2); + expect(result.right.getHeaderRow()).toEqual(['', 'Test', 'true']); + expect(result.right.getData()).toEqual([ + ['', 'Test', 'true'], + ['', 'Test', 'false'], + ]); + } + }); + + it('should diagnose error on selector out of bounds', async () => { + const text = readJvTestAsset('valid-A1-E4.jv'); + + const testWorkbook = await readTestWorkbook('test-A1-C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'The specified cell range does not fit the sheet', + ); + } + }); + + it('should diagnose error on selector on empty sheet', async () => { + const text = readJvTestAsset('valid-A1-C.jv'); + + const testWorkbook = await readTestWorkbook('test-empty.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'The specified cell range does not fit the sheet', + ); + } + }); + + it('should diagnose error on single column selector on empty sheet', async () => { + const text = readJvTestAsset('valid-A1-A4.jv'); + + const testWorkbook = await readTestWorkbook('test-empty.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'The specified cell range does not fit the sheet', + ); + } + }); +}); diff --git a/libs/extensions/tabular/exec/src/lib/cell-writer-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/cell-writer-executor.spec.ts new file mode 100644 index 000000000..ee32c9b3b --- /dev/null +++ b/libs/extensions/tabular/exec/src/lib/cell-writer-executor.spec.ts @@ -0,0 +1,145 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { createWorkbookFromLocalExcelFile } from '../../test/util'; + +import { CellWriterExecutor } from './cell-writer-executor'; + +describe('Validation of CellWriterExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/cell-writer-executor/', + ); + + async function readTestWorkbook(fileName: string): Promise { + const absoluteFileName = path.resolve( + __dirname, + '../../test/assets/cell-writer-executor/', + fileName, + ); + return await createWorkbookFromLocalExcelFile(absoluteFileName); + } + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.Sheet, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new CellWriterExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + it('should diagnose no error on valid single cell writer', async () => { + const text = readJvTestAsset('valid-single-cell-writer.jv'); + + const testWorkbook = await readTestWorkbook('test-A1-C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getHeaderRow()).toEqual(['16', 'Test', 'true']); + expect(result.right.getData()).toEqual( + expect.arrayContaining([ + ['16', 'Test', 'true'], + ['1', 'Test', 'false'], + ['15', 'Test', 'true'], + ]), + ); + } + }); + + it('should diagnose no error on valid cell range writer', async () => { + const text = readJvTestAsset('valid-cell-range-writer.jv'); + + const testWorkbook = await readTestWorkbook('test-A1-C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getHeaderRow()).toEqual(['16', 'Test2', '']); + expect(result.right.getData()).toEqual( + expect.arrayContaining([ + ['16', 'Test2', ''], + ['1', 'Test', 'false'], + ['15', 'Test', 'true'], + ]), + ); + } + }); + + it('should diagnose error on single cell writer on empty sheet', async () => { + const text = readJvTestAsset('valid-single-cell-writer.jv'); + + const testWorkbook = await readTestWorkbook('test-empty.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'Some specified cells do not exist in the sheet', + ); + } + }); +}); diff --git a/libs/extensions/tabular/exec/src/lib/column-deleter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/column-deleter-executor.spec.ts new file mode 100644 index 000000000..d69c7fd2a --- /dev/null +++ b/libs/extensions/tabular/exec/src/lib/column-deleter-executor.spec.ts @@ -0,0 +1,166 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { createWorkbookFromLocalExcelFile } from '../../test/util'; + +import { ColumnDeleterExecutor } from './column-deleter-executor'; + +describe('Validation of ColumnDeleterExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/column-deleter-executor/', + ); + + async function readTestWorkbook(fileName: string): Promise { + const absoluteFileName = path.resolve( + __dirname, + '../../test/assets/column-deleter-executor/', + fileName, + ); + return await createWorkbookFromLocalExcelFile(absoluteFileName); + } + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.Sheet, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new ColumnDeleterExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + it('should diagnose no error on valid single column deleter', async () => { + const text = readJvTestAsset('valid-single-column-deleter.jv'); + + const testWorkbook = await readTestWorkbook('test-A1-C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(2); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getHeaderRow()).toEqual(['Test', 'true']); + expect(result.right.getData()).toEqual( + expect.arrayContaining([ + ['Test', 'true'], + ['Test', 'false'], + ['Test', 'true'], + ]), + ); + } + }); + + it('should diagnose no error on valid multiple column deleter', async () => { + const text = readJvTestAsset('valid-multiple-column-deleter.jv'); + + const testWorkbook = await readTestWorkbook('test-A1-C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(1); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getHeaderRow()).toEqual(['Test']); + expect(result.right.getData()).toEqual( + expect.arrayContaining([['Test'], ['Test'], ['Test']]), + ); + } + }); + + it('should diagnose error on deleting non existing column', async () => { + const text = readJvTestAsset('valid-multiple-column-deleter.jv'); + + const testWorkbook = await readTestWorkbook('test-A1-B2.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'The specified column C does not exist in the sheet', + ); + } + }); + + it('should diagnose only one column deletion on duplicate columns', async () => { + const text = readJvTestAsset('valid-duplicate-column.jv'); + + const testWorkbook = await readTestWorkbook('test-A1-C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(2); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getHeaderRow()).toEqual(['Test', 'true']); + expect(result.right.getData()).toEqual( + expect.arrayContaining([ + ['Test', 'true'], + ['Test', 'false'], + ['Test', 'true'], + ]), + ); + } + }); +}); diff --git a/libs/extensions/tabular/exec/src/lib/csv-interpreter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/csv-interpreter-executor.spec.ts new file mode 100644 index 000000000..c4a8b938e --- /dev/null +++ b/libs/extensions/tabular/exec/src/lib/csv-interpreter-executor.spec.ts @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { + createTextFileFromLocalFile, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { CSVInterpreterExecutor } from './csv-interpreter-executor'; + +describe('Validation of CSVInterpreterExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/csv-interpreter-executor/', + ); + + function readTestFile(fileName: string): R.TextFile { + const absoluteFileName = path.resolve( + __dirname, + '../../test/assets/csv-interpreter-executor/', + fileName, + ); + return createTextFileFromLocalFile(absoluteFileName); + } + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.TextFile, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new CSVInterpreterExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + it('should diagnose no error on valid csv file', async () => { + const text = readJvTestAsset('valid-csv-interpreter.jv'); + + const testCsv = readTestFile('valid-csv.csv'); + const result = await parseAndExecuteExecutor(text, testCsv); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(2); + expect(result.right.getNumberOfRows()).toEqual(2); + expect(result.right.getData()).toEqual([ + ['Test', 'true'], + ['Test', 'false'], + ]); + } + }); +}); diff --git a/libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts new file mode 100644 index 000000000..e77c22fb1 --- /dev/null +++ b/libs/extensions/tabular/exec/src/lib/row-deleter-executor.spec.ts @@ -0,0 +1,161 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { createWorkbookFromLocalExcelFile } from '../../test/util'; + +import { RowDeleterExecutor } from './row-deleter-executor'; + +describe('Validation of RowDeleterExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/row-deleter-executor/', + ); + + async function readTestWorkbook(fileName: string): Promise { + const absoluteFileName = path.resolve( + __dirname, + '../../test/assets/row-deleter-executor/', + fileName, + ); + return await createWorkbookFromLocalExcelFile(absoluteFileName); + } + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.Sheet, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new RowDeleterExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + it('should diagnose no error on valid single row deleter', async () => { + const text = readJvTestAsset('valid-single-row-deleter.jv'); + + const testWorkbook = await readTestWorkbook('test-A1-C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(15); + expect(result.right.getHeaderRow()).toEqual(['1', 'Test', 'false']); + expect(result.right.getData()).not.toEqual( + expect.arrayContaining([['0', 'Test', 'true']]), + ); + } + }); + + it('should diagnose no error on valid multiple row deleter', async () => { + const text = readJvTestAsset('valid-multiple-row-deleter.jv'); + + const testWorkbook = await readTestWorkbook('test-A1-C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(14); + expect(result.right.getHeaderRow()).toEqual(['1', 'Test', 'false']); + expect(result.right.getData()).not.toEqual( + expect.arrayContaining([ + ['0', 'Test', 'true'], + ['4', 'Test', 'true'], + ]), + ); + } + }); + + it('should diagnose error on deleting non existing row', async () => { + const text = readJvTestAsset('valid-multiple-row-deleter.jv'); + + const testWorkbook = await readTestWorkbook('test-A1-B2.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'The specified row 5 does not exist in the sheet', + ); + } + }); + + it('should diagnose only one row deletion on duplicate rows', async () => { + const text = readJvTestAsset('valid-duplicate-row.jv'); + + const testWorkbook = await readTestWorkbook('test-A1-C16.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(15); + expect(result.right.getHeaderRow()).toEqual(['1', 'Test', 'false']); + expect(result.right.getData()).not.toEqual( + expect.arrayContaining([['0', 'Test', 'true']]), + ); + } + }); +}); diff --git a/libs/extensions/tabular/exec/src/lib/sheet-picker-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/sheet-picker-executor.spec.ts new file mode 100644 index 000000000..ff94ce0aa --- /dev/null +++ b/libs/extensions/tabular/exec/src/lib/sheet-picker-executor.spec.ts @@ -0,0 +1,114 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { createWorkbookFromLocalExcelFile } from '../../test/util'; + +import { SheetPickerExecutor } from './sheet-picker-executor'; + +describe('Validation of SheetPickerExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/sheet-picker-executor/', + ); + + async function readTestWorkbook(fileName: string): Promise { + const absoluteFileName = path.resolve( + __dirname, + '../../test/assets/sheet-picker-executor/', + fileName, + ); + return await createWorkbookFromLocalExcelFile(absoluteFileName); + } + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.Workbook, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new SheetPickerExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + it('should diagnose no error on valid workbook', async () => { + const text = readJvTestAsset('valid-sheet-picker.jv'); + + const testWorkbook = await readTestWorkbook('test-A1-C16.xlsx'); + const result = await parseAndExecuteExecutor(text, testWorkbook); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.SHEET); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getHeaderRow()).toEqual(['0', 'Test', 'true']); + expect(result.right.getData()).toEqual( + expect.arrayContaining([ + ['0', 'Test', 'true'], + ['1', 'Test', 'false'], + ['15', 'Test', 'true'], + ]), + ); + } + }); + + it('should diagnose error on sheet not found', async () => { + const text = readJvTestAsset('valid-custom-sheet-name.jv'); + + const testWorkbook = await readTestWorkbook('test-A1-C16.xlsx'); + const result = await parseAndExecuteExecutor(text, testWorkbook); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'Workbook does not contain a sheet named MyCustomSheet', + ); + } + }); +}); diff --git a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.spec.ts new file mode 100644 index 000000000..feb52fac5 --- /dev/null +++ b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.spec.ts @@ -0,0 +1,297 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { createWorkbookFromLocalExcelFile } from '../../test/util'; + +import { TableInterpreterExecutor } from './table-interpreter-executor'; + +describe('Validation of TableInterpreterExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/table-interpreter-executor/', + ); + + async function readTestWorkbook(fileName: string): Promise { + const absoluteFileName = path.resolve( + __dirname, + '../../test/assets/table-interpreter-executor/', + fileName, + ); + return await createWorkbookFromLocalExcelFile(absoluteFileName); + } + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.Sheet, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new TableInterpreterExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + describe('validation of sheet with header', () => { + it('should diagnose no error on valid sheet', async () => { + const text = readJvTestAsset('valid-with-header.jv'); + + const testWorkbook = await readTestWorkbook('test-with-header.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getColumn('index')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining([0, 1, 2, 15]) as number[], + }), + ); + expect(result.right.getColumn('name')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining(['Test']) as string[], + }), + ); + expect(result.right.getColumn('flag')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining([true, false]) as boolean[], + }), + ); + } + }); + + it('should diagnose empty table on empty column parameter', async () => { + const text = readJvTestAsset('valid-empty-columns-with-header.jv'); + + const testWorkbook = await readTestWorkbook('test-with-header.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(0); + expect(result.right.getNumberOfRows()).toEqual(0); + } + }); + + it('should diagnose empty table on wrong header case', async () => { + const text = readJvTestAsset('valid-with-capitalized-header.jv'); + + const testWorkbook = await readTestWorkbook('test-with-header.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(0); + expect(result.right.getNumberOfRows()).toEqual(0); + } + }); + + it('should diagnose error on empty sheet', async () => { + const text = readJvTestAsset('valid-with-header.jv'); + + const testWorkbook = await readTestWorkbook('test-empty.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'The input sheet is empty and thus has no header', + ); + } + }); + + it('should diagnose skipping row on wrong cell valuetype', async () => { + const text = readJvTestAsset('valid-wrong-valuetype-with-header.jv'); + + const testWorkbook = await readTestWorkbook('test-with-header.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(0); + } + }); + }); + + describe('validation of sheet without header', () => { + it('should diagnose no error on valid sheet', async () => { + const text = readJvTestAsset('valid-without-header.jv'); + + const testWorkbook = await readTestWorkbook('test-without-header.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getColumn('index')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining([0, 1, 2, 15]) as number[], + }), + ); + expect(result.right.getColumn('name')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining(['Test']) as string[], + }), + ); + expect(result.right.getColumn('flag')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining([true, false]) as boolean[], + }), + ); + } + }); + + it('should diagnose no error on valid sheet with header', async () => { + const text = readJvTestAsset('valid-without-header.jv'); + + const testWorkbook = await readTestWorkbook('test-with-header.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(16); + expect(result.right.getColumn('index')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining([0, 1, 2, 15]) as number[], + }), + ); + expect(result.right.getColumn('name')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining(['Test']) as string[], + }), + ); + expect(result.right.getColumn('flag')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining([true, false]) as boolean[], + }), + ); + } + }); + + it('should diagnose empty table on empty column parameter', async () => { + const text = readJvTestAsset('valid-empty-columns-without-header.jv'); + + const testWorkbook = await readTestWorkbook('test-without-header.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(0); + expect(result.right.getNumberOfRows()).toEqual(0); + } + }); + + it('should diagnose error on empty sheet', async () => { + const text = readJvTestAsset('valid-without-header.jv'); + + const testWorkbook = await readTestWorkbook('test-empty.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'There are 3 column definitions but the input sheet only has 0 columns', + ); + } + }); + + it('should diagnose skipping row on wrong cell valuetype', async () => { + const text = readJvTestAsset('valid-wrong-valuetype-without-header.jv'); + + const testWorkbook = await readTestWorkbook('test-without-header.xlsx'); + const result = await parseAndExecuteExecutor( + text, + testWorkbook.getSheetByName('Sheet1') as R.Sheet, + ); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(0); + } + }); + }); +}); diff --git a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts index fb7da129c..77a4df8db 100644 --- a/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts +++ b/libs/extensions/tabular/exec/src/lib/table-interpreter-executor.ts @@ -27,7 +27,7 @@ import { rowIndexToString, } from '@jvalue/jayvee-language-server'; -interface ColumnDefinitionEntry { +export interface ColumnDefinitionEntry { sheetColumnIndex: number; columnName: string; valuetype: Valuetype; diff --git a/libs/extensions/tabular/exec/src/lib/table-transformer-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/table-transformer-executor.spec.ts new file mode 100644 index 000000000..a2763b1ff --- /dev/null +++ b/libs/extensions/tabular/exec/src/lib/table-transformer-executor.spec.ts @@ -0,0 +1,220 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { getTestExecutionContext } from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + PrimitiveValuetypes, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { + ReducedColumnDefinitionEntry, + createTableFromLocalExcelFile, +} from '../../test/util'; + +import { TableTransformerExecutor } from './table-transformer-executor'; + +describe('Validation of TableTransformerExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/table-transformer-executor/', + ); + + async function readTestExcelAllColumns() { + return readTestTable('test-excel.xlsx', [ + { + columnName: 'index', + sheetColumnIndex: 0, + valuetype: PrimitiveValuetypes.Integer, + }, + { + columnName: 'name', + sheetColumnIndex: 1, + valuetype: PrimitiveValuetypes.Text, + }, + { + columnName: 'flag', + sheetColumnIndex: 2, + valuetype: PrimitiveValuetypes.Boolean, + }, + ]); + } + async function readTestTable( + fileName: string, + columnDefinitions: ReducedColumnDefinitionEntry[], + ): Promise { + const absoluteFileName = path.resolve( + __dirname, + '../../test/assets/table-transformer-executor/', + fileName, + ); + return await createTableFromLocalExcelFile( + absoluteFileName, + columnDefinitions, + ); + } + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.Table, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new TableTransformerExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + it('should diagnose no error on valid table transform', async () => { + const text = readJvTestAsset('valid-transfomer.jv'); + + const testTable = await readTestExcelAllColumns(); + const result = await parseAndExecuteExecutor(text, testTable); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(4); + expect(result.right.getNumberOfRows()).toEqual(6); + expect(result.right.getColumn('index')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining([0, 1, 2, 5]) as number[], + }), + ); + expect(result.right.getColumn('index2')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining([0, 2, 4, 10]) as number[], + }), + ); + } + }); + + it('should diagnose no error on column overwrite', async () => { + const text = readJvTestAsset('valid-column-overwrite.jv'); + + const testTable = await readTestExcelAllColumns(); + const result = await parseAndExecuteExecutor(text, testTable); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(6); + expect(result.right.getColumn('index')).toEqual( + expect.objectContaining({ + values: expect.arrayContaining([0, 2, 4, 10]) as number[], + }), + ); + } + }); + + it('should diagnose no error on column type change', async () => { + const text = readJvTestAsset('valid-column-type-change.jv'); + + const testTable = await readTestExcelAllColumns(); + const result = await parseAndExecuteExecutor(text, testTable); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(3); + expect(result.right.getNumberOfRows()).toEqual(6); + expect(result.right.getColumn('index')).toEqual( + expect.objectContaining({ + values: [false, true, true, true, true, true], + valuetype: PrimitiveValuetypes.Boolean, + }), + ); + } + }); + + it('should diagnose no error on empty table', async () => { + const text = readJvTestAsset('valid-transfomer.jv'); + + const testTable = await readTestTable('test-empty.xlsx', [ + { + columnName: 'index', + sheetColumnIndex: 0, + valuetype: PrimitiveValuetypes.Integer, + }, + ]); + const result = await parseAndExecuteExecutor(text, testTable); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.TABLE); + expect(result.right.getNumberOfColumns()).toEqual(2); + expect(result.right.getNumberOfRows()).toEqual(0); + expect(result.right.getColumn('index')?.values).toHaveLength(0); + expect(result.right.getColumn('index2')?.values).toHaveLength(0); + } + }); + + it('should diagnose error on missing input column', async () => { + const text = readJvTestAsset('valid-missing-input-column.jv'); + + const testTable = await readTestExcelAllColumns(); + const result = await parseAndExecuteExecutor(text, testTable); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'The specified input column "id" does not exist in the given table', + ); + } + }); + + it('should diagnose error on transform type missmatch', async () => { + const text = readJvTestAsset('valid-transform-type-missmatch.jv'); + + const testTable = await readTestExcelAllColumns(); + const result = await parseAndExecuteExecutor(text, testTable); + + expect(R.isOk(result)).toEqual(false); + if (R.isErr(result)) { + expect(result.left.message).toEqual( + 'Type text of column "name" is not convertible to type integer', + ); + } + }); +}); diff --git a/libs/extensions/tabular/exec/src/lib/xlsx-interpreter-executor.spec.ts b/libs/extensions/tabular/exec/src/lib/xlsx-interpreter-executor.spec.ts new file mode 100644 index 000000000..f289ea96d --- /dev/null +++ b/libs/extensions/tabular/exec/src/lib/xlsx-interpreter-executor.spec.ts @@ -0,0 +1,135 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import * as R from '@jvalue/jayvee-execution'; +import { + createBinaryFileFromLocalFile, + getTestExecutionContext, +} from '@jvalue/jayvee-execution/test'; +import { + BlockDefinition, + IOType, + createJayveeServices, +} from '@jvalue/jayvee-language-server'; +import { + ParseHelperOptions, + expectNoParserAndLexerErrors, + loadTestExtensions, + parseHelper, + readJvTestAssetHelper, +} from '@jvalue/jayvee-language-server/test'; +import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; +import { NodeFileSystem } from 'langium/node'; + +import { XLSXInterpreterExecutor } from './xlsx-interpreter-executor'; + +describe('Validation of XLSXInterpreterExecutor', () => { + let parse: ( + input: string, + options?: ParseHelperOptions, + ) => Promise>; + + let locator: AstNodeLocator; + + const readJvTestAsset = readJvTestAssetHelper( + __dirname, + '../../test/assets/xlsx-interpreter-executor/', + ); + + function readTestFile(fileName: string): R.BinaryFile { + const absoluteFileName = path.resolve( + __dirname, + '../../test/assets/xlsx-interpreter-executor/', + fileName, + ); + return createBinaryFileFromLocalFile(absoluteFileName); + } + + function expectSheetRowAndColumnSize( + sheet: R.Sheet | undefined, + cols: number, + rows: number, + ) { + expect(sheet?.getNumberOfColumns()).toEqual(cols); + expect(sheet?.getNumberOfRows()).toEqual(rows); + } + + async function parseAndExecuteExecutor( + input: string, + IOInput: R.BinaryFile, + ): Promise> { + const document = await parse(input, { validationChecks: 'all' }); + expectNoParserAndLexerErrors(document); + + const block = locator.getAstNode( + document.parseResult.value, + 'pipelines@0/blocks@1', + ) as BlockDefinition; + + return new XLSXInterpreterExecutor().doExecute( + IOInput, + getTestExecutionContext(locator, document, [block]), + ); + } + + beforeAll(async () => { + // Create language services + const services = createJayveeServices(NodeFileSystem).Jayvee; + await loadTestExtensions(services, [ + path.resolve(__dirname, '../../test/test-extension/TestBlockTypes.jv'), + ]); + locator = services.workspace.AstNodeLocator; + // Parse function for Jayvee (without validation) + parse = parseHelper(services); + }); + + it('should diagnose no error on single sheet excel', async () => { + const text = readJvTestAsset('valid-excel-interpreter.jv'); + + const testFile = readTestFile('test-excel.xlsx'); + const result = await parseAndExecuteExecutor(text, testFile); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.WORKBOOK); + const sheets = result.right.getSheets(); + expect(sheets.size).toEqual(1); + expectSheetRowAndColumnSize(sheets.get('Sheet1'), 3, 6); + } + }); + + it('should diagnose no error on multiple sheet excel', async () => { + const text = readJvTestAsset('valid-excel-interpreter.jv'); + + const testFile = readTestFile('test-multiple-sheets.xlsx'); + const result = await parseAndExecuteExecutor(text, testFile); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.WORKBOOK); + const sheets = result.right.getSheets(); + expect(sheets.size).toEqual(3); + expectSheetRowAndColumnSize(sheets.get('Sheet1'), 3, 6); + expectSheetRowAndColumnSize(sheets.get('CustomSheet'), 0, 0); + expectSheetRowAndColumnSize(sheets.get('Sheet3'), 0, 0); + } + }); + + it('should diagnose no error on empty excel', async () => { + const text = readJvTestAsset('valid-excel-interpreter.jv'); + + const testFile = readTestFile('test-empty.xlsx'); + const result = await parseAndExecuteExecutor(text, testFile); + + expect(R.isErr(result)).toEqual(false); + if (R.isOk(result)) { + expect(result.right.ioType).toEqual(IOType.WORKBOOK); + const sheets = result.right.getSheets(); + expect(sheets.size).toEqual(1); + expectSheetRowAndColumnSize(sheets.get('Sheet1'), 0, 0); + } + }); +}); diff --git a/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-A1-C16.xlsx b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-A1-C16.xlsx new file mode 100644 index 000000000..0544fa281 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-A1-C16.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-A1-C16.xlsx.license b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-A1-C16.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-A1-C16.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-B1-C2.xlsx b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-B1-C2.xlsx new file mode 100644 index 000000000..fdc4858c8 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-B1-C2.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-B1-C2.xlsx.license b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-B1-C2.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-B1-C2.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-empty.xlsx b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-empty.xlsx new file mode 100644 index 000000000..af7b184be Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-empty.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-empty.xlsx.license b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-empty.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/test-empty.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/valid-A1-A4.jv b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/valid-A1-A4.jv new file mode 100644 index 000000000..4fa94a1c9 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/valid-A1-A4.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype CellRangeSelector { + select: range A1:A4; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/valid-A1-C.jv b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/valid-A1-C.jv new file mode 100644 index 000000000..0ed56a14a --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/valid-A1-C.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype CellRangeSelector { + select: range A1:C*; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/valid-A1-E4.jv b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/valid-A1-E4.jv new file mode 100644 index 000000000..e004eb928 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-range-selector-executor/valid-A1-E4.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype CellRangeSelector { + select: range A1:E4; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-A1-C16.xlsx b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-A1-C16.xlsx new file mode 100644 index 000000000..0544fa281 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-A1-C16.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-A1-C16.xlsx.license b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-A1-C16.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-A1-C16.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-empty.xlsx b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-empty.xlsx new file mode 100644 index 000000000..af7b184be Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-empty.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-empty.xlsx.license b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-empty.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/test-empty.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/cell-writer-executor/valid-cell-range-writer.jv b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/valid-cell-range-writer.jv new file mode 100644 index 000000000..327d9dbaa --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/valid-cell-range-writer.jv @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype CellWriter { + write: ["16", "Test2", ""]; + at: range A1:C1; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/cell-writer-executor/valid-single-cell-writer.jv b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/valid-single-cell-writer.jv new file mode 100644 index 000000000..844a8fd88 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/cell-writer-executor/valid-single-cell-writer.jv @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype CellWriter { + write: ["16"]; + at: cell A1; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1-B2.xlsx b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1-B2.xlsx new file mode 100644 index 000000000..3465ea34d Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1-B2.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1-B2.xlsx.license b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1-B2.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1-B2.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1-C16.xlsx b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1-C16.xlsx new file mode 100644 index 000000000..0544fa281 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1-C16.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1-C16.xlsx.license b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1-C16.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/test-A1-C16.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/column-deleter-executor/valid-duplicate-column.jv b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/valid-duplicate-column.jv new file mode 100644 index 000000000..b3b6a6ae4 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/valid-duplicate-column.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype ColumnDeleter { + delete: [column A, column A]; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/column-deleter-executor/valid-multiple-column-deleter.jv b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/valid-multiple-column-deleter.jv new file mode 100644 index 000000000..6a2aa56d2 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/valid-multiple-column-deleter.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype ColumnDeleter { + delete: [column A, column C]; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/column-deleter-executor/valid-single-column-deleter.jv b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/valid-single-column-deleter.jv new file mode 100644 index 000000000..ad5f6733b --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/column-deleter-executor/valid-single-column-deleter.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype ColumnDeleter { + delete: [column A]; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/valid-csv-interpreter.jv b/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/valid-csv-interpreter.jv new file mode 100644 index 000000000..856c67eb0 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/valid-csv-interpreter.jv @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestTextFileExtractor { + } + + block TestBlock oftype CSVInterpreter { + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/valid-csv.csv b/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/valid-csv.csv new file mode 100644 index 000000000..086eb44a1 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/valid-csv.csv @@ -0,0 +1,2 @@ +Test,true +Test,false diff --git a/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/valid-csv.csv.license b/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/valid-csv.csv.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/csv-interpreter-executor/valid-csv.csv.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1-B2.xlsx b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1-B2.xlsx new file mode 100644 index 000000000..3465ea34d Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1-B2.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1-B2.xlsx.license b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1-B2.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1-B2.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1-C16.xlsx b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1-C16.xlsx new file mode 100644 index 000000000..0544fa281 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1-C16.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1-C16.xlsx.license b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1-C16.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/test-A1-C16.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/row-deleter-executor/valid-duplicate-row.jv b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/valid-duplicate-row.jv new file mode 100644 index 000000000..cbb08f0f9 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/valid-duplicate-row.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype RowDeleter { + delete: [row 1, row 1]; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/row-deleter-executor/valid-multiple-row-deleter.jv b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/valid-multiple-row-deleter.jv new file mode 100644 index 000000000..d1384e505 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/valid-multiple-row-deleter.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype RowDeleter { + delete: [row 1, row 5]; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/row-deleter-executor/valid-single-row-deleter.jv b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/valid-single-row-deleter.jv new file mode 100644 index 000000000..cbb6610ff --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/row-deleter-executor/valid-single-row-deleter.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype RowDeleter { + delete: [row 1]; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/test-A1-C16.xlsx b/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/test-A1-C16.xlsx new file mode 100644 index 000000000..0544fa281 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/test-A1-C16.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/test-A1-C16.xlsx.license b/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/test-A1-C16.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/test-A1-C16.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/valid-custom-sheet-name.jv b/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/valid-custom-sheet-name.jv new file mode 100644 index 000000000..11b753171 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/valid-custom-sheet-name.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestWorkbookExtractor { + } + + block TestBlock oftype SheetPicker { + sheetName: "MyCustomSheet"; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/valid-sheet-picker.jv b/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/valid-sheet-picker.jv new file mode 100644 index 000000000..40654c112 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/sheet-picker-executor/valid-sheet-picker.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestWorkbookExtractor { + } + + block TestBlock oftype SheetPicker { + sheetName: "Sheet1"; + } + + block TestLoader oftype TestSheetLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-empty.xlsx b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-empty.xlsx new file mode 100644 index 000000000..af7b184be Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-empty.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-empty.xlsx.license b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-empty.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-empty.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-with-header.xlsx b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-with-header.xlsx new file mode 100644 index 000000000..9417e87a4 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-with-header.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-with-header.xlsx.license b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-with-header.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-with-header.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-without-header.xlsx b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-without-header.xlsx new file mode 100644 index 000000000..880859630 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-without-header.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-without-header.xlsx.license b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-without-header.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/test-without-header.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-with-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-with-header.jv new file mode 100644 index 000000000..7e46a44c6 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-with-header.jv @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype TableInterpreter { + header: true; + columns: []; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-without-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-without-header.jv new file mode 100644 index 000000000..b6e423edc --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-empty-columns-without-header.jv @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype TableInterpreter { + header: false; + columns: []; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-capitalized-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-capitalized-header.jv new file mode 100644 index 000000000..3d9929495 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-capitalized-header.jv @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype TableInterpreter { + header: true; + columns: [ + "Index" oftype integer, + "Name" oftype text, + "Flag" oftype boolean + ]; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-header.jv new file mode 100644 index 000000000..cd368186b --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-with-header.jv @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype TableInterpreter { + header: true; + columns: [ + "index" oftype integer, + "name" oftype text, + "flag" oftype boolean + ]; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header.jv new file mode 100644 index 000000000..6fa8f5c1d --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-without-header.jv @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype TableInterpreter { + header: false; + columns: [ + "index" oftype integer, + "name" oftype text, + "flag" oftype boolean + ]; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-valuetype-with-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-valuetype-with-header.jv new file mode 100644 index 000000000..f8ec03446 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-valuetype-with-header.jv @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype TableInterpreter { + header: true; + columns: [ + "index" oftype integer, + "name" oftype text, + "flag" oftype integer + ]; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-valuetype-without-header.jv b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-valuetype-without-header.jv new file mode 100644 index 000000000..ebc269171 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-interpreter-executor/valid-wrong-valuetype-without-header.jv @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + + block TestExtractor oftype TestSheetExtractor { + } + + block TestBlock oftype TableInterpreter { + header: false; + columns: [ + "index" oftype integer, + "name" oftype text, + "flag" oftype integer + ]; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-empty.xlsx b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-empty.xlsx new file mode 100644 index 000000000..af7b184be Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-empty.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-empty.xlsx.license b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-empty.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-empty.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-excel.xlsx b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-excel.xlsx new file mode 100644 index 000000000..8869be3b3 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-excel.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-excel.xlsx.license b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-excel.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/test-excel.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-column-overwrite.jv b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-column-overwrite.jv new file mode 100644 index 000000000..91c301ed4 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-column-overwrite.jv @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + transform Double { + from original oftype integer; + to double oftype integer; + + double: original * 2; + } + + block TestExtractor oftype TestTableExtractor { + } + + block TestBlock oftype TableTransformer { + inputColumns: ['index']; + outputColumn: 'index'; + use: Double; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-column-type-change.jv b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-column-type-change.jv new file mode 100644 index 000000000..a85a6b2ad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-column-type-change.jv @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + transform ToBool { + from asInteger oftype integer; + to asBool oftype boolean; + + asBool: asInteger != 0; + } + + block TestExtractor oftype TestTableExtractor { + } + + block TestBlock oftype TableTransformer { + inputColumns: ['index']; + outputColumn: 'index'; + use: ToBool; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-missing-input-column.jv b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-missing-input-column.jv new file mode 100644 index 000000000..44e535ffc --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-missing-input-column.jv @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + transform Add { + from left oftype integer; + from right oftype integer; + to added oftype integer; + + added: left + right; + } + + block TestExtractor oftype TestTableExtractor { + } + + block TestBlock oftype TableTransformer { + inputColumns: ['index', 'id']; + outputColumn: 'add'; + use: Add; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-transfomer.jv b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-transfomer.jv new file mode 100644 index 000000000..856d8024b --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-transfomer.jv @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + transform Double { + from original oftype integer; + to double oftype integer; + + double: original * 2; + } + + block TestExtractor oftype TestTableExtractor { + } + + block TestBlock oftype TableTransformer { + inputColumns: ['index']; + outputColumn: 'index2'; + use: Double; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-transform-type-missmatch.jv b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-transform-type-missmatch.jv new file mode 100644 index 000000000..52675e73d --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/table-transformer-executor/valid-transform-type-missmatch.jv @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + transform Double { + from original oftype integer; + to double oftype integer; + + double: original * 2; + } + + block TestExtractor oftype TestTableExtractor { + } + + block TestBlock oftype TableTransformer { + inputColumns: ['name']; + outputColumn: 'index2'; + use: Double; + } + + block TestLoader oftype TestTableLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-empty.xlsx b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-empty.xlsx new file mode 100644 index 000000000..af7b184be Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-empty.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-empty.xlsx.license b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-empty.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-empty.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-excel.xlsx b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-excel.xlsx new file mode 100644 index 000000000..8869be3b3 Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-excel.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-excel.xlsx.license b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-excel.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-excel.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-multiple-sheets.xlsx b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-multiple-sheets.xlsx new file mode 100644 index 000000000..3f3619fcc Binary files /dev/null and b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-multiple-sheets.xlsx differ diff --git a/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-multiple-sheets.xlsx.license b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-multiple-sheets.xlsx.license new file mode 100644 index 000000000..17c5d2bad --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/test-multiple-sheets.xlsx.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg + +SPDX-License-Identifier: AGPL-3.0-only diff --git a/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/valid-excel-interpreter.jv b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/valid-excel-interpreter.jv new file mode 100644 index 000000000..a267981e2 --- /dev/null +++ b/libs/extensions/tabular/exec/test/assets/xlsx-interpreter-executor/valid-excel-interpreter.jv @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +pipeline TestPipeline { + block TestExtractor oftype TestFileExtractor { + } + + block TestBlock oftype XLSXInterpreter { + } + + block TestLoader oftype TestWorkbookLoader { + } + + TestExtractor -> TestBlock -> TestLoader; +} diff --git a/libs/extensions/tabular/exec/test/jsdom-environment-fix.ts b/libs/extensions/tabular/exec/test/jsdom-environment-fix.ts new file mode 100644 index 000000000..d5af83010 --- /dev/null +++ b/libs/extensions/tabular/exec/test/jsdom-environment-fix.ts @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import JSDOMEnvironment from 'jest-environment-jsdom'; + +// @nrwl/jest/preset loads in jsdom environment which does not support a few node.js functions. +// This class adds these node.js functions. +export default class JSDOMEnvironmentFix extends JSDOMEnvironment { + constructor(...args: ConstructorParameters) { + super(...args); + + // https://github.com/jsdom/jsdom/issues/3363 + this.global.structuredClone = structuredClone; + // https://github.com/jestjs/jest/issues/11204 + // https://github.com/jestjs/jest/issues/12622 + this.global.setImmediate = setImmediate; + } +} diff --git a/libs/extensions/tabular/exec/test/test-extension/TestBlockTypes.jv b/libs/extensions/tabular/exec/test/test-extension/TestBlockTypes.jv new file mode 100644 index 000000000..ba08556c4 --- /dev/null +++ b/libs/extensions/tabular/exec/test/test-extension/TestBlockTypes.jv @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +builtin blocktype TestSheetExtractor { + input inPort oftype None; + output outPort oftype Sheet; +} + +builtin blocktype TestSheetLoader { + input inPort oftype Sheet; + output outPort oftype None; +} + +builtin blocktype TestTextFileExtractor { + input inPort oftype None; + output outPort oftype TextFile; +} + +builtin blocktype TestFileExtractor { + input inPort oftype None; + output outPort oftype File; +} + +builtin blocktype TestWorkbookExtractor { + input inPort oftype None; + output outPort oftype Workbook; +} + +builtin blocktype TestWorkbookLoader { + input inPort oftype Workbook; + output outPort oftype None; +} + +builtin blocktype TestTableExtractor { + input inPort oftype None; + output outPort oftype Table; +} + +builtin blocktype TestTableLoader { + input inPort oftype Table; + output outPort oftype None; +} diff --git a/libs/extensions/tabular/exec/test/util.ts b/libs/extensions/tabular/exec/test/util.ts new file mode 100644 index 000000000..68180db6f --- /dev/null +++ b/libs/extensions/tabular/exec/test/util.ts @@ -0,0 +1,124 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as path from 'path'; + +import { + Table, + TableRow, + Workbook, + parseValueToInternalRepresentation, +} from '@jvalue/jayvee-execution'; +import { + InternalValueRepresentation, + Valuetype, +} from '@jvalue/jayvee-language-server'; +import * as exceljs from 'exceljs'; + +import { ColumnDefinitionEntry } from '../src/lib/table-interpreter-executor'; + +export async function createWorkbookFromLocalExcelFile( + fileName: string, +): Promise { + const workBookFromFile = new exceljs.Workbook(); + await workBookFromFile.xlsx.readFile(path.resolve(__dirname, fileName)); + + const workbook = new Workbook(); + + workBookFromFile.eachSheet((workSheet) => { + const workSheetDataArray: string[][] = []; + workSheet.eachRow((row, rowNumber) => { + const cellValues: string[] = []; + + // ExcelJS Rows and Columns are indexed from 1 + // We reduce their index to match Sheets being zero indexed + row.eachCell( + { includeEmpty: true }, + (cell: exceljs.Cell, colNumber: number) => { + cellValues[colNumber - 1] = cell.text; + }, + ); + + workSheetDataArray[rowNumber - 1] = cellValues; + }); + + workbook.addSheet(workSheetDataArray, workSheet.name); + }); + + return workbook; +} + +export type ReducedColumnDefinitionEntry = Pick< + ColumnDefinitionEntry, + 'sheetColumnIndex' | 'columnName' | 'valuetype' +>; + +/** + * Creates a Table from the first sheet of the excel file pointed to by {@link fileName} + * and parsed using the given column definitions. + * Note: The parsing is static and thus cannot detect runtime types! + * @param fileName + * @param columnDefinitions columns to be read from table (no header matching) + * @returns Table containing data of excel + */ +export async function createTableFromLocalExcelFile( + fileName: string, + columnDefinitions: ReducedColumnDefinitionEntry[], +): Promise { + const workBookFromFile = new exceljs.Workbook(); + await workBookFromFile.xlsx.readFile(path.resolve(__dirname, fileName)); + + const workSheet = workBookFromFile.worksheets[0] as exceljs.Worksheet; + const table = new Table(); + + columnDefinitions.forEach((columnDefinition) => { + table.addColumn(columnDefinition.columnName, { + values: [], + valuetype: columnDefinition.valuetype, + }); + }); + + workSheet.eachRow((row) => { + const tableRow = constructTableRow(row, columnDefinitions); + table.addRow(tableRow); + }); + + return table; +} +function constructTableRow( + row: exceljs.Row, + columnDefinitions: ReducedColumnDefinitionEntry[], +): TableRow { + const tableRow: TableRow = {}; + + row.eachCell( + { includeEmpty: true }, + (cell: exceljs.Cell, colNumber: number) => { + if (colNumber > columnDefinitions.length) return; + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + const columnDefinition = columnDefinitions[colNumber - 1]!; + const value = cell.text; + const valuetype = columnDefinition.valuetype; + + const parsedValue = parseAndValidatePrimitiveValue(value, valuetype); + if (parsedValue === undefined) { + return; + } + + tableRow[columnDefinition.columnName] = parsedValue; + }, + ); + return tableRow; +} +function parseAndValidatePrimitiveValue( + value: string, + valuetype: Valuetype, +): InternalValueRepresentation | undefined { + const parsedValue = parseValueToInternalRepresentation(value, valuetype); + if (parsedValue === undefined) { + return undefined; + } + + return parsedValue; +} diff --git a/libs/extensions/tabular/exec/tsconfig.spec.json b/libs/extensions/tabular/exec/tsconfig.spec.json index 6668655fc..ab8d1573c 100644 --- a/libs/extensions/tabular/exec/tsconfig.spec.json +++ b/libs/extensions/tabular/exec/tsconfig.spec.json @@ -9,6 +9,7 @@ "jest.config.ts", "src/**/*.test.ts", "src/**/*.spec.ts", - "src/**/*.d.ts" + "src/**/*.d.ts", + "test/**/*.ts" ] } diff --git a/libs/interpreter-lib/src/interpreter.ts b/libs/interpreter-lib/src/interpreter.ts index e63bac142..f0d114673 100644 --- a/libs/interpreter-lib/src/interpreter.ts +++ b/libs/interpreter-lib/src/interpreter.ts @@ -4,10 +4,11 @@ import { strict as assert } from 'assert'; +import * as R from '@jvalue/jayvee-execution'; import { + DebugGranularity, ExecutionContext, Logger, - NONE, executeBlocks, isDebugGranularity, logExecutionDuration, @@ -15,7 +16,6 @@ import { registerDefaultConstraintExecutors, useExtension as useExecutionExtension, } from '@jvalue/jayvee-execution'; -import * as R from '@jvalue/jayvee-execution'; import { StdExecExtension } from '@jvalue/jayvee-extensions/std/exec'; import { BlockDefinition, @@ -23,19 +23,17 @@ import { JayveeModel, JayveeServices, PipelineDefinition, + PipelineWrapper, RuntimeParameterProvider, - collectChildren, - collectStartingBlocks, createJayveeServices, - getBlocksInTopologicalSorting, initializeWorkspace, } from '@jvalue/jayvee-language-server'; import * as chalk from 'chalk'; import { NodeFileSystem } from 'langium/node'; -import { LoggerFactory } from './logging/logger-factory'; +import { LoggerFactory } from './logging'; import { ExitCode, extractAstNodeFromString } from './parsing-util'; -import { validateRuntimeParameterLiteral } from './validation-checks/runtime-parameter-literal'; +import { validateRuntimeParameterLiteral } from './validation-checks'; interface InterpreterOptions { debugGranularity: R.DebugGranularity; @@ -48,6 +46,7 @@ export interface RunOptions { debug: boolean; debugGranularity: string; debugTarget: string | undefined; + parseOnly?: boolean; } export async function interpretString( @@ -66,13 +65,25 @@ export async function interpretString( return await interpretModel(extractAstNodeFn, options); } -export async function interpretModel( +/** + * Parses a model without executing it. + * Also sets up the environment so that the model can be properly executed. + * + * @returns non-null model, services and loggerFactory on success. + */ +export async function parseModel( extractAstNodeFn: ( services: JayveeServices, loggerFactory: LoggerFactory, ) => Promise, options: RunOptions, -): Promise { +): Promise<{ + model: JayveeModel | null; + loggerFactory: LoggerFactory; + services: JayveeServices | null; +}> { + let services: JayveeServices | null = null; + let model: JayveeModel | null = null; const loggerFactory = new LoggerFactory(options.debug); if (!isDebugGranularity(options.debugGranularity)) { loggerFactory @@ -83,23 +94,40 @@ export async function interpretModel( ', ', )}.`, ); - return ExitCode.FAILURE; + return { model, services, loggerFactory }; } useStdExtension(); registerDefaultConstraintExecutors(); - const services = createJayveeServices(NodeFileSystem).Jayvee; + services = createJayveeServices(NodeFileSystem).Jayvee; await initializeWorkspace(services); setupJayveeServices(services, options.env); - let model: JayveeModel; try { model = await extractAstNodeFn(services, loggerFactory); + return { model, services, loggerFactory }; } catch (e) { loggerFactory .createLogger() .logErr('Could not extract the AST node of the given model.'); + return { model, services, loggerFactory }; + } +} + +export async function interpretModel( + extractAstNodeFn: ( + services: JayveeServices, + loggerFactory: LoggerFactory, + ) => Promise, + options: RunOptions, +): Promise { + const { model, services, loggerFactory } = await parseModel( + extractAstNodeFn, + options, + ); + + if (model == null || services == null) { return ExitCode.FAILURE; } @@ -111,7 +139,8 @@ export async function interpretModel( loggerFactory, { debug: options.debug, - debugGranularity: options.debugGranularity, + // type of options.debugGranularity is asserted in parseModel + debugGranularity: options.debugGranularity as DebugGranularity, debugTargets: debugTargets, }, ); @@ -196,12 +225,7 @@ async function runPipeline( const startTime = new Date(); - const executionOrder = getBlocksInTopologicalSorting(pipeline).map( - (block) => { - return { block: block, value: NONE }; - }, - ); - const executionResult = await executeBlocks(executionContext, executionOrder); + const executionResult = await executeBlocks(executionContext, pipeline); if (R.isErr(executionResult)) { const diagnosticError = executionResult.left; @@ -222,13 +246,16 @@ export function logPipelineOverview( runtimeParameterProvider: RuntimeParameterProvider, logger: Logger, ) { + const pipelineWrapper = new PipelineWrapper(pipeline); + const toString = (block: BlockDefinition, depth = 0): string => { const blockTypeName = block.type.ref?.name; assert(blockTypeName !== undefined); const blockString = `${'\t'.repeat(depth)} -> ${ block.name } (${blockTypeName})`; - const childString = collectChildren(block) + const childString = pipelineWrapper + .getChildBlocks(block) .map((child) => toString(child, depth + 1)) .join('\n'); return blockString + '\n' + childString; @@ -248,7 +275,7 @@ export function logPipelineOverview( linesBuffer.push( `\tBlocks (${pipeline.blocks.length} blocks with ${pipeline.pipes.length} pipes):`, ); - for (const block of collectStartingBlocks(pipeline)) { + for (const block of pipelineWrapper.getStartingBlocks()) { linesBuffer.push(toString(block, 1)); } logger.logInfo(linesBuffer.join('\n')); diff --git a/libs/language-server/src/grammar/block.langium b/libs/language-server/src/grammar/block.langium new file mode 100644 index 000000000..cbf73111c --- /dev/null +++ b/libs/language-server/src/grammar/block.langium @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import './property' +import './blocktype' +import './terminal' + +BlockDefinition: + 'block' name=ID 'oftype' type=[ReferenceableBlocktypeDefinition] body=PropertyBody; \ No newline at end of file diff --git a/libs/language-server/src/grammar/blocktype.langium b/libs/language-server/src/grammar/blocktype.langium index 731e40986..71c05cc98 100644 --- a/libs/language-server/src/grammar/blocktype.langium +++ b/libs/language-server/src/grammar/blocktype.langium @@ -7,6 +7,7 @@ import './iotype' import './terminal' import './transform' import './valuetype' +import './block' import './main' ReferenceableBlocktypeDefinition: diff --git a/libs/language-server/src/grammar/main.langium b/libs/language-server/src/grammar/main.langium index 46097adcb..2a933526f 100644 --- a/libs/language-server/src/grammar/main.langium +++ b/libs/language-server/src/grammar/main.langium @@ -5,6 +5,7 @@ grammar Jayvee import './blocktype' +import './pipeline' import './terminal' import './property' import './valuetype' @@ -23,28 +24,4 @@ entry JayveeModel: | iotypes+=IotypeDefinition )*; -PipelineDefinition: - 'pipeline' name=ID '{' - ( - blocks+=BlockDefinition - | pipes+=PipeDefinition - | valuetypes+=CustomValuetypeDefinition - | constraints+=ConstraintDefinition - | transforms+=TransformDefinition - )* - '}'; -BlockDefinition: - 'block' name=ID 'oftype' type=[ReferenceableBlocktypeDefinition] body=PropertyBody; - -PipeDefinition: - SinglePipeDefinition | ChainedPipeDefinition; - -SinglePipeDefinition: - 'pipe' '{' - 'from' ':' from=[BlockDefinition] ';' - 'to' ':' to=[BlockDefinition] ';' - '}'; - -ChainedPipeDefinition: - blocks+=[BlockDefinition] ('->' blocks+=[BlockDefinition])+ ';'; diff --git a/libs/language-server/src/grammar/pipeline.langium b/libs/language-server/src/grammar/pipeline.langium new file mode 100644 index 000000000..8e6a3e3d8 --- /dev/null +++ b/libs/language-server/src/grammar/pipeline.langium @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import './block' +import './transform' +import './valuetype' +import './constraint' +import './terminal' + +PipelineDefinition: + 'pipeline' name=ID '{' + ( + blocks+=BlockDefinition + | pipes+=PipeDefinition + | valuetypes+=CustomValuetypeDefinition + | constraints+=ConstraintDefinition + | transforms+=TransformDefinition + )* + '}'; + +PipeDefinition: + blocks+=[BlockDefinition] ('->' blocks+=[BlockDefinition])+ ';'; \ No newline at end of file diff --git a/libs/language-server/src/lib/ast/model-util.ts b/libs/language-server/src/lib/ast/model-util.ts index 25606bc9b..b5c804a2a 100644 --- a/libs/language-server/src/lib/ast/model-util.ts +++ b/libs/language-server/src/lib/ast/model-util.ts @@ -2,172 +2,18 @@ // // SPDX-License-Identifier: AGPL-3.0-only -import { strict as assert } from 'assert'; - -import { - AstNode, - LangiumDocuments, - Reference, - assertUnreachable, -} from 'langium'; +import { AstNode, LangiumDocuments } from 'langium'; import { BinaryExpression, - BlockDefinition, BuiltinBlocktypeDefinition, BuiltinConstrainttypeDefinition, - CompositeBlocktypeDefinition, - PipelineDefinition, UnaryExpression, isBuiltinBlocktypeDefinition, - isCompositeBlocktypeDefinition, isJayveeModel, } from './generated/ast'; // eslint-disable-next-line import/no-cycle import { BlockTypeWrapper, ConstraintTypeWrapper } from './wrappers'; -import { PipeWrapper, createSemanticPipes } from './wrappers/pipe-wrapper'; - -export function collectStartingBlocks( - container: PipelineDefinition | CompositeBlocktypeDefinition, -): BlockDefinition[] { - // For composite blocks the first blocks of all pipelines are starting blocks as they have inputs - if (isCompositeBlocktypeDefinition(container)) { - const startingBlocks = container.pipes - .map((pipe) => pipe.blocks[0]) - .map((blockRef: Reference | undefined) => { - if ( - blockRef?.ref !== undefined && - BlockTypeWrapper.canBeWrapped(blockRef.ref.type) - ) { - return blockRef.ref; - } - return undefined; - }) - .filter((x): x is BlockDefinition => x !== undefined); - - return startingBlocks; - } - - const result: BlockDefinition[] = []; - - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - const blocks = container?.blocks ?? []; - for (const block of blocks) { - if (!BlockTypeWrapper.canBeWrapped(block.type)) { - continue; - } - const blockType = new BlockTypeWrapper(block.type); - - if (!blockType.hasInput()) { - result.push(block); - } - } - return result; -} - -export function collectChildren(block: BlockDefinition): BlockDefinition[] { - const outgoingPipes = collectOutgoingPipes(block); - return outgoingPipes.map((pipe) => pipe.to); -} - -export function collectParents(block: BlockDefinition): BlockDefinition[] { - const ingoingPipes = collectIngoingPipes(block); - return ingoingPipes.map((pipe) => pipe.from); -} - -export function collectOutgoingPipes(block: BlockDefinition) { - return collectPipes(block, 'outgoing'); -} - -export function collectIngoingPipes(block: BlockDefinition) { - return collectPipes(block, 'ingoing'); -} - -function collectPipes( - block: BlockDefinition, - kind: 'outgoing' | 'ingoing', -): PipeWrapper[] { - const pipeline = block.$container; - const allPipes = collectAllPipes(pipeline); - - return allPipes.filter((semanticPipe) => { - switch (kind) { - case 'outgoing': - return semanticPipe.from === block; - case 'ingoing': - return semanticPipe.to === block; - case undefined: - return false; - } - return assertUnreachable(kind); - }); -} - -export function collectAllPipes( - container: PipelineDefinition | CompositeBlocktypeDefinition, -): PipeWrapper[] { - const result: PipeWrapper[] = []; - for (const pipe of container.pipes) { - result.push(...createSemanticPipes(pipe)); - } - return result; -} - -/** - * Returns blocks in a pipeline in topological order, based on - * Kahn's algorithm. - * - * Considers a pipeline as a directed, acyclical graph where - * blocks are nodes and pipes are edges. A list in topological - * order has the property that parent nodes are always listed - * before their children. - * - * "[...] a list in topological order is such that no element - * appears in it until after all elements appearing on all paths - * leading to the particular element have been listed." - * - * Kahn, A. B. (1962). Topological sorting of large networks. Communications of the ACM, 5(11), 558–562. - */ -export function getBlocksInTopologicalSorting( - pipeline: PipelineDefinition | CompositeBlocktypeDefinition, -): BlockDefinition[] { - const sortedNodes = []; - const currentNodes = [...collectStartingBlocks(pipeline)]; - let unvisitedEdges = [...collectAllPipes(pipeline)]; - - while (currentNodes.length > 0) { - const node = currentNodes.pop(); - assert(node !== undefined); - - sortedNodes.push(node); - - for (const childNode of collectChildren(node)) { - // Mark edges between parent and child as visited - collectIngoingPipes(childNode) - .filter((e) => e.from === node) - .forEach((e) => { - unvisitedEdges = unvisitedEdges.filter((edge) => !edge.equals(e)); - }); - - // If all edges to the child have been visited - const notRemovedEdges = collectIngoingPipes(childNode).filter((e) => - unvisitedEdges.some((edge) => edge.equals(e)), - ); - if (notRemovedEdges.length === 0) { - // Insert it into currentBlocks - currentNodes.push(childNode); - } - } - } - - // If the graph still contains unvisited edges it is not a DAG - assert( - unvisitedEdges.length === 0, - `The pipeline ${pipeline.name} is expected to have no cycles`, - ); - - return sortedNodes; -} export type UnaryExpressionOperator = UnaryExpression['operator']; export type BinaryExpressionOperator = BinaryExpression['operator']; diff --git a/libs/language-server/src/lib/ast/wrappers/index.ts b/libs/language-server/src/lib/ast/wrappers/index.ts index d757e8cda..81abbe4f1 100644 --- a/libs/language-server/src/lib/ast/wrappers/index.ts +++ b/libs/language-server/src/lib/ast/wrappers/index.ts @@ -10,3 +10,6 @@ export * from './ast-node-wrapper'; export * from './cell-range-wrapper'; export * from './typed-object'; + +export * from './pipe-wrapper'; +export * from './pipeline-wrapper'; diff --git a/libs/language-server/src/lib/ast/wrappers/pipe-wrapper.ts b/libs/language-server/src/lib/ast/wrappers/pipe-wrapper.ts index b69d6c643..70715eb43 100644 --- a/libs/language-server/src/lib/ast/wrappers/pipe-wrapper.ts +++ b/libs/language-server/src/lib/ast/wrappers/pipe-wrapper.ts @@ -9,102 +9,59 @@ import { DiagnosticInfo } from 'langium'; import { BlockDefinition, BlocktypePipeline, - ChainedPipeDefinition, PipeDefinition, - SinglePipeDefinition, - isSinglePipeDefinition, } from '../generated/ast'; import { AstNodeWrapper } from './ast-node-wrapper'; -export class PipeWrapper - implements AstNodeWrapper +export class PipeWrapper< + T extends PipeDefinition | BlocktypePipeline = + | PipeDefinition + | BlocktypePipeline, +> implements AstNodeWrapper { - public readonly astNode: N; - private readonly chainIndex?: number; + public readonly astNode: T; + private readonly chainIndex: number; public readonly from: BlockDefinition; public readonly to: BlockDefinition; - constructor( - pipe: ChainedPipeDefinition | BlocktypePipeline, - chainIndex: number, - ); - constructor(pipe: SinglePipeDefinition); - constructor(pipe: N, chainIndex?: number) { + constructor(pipe: T, chainIndex: number) { this.astNode = pipe; - if (isSinglePipeDefinition(pipe)) { - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - assert(pipe.from?.ref !== undefined); - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - assert(pipe.to?.ref !== undefined); - this.from = pipe.from.ref; - this.to = pipe.to.ref; - } else { - assert(chainIndex !== undefined); - assert(0 <= chainIndex && chainIndex + 1 < pipe.blocks.length); - assert(pipe.blocks[chainIndex]?.ref !== undefined); - assert(pipe.blocks[chainIndex + 1]?.ref !== undefined); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - this.from = pipe.blocks[chainIndex]!.ref!; - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion - this.to = pipe.blocks[chainIndex + 1]!.ref!; - this.chainIndex = chainIndex; - } + assert(0 <= chainIndex && chainIndex + 1 < pipe.blocks.length); + assert(pipe.blocks[chainIndex]?.ref !== undefined); + assert(pipe.blocks[chainIndex + 1]?.ref !== undefined); + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + this.from = pipe.blocks[chainIndex]!.ref!; + // eslint-disable-next-line @typescript-eslint/no-non-null-assertion + this.to = pipe.blocks[chainIndex + 1]!.ref!; + this.chainIndex = chainIndex; } - getFromDiagnostic(): DiagnosticInfo { - if (isSinglePipeDefinition(this.astNode)) { - const result: DiagnosticInfo = { - node: this.astNode, - property: 'from', - }; - return result; - } - assert(this.chainIndex !== undefined); - const result: DiagnosticInfo = { + getFromDiagnostic(): DiagnosticInfo { + return { node: this.astNode, property: 'blocks', index: this.chainIndex, }; - return result; } - getToDiagnostic(): DiagnosticInfo { - if (isSinglePipeDefinition(this.astNode)) { - const result: DiagnosticInfo = { - node: this.astNode, - property: 'to', - }; - return result; - } - assert(this.chainIndex !== undefined); - const result: DiagnosticInfo = { + getToDiagnostic(): DiagnosticInfo { + return { node: this.astNode, property: 'blocks', index: this.chainIndex + 1, }; - return result; } - equals(pipe: PipeWrapper): boolean { + equals(pipe: PipeWrapper): boolean { return this.from === pipe.from && this.to === pipe.to; } static canBeWrapped( - pipe: ChainedPipeDefinition | BlocktypePipeline, + pipe: PipeDefinition | BlocktypePipeline, chainIndex: number, - ): boolean; - static canBeWrapped(pipe: SinglePipeDefinition): boolean; - static canBeWrapped( - pipe: N, - chainIndex?: number, ): boolean { - if (isSinglePipeDefinition(pipe)) { - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - return pipe.from?.ref !== undefined && pipe.to?.ref !== undefined; - } return ( - chainIndex !== undefined && 0 <= chainIndex && chainIndex + 1 < pipe.blocks.length && pipe.blocks[chainIndex]?.ref !== undefined && @@ -113,32 +70,16 @@ export class PipeWrapper } } -export function createSemanticPipes( - pipe: PipeDefinition | BlocktypePipeline, -): PipeWrapper[] { - if (isSinglePipeDefinition(pipe)) { - return createFromSinglePipe(pipe); - } - return createFromChainedPipe(pipe); -} - -function createFromSinglePipe(pipe: SinglePipeDefinition): PipeWrapper[] { - if (PipeWrapper.canBeWrapped(pipe)) { - return [new PipeWrapper(pipe)]; - } - return []; -} - -function createFromChainedPipe( - pipe: ChainedPipeDefinition | BlocktypePipeline, -): PipeWrapper[] { - const result: PipeWrapper[] = []; +export function createWrappersFromPipeChain< + A extends PipeDefinition | BlocktypePipeline, +>(pipe: A): PipeWrapper[] { + const result: PipeWrapper[] = []; for (let chainIndex = 0; chainIndex < pipe.blocks.length - 1; ++chainIndex) { if (!PipeWrapper.canBeWrapped(pipe, chainIndex)) { continue; } - const semanticPipe = new PipeWrapper(pipe, chainIndex); - result.push(semanticPipe); + const pipeWrapper = new PipeWrapper(pipe, chainIndex); + result.push(pipeWrapper); } return result; } diff --git a/libs/language-server/src/lib/ast/wrappers/pipeline-wrapper.ts b/libs/language-server/src/lib/ast/wrappers/pipeline-wrapper.ts new file mode 100644 index 000000000..aa4e2810b --- /dev/null +++ b/libs/language-server/src/lib/ast/wrappers/pipeline-wrapper.ts @@ -0,0 +1,151 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import { strict as assert } from 'assert'; + +import { + BlockDefinition, + CompositeBlocktypeDefinition, + PipelineDefinition, +} from '../generated/ast'; + +import { AstNodeWrapper } from './ast-node-wrapper'; +import { PipeWrapper, createWrappersFromPipeChain } from './pipe-wrapper'; + +export class PipelineWrapper< + T extends PipelineDefinition | CompositeBlocktypeDefinition, +> implements AstNodeWrapper +{ + public readonly astNode: T; + + allPipes: PipeWrapper[] = []; + + constructor(pipesContainer: T) { + this.astNode = pipesContainer; + + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (pipesContainer.pipes === undefined) { + this.allPipes = []; + return; + } + + this.allPipes = pipesContainer.pipes.flatMap((pipe) => + createWrappersFromPipeChain(pipe), + ); + } + + static canBeWrapped( + pipesContainer: PipelineDefinition | CompositeBlocktypeDefinition, + ): boolean { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (pipesContainer.pipes === undefined) { + return true; + } + + for (const pipeDefinition of pipesContainer.pipes) { + for ( + let chainIndex = 0; + chainIndex < pipeDefinition.blocks.length - 1; + ++chainIndex + ) { + if (!PipeWrapper.canBeWrapped(pipeDefinition, chainIndex)) { + return false; + } + } + } + return true; + } + + getStartingBlockPipes(): PipeWrapper[] { + return this.allPipes.filter((pipe) => { + const parentBlock = pipe.from; + const isToOfOtherPipe = + this.allPipes.filter((p) => p.to === parentBlock).length > 0; + return !isToOfOtherPipe; + }); + } + + getStartingBlocks(): BlockDefinition[] { + const startingBlocks = this.getStartingBlockPipes().map((p) => p.from); + + // Special case: the extractor is reused for multiple paths + // Thus, we remove duplicates + const withoutDuplicates = [...new Set(startingBlocks)]; + return withoutDuplicates; + } + + getOutgoingPipes(blockDefinition: BlockDefinition): PipeWrapper[] { + return this.allPipes.filter((pipe) => { + return pipe.from === blockDefinition; + }); + } + + getChildBlocks(blockDefinition: BlockDefinition): BlockDefinition[] { + return this.getOutgoingPipes(blockDefinition).map((p) => p.to); + } + + getIngoingPipes(blockDefinition: BlockDefinition): PipeWrapper[] { + return this.allPipes.filter((pipe) => { + return pipe.to === blockDefinition; + }); + } + + getParentBlocks(blockDefinition: BlockDefinition): BlockDefinition[] { + return this.getIngoingPipes(blockDefinition).map((p) => p.from); + } + + /** + * Returns blocks in a pipeline in topological order, based on + * Kahn's algorithm. + * + * Considers a pipeline as a directed, acyclical graph where + * blocks are nodes and pipes are edges. A list in topological + * order has the property that parent nodes are always listed + * before their children. + * + * "[...] a list in topological order is such that no element + * appears in it until after all elements appearing on all paths + * leading to the particular element have been listed." + * + * Kahn, A. B. (1962). Topological sorting of large networks. Communications of the ACM, 5(11), 558–562. + */ + getBlocksInTopologicalSorting(): BlockDefinition[] { + const sortedNodes = []; + const currentNodes = [...this.getStartingBlocks()]; + let unvisitedEdges = [...this.allPipes]; + + while (currentNodes.length > 0) { + const node = currentNodes.pop(); + assert(node !== undefined); + + sortedNodes.push(node); + + for (const childNode of this.getChildBlocks(node)) { + // Mark edges between parent and child as visited + this.getIngoingPipes(childNode) + .filter((e) => e.from === node) + .forEach((e) => { + unvisitedEdges = unvisitedEdges.filter((edge) => !edge.equals(e)); + }); + + // If all edges to the child have been visited + const notRemovedEdges = this.getIngoingPipes(childNode).filter((e) => + unvisitedEdges.some((edge) => edge.equals(e)), + ); + if (notRemovedEdges.length === 0) { + // Insert it into currentBlocks + currentNodes.push(childNode); + } + } + } + + // If the graph still contains unvisited edges it is not a DAG + assert( + unvisitedEdges.length === 0, + `The pipeline ${this.astNode.name} is expected to have no cycles`, + ); + + return sortedNodes; + } +} diff --git a/libs/language-server/src/lib/validation/checks/block-definition.spec.ts b/libs/language-server/src/lib/validation/checks/block-definition.spec.ts deleted file mode 100644 index a8da55530..000000000 --- a/libs/language-server/src/lib/validation/checks/block-definition.spec.ts +++ /dev/null @@ -1,117 +0,0 @@ -// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg -// -// SPDX-License-Identifier: AGPL-3.0-only - -import { AstNode, AstNodeLocator, LangiumDocument } from 'langium'; -import { NodeFileSystem } from 'langium/node'; - -import { - BlockDefinition, - ValidationContext, - createJayveeServices, -} from '../..'; -import { - ParseHelperOptions, - expectNoParserAndLexerErrors, - parseHelper, - readJvTestAssetHelper, - validationAcceptorMockImpl, -} from '../../../test'; - -import { validateBlockDefinition } from './block-definition'; - -describe('Validation of BlockDefinition', () => { - let parse: ( - input: string, - options?: ParseHelperOptions, - ) => Promise>; - - const validationAcceptorMock = jest.fn(validationAcceptorMockImpl); - - let locator: AstNodeLocator; - - const readJvTestAsset = readJvTestAssetHelper( - __dirname, - '../../../test/assets/', - ); - - async function parseAndValidateBlock(input: string) { - const document = await parse(input); - expectNoParserAndLexerErrors(document); - - const block = locator.getAstNode( - document.parseResult.value, - 'pipelines@0/blocks@0', - ) as BlockDefinition; - - validateBlockDefinition( - block, - new ValidationContext(validationAcceptorMock), - ); - } - - beforeAll(() => { - // Create language services - const services = createJayveeServices(NodeFileSystem).Jayvee; - locator = services.workspace.AstNodeLocator; - // Parse function for Jayvee (without validation) - parse = parseHelper(services); - }); - - afterEach(() => { - // Reset mock - validationAcceptorMock.mockReset(); - }); - - it('should diagnose error on block without pipe', async () => { - const text = readJvTestAsset('block-definition/invalid-missing-pipe.jv'); - - await parseAndValidateBlock(text); - - expect(validationAcceptorMock).toHaveBeenCalledTimes(1); - expect(validationAcceptorMock).toHaveBeenCalledWith( - 'warning', - 'A pipe should be connected to the output of this block', - expect.any(Object), - ); - }); - - it('should diagnose error on block as output without having an output', async () => { - const text = readJvTestAsset( - 'block-definition/invalid-output-block-as-input.jv', - ); - - await parseAndValidateBlock(text); - - expect(validationAcceptorMock).toHaveBeenCalledTimes(1); - expect(validationAcceptorMock).toHaveBeenCalledWith( - 'error', - 'Blocks of type TestTableLoader do not have an output', - expect.any(Object), - ); - }); - - it('should diagnose error on block as input for multiple pipes', async () => { - const text = readJvTestAsset( - 'block-definition/invalid-block-as-multiple-pipe-inputs.jv', - ); - - await parseAndValidateBlock(text); - - expect(validationAcceptorMock).toHaveBeenCalledTimes(2); - expect(validationAcceptorMock).toHaveBeenNthCalledWith( - 2, - 'error', - 'At most one pipe can be connected to the input of a TestTableLoader', - expect.any(Object), - ); - }); - - it('should have no error on valid block definition', async () => { - const text = readJvTestAsset('block-definition/valid-block-definition.jv'); - - await parseAndValidateBlock(text); - - expect(validationAcceptorMock).toHaveBeenCalledTimes(0); - }); -}); diff --git a/libs/language-server/src/lib/validation/checks/block-definition.ts b/libs/language-server/src/lib/validation/checks/block-definition.ts index d8650c037..3a176cc8d 100644 --- a/libs/language-server/src/lib/validation/checks/block-definition.ts +++ b/libs/language-server/src/lib/validation/checks/block-definition.ts @@ -2,94 +2,6 @@ // // SPDX-License-Identifier: AGPL-3.0-only -/** - * See https://jvalue.github.io/jayvee/docs/dev/guides/working-with-the-ast/ for why the following ESLint rule is disabled for this file. - */ -/* eslint-disable @typescript-eslint/no-unnecessary-condition */ - -import { assertUnreachable } from 'langium'; - -import { - BlockDefinition, - isCompositeBlocktypeDefinition, -} from '../../ast/generated/ast'; -import { - collectIngoingPipes, - collectOutgoingPipes, -} from '../../ast/model-util'; -import { PipeWrapper } from '../../ast/wrappers/pipe-wrapper'; -import { BlockTypeWrapper } from '../../ast/wrappers/typed-object/blocktype-wrapper'; -import { ValidationContext } from '../validation-context'; - -export function validateBlockDefinition( - block: BlockDefinition, - context: ValidationContext, -): void { - checkPipesOfBlock(block, 'input', context); - checkPipesOfBlock(block, 'output', context); -} - -function checkPipesOfBlock( - block: BlockDefinition, - whatToCheck: 'input' | 'output', - context: ValidationContext, -): void { - if (!BlockTypeWrapper.canBeWrapped(block?.type)) { - return; - } - const blockType = new BlockTypeWrapper(block?.type); - - let pipes: PipeWrapper[]; - switch (whatToCheck) { - case 'input': { - pipes = collectIngoingPipes(block); - break; - } - case 'output': { - pipes = collectOutgoingPipes(block); - break; - } - default: { - assertUnreachable(whatToCheck); - } - } - - if ( - (whatToCheck === 'input' && !blockType.hasInput()) || - (whatToCheck === 'output' && !blockType.hasOutput()) - ) { - for (const pipe of pipes) { - context.accept( - 'error', - `Blocks of type ${blockType.type} do not have an ${whatToCheck}`, - whatToCheck === 'input' - ? pipe.getToDiagnostic() - : pipe.getFromDiagnostic(), - ); - } - } else if (pipes.length > 1 && whatToCheck === 'input') { - for (const pipe of pipes) { - context.accept( - 'error', - `At most one pipe can be connected to the ${whatToCheck} of a ${blockType.type}`, - pipe.getToDiagnostic(), - ); - } - } else if (pipes.length === 0) { - const isLastBlockOfCompositeBlocktype = - isCompositeBlocktypeDefinition(block.$container) && - block.$container.blocks.at(-1)?.name === block.name; - - // The last block in a composite block is connected to the output, not another block - if (!isLastBlockOfCompositeBlocktype) { - context.accept( - 'warning', - `A pipe should be connected to the ${whatToCheck} of this block`, - { - node: block, - property: 'name', - }, - ); - } - } +export function validateBlockDefinition(): void { + // Nothing to do } diff --git a/libs/language-server/src/lib/validation/checks/composite-blocktype-definition.spec.ts b/libs/language-server/src/lib/validation/checks/composite-blocktype-definition.spec.ts index 3da9d6656..5f719bd85 100644 --- a/libs/language-server/src/lib/validation/checks/composite-blocktype-definition.spec.ts +++ b/libs/language-server/src/lib/validation/checks/composite-blocktype-definition.spec.ts @@ -115,4 +115,47 @@ describe('Validation of CompositeBlocktypeDefinition', () => { expect(validationAcceptorMock).toHaveBeenCalledTimes(0); }); + + it('should diagnose error on block as input for multiple pipes', async () => { + const text = readJvTestAsset( + 'composite-blocktype-definition/invalid-block-as-multiple-pipe-inputs.jv', + ); + + await parseAndValidateBlocktype(text); + + // first 2 errors for multiple pipelines in test file + expect(validationAcceptorMock).toHaveBeenCalledTimes(4); + expect(validationAcceptorMock).toHaveBeenNthCalledWith( + 3, + 'error', + 'At most one pipe can be connected to the input of a block. Currently, the following 2 blocks are connected via pipes: "BlockFrom1", "BlockFrom2"', + expect.any(Object), + ); + expect(validationAcceptorMock).toHaveBeenNthCalledWith( + 4, + 'error', + 'At most one pipe can be connected to the input of a block. Currently, the following 2 blocks are connected via pipes: "BlockFrom1", "BlockFrom2"', + expect.any(Object), + ); + }); + + it('should diagnose error on block without pipe', async () => { + const text = readJvTestAsset( + 'composite-blocktype-definition/invalid-unconnected-block.jv', + ); + + await parseAndValidateBlocktype(text); + + expect(validationAcceptorMock).toHaveBeenCalledTimes(2); + expect(validationAcceptorMock).toHaveBeenCalledWith( + 'warning', + 'A pipe should be connected to the input of this block', + expect.any(Object), + ); + expect(validationAcceptorMock).toHaveBeenCalledWith( + 'warning', + 'A pipe should be connected to the output of this block', + expect.any(Object), + ); + }); }); diff --git a/libs/language-server/src/lib/validation/checks/composite-blocktype-definition.ts b/libs/language-server/src/lib/validation/checks/composite-blocktype-definition.ts index 9bf5ced56..5686a9d6c 100644 --- a/libs/language-server/src/lib/validation/checks/composite-blocktype-definition.ts +++ b/libs/language-server/src/lib/validation/checks/composite-blocktype-definition.ts @@ -2,11 +2,16 @@ // // SPDX-License-Identifier: AGPL-3.0-only +import { BlockTypeWrapper, PipelineWrapper } from '../../ast'; import { EvaluationContext } from '../../ast/expressions/evaluation'; -import { CompositeBlocktypeDefinition } from '../../ast/generated/ast'; +import { + BlockDefinition, + CompositeBlocktypeDefinition, +} from '../../ast/generated/ast'; import { ValidationContext } from '../validation-context'; import { validateBlocktypeDefinition } from './blocktype-definition'; +import { checkMultipleBlockInputs } from './pipeline-definition'; export function validateCompositeBlockTypeDefinition( blockType: CompositeBlocktypeDefinition, @@ -16,12 +21,20 @@ export function validateCompositeBlockTypeDefinition( validateBlocktypeDefinition(blockType, validationContext, evaluationContext); checkHasPipeline(blockType, validationContext); checkExactlyOnePipeline(blockType, validationContext); + + checkMultipleBlockInputs(blockType, validationContext); + checkDefinedBlocksAreUsed(blockType, validationContext); } function checkHasPipeline( blockType: CompositeBlocktypeDefinition, context: ValidationContext, ): void { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (blockType.pipes === undefined) { + return; + } + if (blockType.pipes.length === 0) { context.accept( 'error', @@ -38,6 +51,11 @@ function checkExactlyOnePipeline( blockType: CompositeBlocktypeDefinition, context: ValidationContext, ): void { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (blockType.pipes === undefined) { + return; + } + if (blockType.pipes.length > 1) { blockType.pipes.forEach((pipe) => { context.accept( @@ -50,3 +68,73 @@ function checkExactlyOnePipeline( }); } } + +export function checkDefinedBlocksAreUsed( + blocktypeDefinition: CompositeBlocktypeDefinition, + context: ValidationContext, +): void { + if (!PipelineWrapper.canBeWrapped(blocktypeDefinition)) { + return; + } + const pipelineWrapper = new PipelineWrapper(blocktypeDefinition); + + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (blocktypeDefinition.blocks === undefined) { + return; + } + + const containedBlocks = blocktypeDefinition.blocks; + for (const block of containedBlocks) { + doCheckDefinedBlockIsUsed(pipelineWrapper, block, context); + } +} + +function doCheckDefinedBlockIsUsed( + pipelineWrapper: PipelineWrapper, + block: BlockDefinition, + context: ValidationContext, +): void { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (block.type === undefined || !BlockTypeWrapper.canBeWrapped(block.type)) { + return; + } + const pipes = pipelineWrapper.astNode.pipes; + + const isConnectedToInput = pipes.some( + (pipe) => + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + pipe?.blocks?.at(0)?.ref === block, + ); + if (!isConnectedToInput) { + const parents = pipelineWrapper.getParentBlocks(block); + if (parents.length === 0) { + context.accept( + 'warning', + `A pipe should be connected to the input of this block`, + { + node: block, + property: 'name', + }, + ); + } + } + + const isConnectedToOutput = pipes.some( + (pipeline) => + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + pipeline?.blocks?.at(-1)?.ref === block, + ); + if (!isConnectedToOutput) { + const children = pipelineWrapper.getChildBlocks(block); + if (children.length === 0) { + context.accept( + 'warning', + `A pipe should be connected to the output of this block`, + { + node: block, + property: 'name', + }, + ); + } + } +} diff --git a/libs/language-server/src/lib/validation/checks/pipe-definition.spec.ts b/libs/language-server/src/lib/validation/checks/pipe-definition.spec.ts index 9aa8ff203..28952ae66 100644 --- a/libs/language-server/src/lib/validation/checks/pipe-definition.spec.ts +++ b/libs/language-server/src/lib/validation/checks/pipe-definition.spec.ts @@ -60,12 +60,10 @@ describe('Validation of PipeDefinition', () => { validationAcceptorMock.mockReset(); }); - describe('SinglePipeDefinition syntax', () => { + describe('PipeDefinition syntax', () => { // This test should succeed, because the error is thrown by langium during linking, not during validation! it('should have no error even if pipe references non existing block', async () => { - const text = readJvTestAsset( - 'pipe-definition/single/valid-undefined-block.jv', - ); + const text = readJvTestAsset('pipe-definition/valid-undefined-block.jv'); await parseAndValidatePipe(text); @@ -74,7 +72,7 @@ describe('Validation of PipeDefinition', () => { it('should have no error even if pipe references block of non existing type', async () => { const text = readJvTestAsset( - 'pipe-definition/single/valid-unknown-blocktype.jv', + 'pipe-definition/valid-unknown-blocktype.jv', ); await parseAndValidatePipe(text); @@ -84,7 +82,7 @@ describe('Validation of PipeDefinition', () => { it('should diagnose error on unsupported pipe between Blocktypes', async () => { const text = readJvTestAsset( - 'pipe-definition/single/invalid-pipe-between-blocktypes.jv', + 'pipe-definition/invalid-pipe-between-blocktypes.jv', ); await parseAndValidatePipe(text); @@ -93,46 +91,29 @@ describe('Validation of PipeDefinition', () => { expect(validationAcceptorMock).toHaveBeenNthCalledWith( 2, 'error', - `The output type "File" of TestFileExtractor is incompatible with the input type "Table" of TestTableLoader`, + 'The output type "File" of block "TestExtractor" (of type "TestFileExtractor") is not compatible with the input type "Table" of block "TestLoader" (of type "TestTableLoader")', expect.any(Object), ); }); - }); - - describe('ChainedPipeDefinition syntax', () => { - // This test should succeed, because the error is thrown by langium during linking, not during validation! - it('should have no error even if pipe references non existing block', async () => { - const text = readJvTestAsset( - 'pipe-definition/chained/valid-undefined-block.jv', - ); - - await parseAndValidatePipe(text); - - expect(validationAcceptorMock).toHaveBeenCalledTimes(0); - }); - it('should have no error even if pipe references block of non existing type', async () => { + it('should diagnose error on connecting loader block to extractor block with a pipe', async () => { const text = readJvTestAsset( - 'pipe-definition/chained/valid-unknown-blocktype.jv', - ); - - await parseAndValidatePipe(text); - - expect(validationAcceptorMock).toHaveBeenCalledTimes(0); - }); - - it('should diagnose error on unsupported pipe between Blocktypes', async () => { - const text = readJvTestAsset( - 'pipe-definition/chained/invalid-pipe-between-blocktypes.jv', + 'pipe-definition/invalid-output-block-as-input.jv', ); await parseAndValidatePipe(text); expect(validationAcceptorMock).toHaveBeenCalledTimes(2); + expect(validationAcceptorMock).toHaveBeenNthCalledWith( + 1, + 'error', + 'Block "BlockTo" cannot be connected to other blocks. Its blocktype "TestFileLoader" has output type "None".', + expect.any(Object), + ); expect(validationAcceptorMock).toHaveBeenNthCalledWith( 2, 'error', - `The output type "File" of TestFileExtractor is incompatible with the input type "Table" of TestTableLoader`, + 'Block "BlockFrom" cannot be connected to from other blocks. Its blocktype "TestFileExtractor" has input type "None".', expect.any(Object), ); }); diff --git a/libs/language-server/src/lib/validation/checks/pipe-definition.ts b/libs/language-server/src/lib/validation/checks/pipe-definition.ts index fbb24e5d5..c1fc581dd 100644 --- a/libs/language-server/src/lib/validation/checks/pipe-definition.ts +++ b/libs/language-server/src/lib/validation/checks/pipe-definition.ts @@ -8,7 +8,7 @@ /* eslint-disable @typescript-eslint/no-unnecessary-condition */ import { PipeDefinition } from '../../ast/generated/ast'; -import { createSemanticPipes } from '../../ast/wrappers/pipe-wrapper'; +import { createWrappersFromPipeChain } from '../../ast/wrappers/pipe-wrapper'; import { BlockTypeWrapper } from '../../ast/wrappers/typed-object/blocktype-wrapper'; import { ValidationContext } from '../validation-context'; @@ -23,10 +23,10 @@ function checkBlockCompatibility( pipe: PipeDefinition, context: ValidationContext, ): void { - const semanticPipes = createSemanticPipes(pipe); - for (const semanticPipe of semanticPipes) { - const fromBlockTypeDefinition = semanticPipe.from?.type; - const toBlockTypeDefinition = semanticPipe.to?.type; + const pipeWrappers = createWrappersFromPipeChain(pipe); + for (const pipeWrapper of pipeWrappers) { + const fromBlockTypeDefinition = pipeWrapper.from?.type; + const toBlockTypeDefinition = pipeWrapper.to?.type; if ( !BlockTypeWrapper.canBeWrapped(fromBlockTypeDefinition) || @@ -37,12 +37,23 @@ function checkBlockCompatibility( const fromBlockType = new BlockTypeWrapper(fromBlockTypeDefinition); const toBlockType = new BlockTypeWrapper(toBlockTypeDefinition); - if (fromBlockType.hasOutput() && toBlockType.hasInput()) { - if (!fromBlockType.canBeConnectedTo(toBlockType)) { - const errorMessage = `The output type "${fromBlockType.outputType}" of ${fromBlockType.type} is incompatible with the input type "${toBlockType.inputType}" of ${toBlockType.type}`; - context.accept('error', errorMessage, semanticPipe.getFromDiagnostic()); - context.accept('error', errorMessage, semanticPipe.getToDiagnostic()); - } + const isFromBlockLoader = !fromBlockType.hasOutput(); + const isToBlockExtractor = !toBlockType.hasInput(); + + if (isFromBlockLoader) { + const errorMessage = `Block "${pipeWrapper.from?.name}" cannot be connected to other blocks. Its blocktype "${fromBlockType.astNode.name}" has output type "${fromBlockType.outputType}".`; + context.accept('error', errorMessage, pipeWrapper.getFromDiagnostic()); + } + + if (isToBlockExtractor) { + const errorMessage = `Block "${pipeWrapper.to?.name}" cannot be connected to from other blocks. Its blocktype "${toBlockType.astNode.name}" has input type "${toBlockType.inputType}".`; + context.accept('error', errorMessage, pipeWrapper.getToDiagnostic()); + } + + if (!fromBlockType.canBeConnectedTo(toBlockType)) { + const errorMessage = `The output type "${fromBlockType.outputType}" of block "${pipeWrapper.from?.name}" (of type "${fromBlockType.astNode.name}") is not compatible with the input type "${toBlockType.inputType}" of block "${pipeWrapper.to?.name}" (of type "${toBlockType.astNode.name}")`; + context.accept('error', errorMessage, pipeWrapper.getFromDiagnostic()); + context.accept('error', errorMessage, pipeWrapper.getToDiagnostic()); } } } diff --git a/libs/language-server/src/lib/validation/checks/pipeline-definition.spec.ts b/libs/language-server/src/lib/validation/checks/pipeline-definition.spec.ts index 9b1dee65e..5988ad1e7 100644 --- a/libs/language-server/src/lib/validation/checks/pipeline-definition.spec.ts +++ b/libs/language-server/src/lib/validation/checks/pipeline-definition.spec.ts @@ -63,7 +63,7 @@ describe('Validation of PipelineDefinition', () => { validationAcceptorMock.mockReset(); }); - it('should diagnose error on missing extractor block', async () => { + it('should diagnose error on missing starting block (no blocks)', async () => { const text = readJvTestAsset( 'pipeline-definition/invalid-empty-pipeline.jv', ); @@ -78,6 +78,21 @@ describe('Validation of PipelineDefinition', () => { ); }); + it('should diagnose error on missing starting block (no pipes)', async () => { + const text = readJvTestAsset( + 'pipeline-definition/invalid-pipeline-only-blocks.jv', + ); + + await parseAndValidatePipeline(text); + + expect(validationAcceptorMock).toHaveBeenCalledTimes(2); // one warning for unused blocks + expect(validationAcceptorMock).toHaveBeenCalledWith( + 'error', + `An extractor block is required for this pipeline`, + expect.any(Object), + ); + }); + it('should have no error on valid pipeline', async () => { const text = readJvTestAsset('pipeline-definition/valid-pipeline.jv'); @@ -85,4 +100,33 @@ describe('Validation of PipelineDefinition', () => { expect(validationAcceptorMock).toHaveBeenCalledTimes(0); }); + + it('should diagnose error on block as input for multiple pipes', async () => { + const text = readJvTestAsset( + 'pipeline-definition/invalid-block-as-multiple-pipe-inputs.jv', + ); + + await parseAndValidatePipeline(text); + + expect(validationAcceptorMock).toHaveBeenCalledTimes(2); + expect(validationAcceptorMock).toHaveBeenNthCalledWith( + 2, + 'error', + 'At most one pipe can be connected to the input of a block. Currently, the following 2 blocks are connected via pipes: "BlockFrom1", "BlockFrom2"', + expect.any(Object), + ); + }); + + it('should diagnose error on block without pipe', async () => { + const text = readJvTestAsset('pipeline-definition/invalid-missing-pipe.jv'); + + await parseAndValidatePipeline(text); + + expect(validationAcceptorMock).toHaveBeenCalledTimes(2); // one error since missing extractor + expect(validationAcceptorMock).toHaveBeenCalledWith( + 'warning', + 'A pipe should be connected to the output of this block', + expect.any(Object), + ); + }); }); diff --git a/libs/language-server/src/lib/validation/checks/pipeline-definition.ts b/libs/language-server/src/lib/validation/checks/pipeline-definition.ts index c3260e748..4c3811b8b 100644 --- a/libs/language-server/src/lib/validation/checks/pipeline-definition.ts +++ b/libs/language-server/src/lib/validation/checks/pipeline-definition.ts @@ -2,8 +2,12 @@ // // SPDX-License-Identifier: AGPL-3.0-only -import { PipelineDefinition } from '../../ast/generated/ast'; -import { collectStartingBlocks } from '../../ast/model-util'; +import { BlockTypeWrapper, PipeWrapper, PipelineWrapper } from '../../ast'; +import { + BlockDefinition, + CompositeBlocktypeDefinition, + PipelineDefinition, +} from '../../ast/generated/ast'; import { ValidationContext } from '../validation-context'; import { checkUniqueNames } from '../validation-util'; @@ -16,13 +20,21 @@ export function validatePipelineDefinition( checkUniqueNames(pipeline.transforms, context); checkUniqueNames(pipeline.valuetypes, context); checkUniqueNames(pipeline.constraints, context); + + checkMultipleBlockInputs(pipeline, context); + checkDefinedBlocksAreUsed(pipeline, context); } function checkStartingBlocks( pipeline: PipelineDefinition, context: ValidationContext, ): void { - const startingBlocks = collectStartingBlocks(pipeline); + if (!PipelineWrapper.canBeWrapped(pipeline)) { + return; + } + const pipelineWrapper = new PipelineWrapper(pipeline); + + const startingBlocks = pipelineWrapper.getStartingBlocks(); if (startingBlocks.length === 0) { context.accept( 'error', @@ -34,3 +46,136 @@ function checkStartingBlocks( ); } } + +export function checkMultipleBlockInputs( + pipeline: PipelineDefinition | CompositeBlocktypeDefinition, + context: ValidationContext, +): void { + if (!PipelineWrapper.canBeWrapped(pipeline)) { + return; + } + const pipelineWrapper = new PipelineWrapper(pipeline); + + const startingBlocks = pipelineWrapper.getStartingBlocks(); + let alreadyMarkedPipes: PipeWrapper[] = []; + for (const startingBlock of startingBlocks) { + alreadyMarkedPipes = doCheckMultipleBlockInputs( + pipelineWrapper, + startingBlock, + alreadyMarkedPipes, + context, + ); + } +} + +/** + * Inner method to check recursively whether blocks in a pipeline have multiple inputs + * @param pipelineWrapper The wrapping pipeline + * @param block The current block + * @param alreadyMarkedPipes List of already visited pipes to avoid duplicate errors + * @param context The validation context + * @returns the updated @alreadyMarkedPipes with all marked pipes + */ +function doCheckMultipleBlockInputs( + pipelineWrapper: PipelineWrapper< + PipelineDefinition | CompositeBlocktypeDefinition + >, + block: BlockDefinition, + alreadyMarkedPipes: Array, + context: ValidationContext, +): Array { + const pipesFromParents = pipelineWrapper.getIngoingPipes(block); + if (pipesFromParents.length > 1) { + const parentBlockNames = pipesFromParents.map( + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + (pipe) => '"' + pipe.from?.name + '"', + ); + for (const pipe of pipesFromParents) { + const wasAlreadyMarked = alreadyMarkedPipes.some((x) => pipe.equals(x)); + if (wasAlreadyMarked) { + continue; + } + + context.accept( + 'error', + `At most one pipe can be connected to the input of a block. Currently, the following ${ + pipesFromParents.length + } blocks are connected via pipes: ${parentBlockNames.join(', ')}`, + pipe.getToDiagnostic(), + ); + + alreadyMarkedPipes.push(pipe); + } + } + + const children = pipelineWrapper.getChildBlocks(block); + for (const child of children) { + alreadyMarkedPipes = doCheckMultipleBlockInputs( + pipelineWrapper, + child, + alreadyMarkedPipes, + context, + ); + } + + return alreadyMarkedPipes; +} + +export function checkDefinedBlocksAreUsed( + pipeline: PipelineDefinition | CompositeBlocktypeDefinition, + context: ValidationContext, +): void { + if (!PipelineWrapper.canBeWrapped(pipeline)) { + return; + } + const pipelineWrapper = new PipelineWrapper(pipeline); + + const containedBlocks = pipeline.blocks; + for (const block of containedBlocks) { + doCheckDefinedBlockIsUsed(pipelineWrapper, block, context); + } +} + +function doCheckDefinedBlockIsUsed( + pipelineWrapper: PipelineWrapper< + PipelineDefinition | CompositeBlocktypeDefinition + >, + block: BlockDefinition, + context: ValidationContext, +): void { + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (block.type === undefined || !BlockTypeWrapper.canBeWrapped(block.type)) { + return; + } + const blockType = new BlockTypeWrapper(block.type); + + const isExtractorBlock = !blockType.hasInput(); + if (!isExtractorBlock) { + const parents = pipelineWrapper.getParentBlocks(block); + if (parents.length === 0) { + context.accept( + 'warning', + `A pipe should be connected to the input of this block`, + { + node: block, + property: 'name', + }, + ); + } + } + + const isLoaderBlock = !blockType.hasOutput(); + if (!isLoaderBlock) { + const children = pipelineWrapper.getChildBlocks(block); + if (children.length === 0) { + context.accept( + 'warning', + `A pipe should be connected to the output of this block`, + { + node: block, + property: 'name', + }, + ); + } + } +} diff --git a/libs/language-server/src/stdlib/CSVExtractor.jv b/libs/language-server/src/stdlib/CSVExtractor.jv index 29c88be6f..066a1bbff 100644 --- a/libs/language-server/src/stdlib/CSVExtractor.jv +++ b/libs/language-server/src/stdlib/CSVExtractor.jv @@ -10,22 +10,24 @@ composite blocktype CSVExtractor { property delimiter oftype text: ','; property enclosing oftype text: ''; property enclosingEscape oftype text: ''; + property encoding oftype text: 'utf-8'; input inputPort oftype None; output outputPort oftype Sheet; - block FileExtractor oftype HttpExtractor { url: url; } - block FileTextInterpreter oftype TextFileInterpreter {} + inputPort + -> FileExtractor + -> FileCSVInterpreter + -> outputPort; + + block FileExtractor oftype HttpExtractor { + url: url; + } - block FileCSVInterpreter oftype CSVInterpreter { + block FileCSVInterpreter oftype CSVFileInterpreter { delimiter: delimiter; enclosing: enclosing; enclosingEscape: enclosingEscape; + enoding: encoding; } - - inputPort - ->FileExtractor - ->FileTextInterpreter - ->FileCSVInterpreter - ->outputPort; } diff --git a/libs/language-server/src/stdlib/CSVFileInterpreter.jv b/libs/language-server/src/stdlib/CSVFileInterpreter.jv new file mode 100644 index 000000000..4bedc3ef6 --- /dev/null +++ b/libs/language-server/src/stdlib/CSVFileInterpreter.jv @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +/** +* A CSVFileInterpreter interprets a file as CSV +*/ +composite blocktype CSVFileInterpreter { + property delimiter oftype text: ','; + property enclosing oftype text: ''; + property enclosingEscape oftype text: ''; + property enoding oftype text: 'utf-8'; + + input inputPort oftype File; + output outputPort oftype Sheet; + + inputPort + -> FileTextInterpreter + -> FileCSVInterpreter + -> outputPort; + + block FileTextInterpreter oftype TextFileInterpreter { + encoding: enoding; + } + + block FileCSVInterpreter oftype CSVInterpreter { + delimiter: delimiter; + enclosing: enclosing; + enclosingEscape: enclosingEscape; + } +} diff --git a/libs/language-server/src/stdlib/GTFSExtractor.jv b/libs/language-server/src/stdlib/GTFSExtractor.jv index b1741e36a..f1ba6aba1 100644 --- a/libs/language-server/src/stdlib/GTFSExtractor.jv +++ b/libs/language-server/src/stdlib/GTFSExtractor.jv @@ -11,12 +11,12 @@ composite blocktype GTFSExtractor { input inputPort oftype None; output outputPort oftype FileSystem; + inputPort + -> FileExtractor + -> ZipArchiveInterpreter + -> outputPort; + block FileExtractor oftype HttpExtractor { url: url; } block ZipArchiveInterpreter oftype ArchiveInterpreter { archiveType: "zip"; } - - inputPort - ->FileExtractor - ->ZipArchiveInterpreter - ->outputPort; } diff --git a/libs/language-server/src/test/assets/composite-blocktype-definition/invalid-block-as-multiple-pipe-inputs.jv b/libs/language-server/src/test/assets/composite-blocktype-definition/invalid-block-as-multiple-pipe-inputs.jv new file mode 100644 index 000000000..8d5a20f54 --- /dev/null +++ b/libs/language-server/src/test/assets/composite-blocktype-definition/invalid-block-as-multiple-pipe-inputs.jv @@ -0,0 +1,27 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +composite blocktype TestBlockType { + input inPort oftype None; + output outPort oftype None; + + block BlockTo oftype TestTableLoader { } + + block BlockFrom1 oftype TestFileExtractor { } + + block BlockFrom2 oftype TestFileExtractor { } + + inPort -> BlockFrom1 -> BlockTo -> outPort; + inPort -> BlockFrom2 -> BlockTo -> outPort; +} + +builtin blocktype TestFileExtractor { + input inPort oftype None; + output outPort oftype Table; +} + +builtin blocktype TestTableLoader { + input inPort oftype Table; + output outPort oftype None; +} diff --git a/libs/language-server/src/test/assets/composite-blocktype-definition/invalid-composite-blocktype-multiple-pipelines.jv b/libs/language-server/src/test/assets/composite-blocktype-definition/invalid-composite-blocktype-multiple-pipelines.jv index 60e4550ae..8ddd60f53 100644 --- a/libs/language-server/src/test/assets/composite-blocktype-definition/invalid-composite-blocktype-multiple-pipelines.jv +++ b/libs/language-server/src/test/assets/composite-blocktype-definition/invalid-composite-blocktype-multiple-pipelines.jv @@ -5,23 +5,19 @@ composite blocktype TestBlock { input inputName oftype None; - output outputName oftype Sheet; + output outputName oftype TextFile; block FileExtractor oftype HttpExtractor { url: 'url'; } - block FileTextInterpreter oftype TextFileInterpreter {} - - block FileCSVInterpreter oftype CSVInterpreter { - } + block FileTextInterpreter1 oftype TextFileInterpreter {} + block FileTextInterpreter2 oftype TextFileInterpreter {} inputName ->FileExtractor - ->FileTextInterpreter - ->FileCSVInterpreter + ->FileTextInterpreter1 ->outputName; inputName ->FileExtractor - ->FileTextInterpreter - ->FileCSVInterpreter + ->FileTextInterpreter2 ->outputName; } \ No newline at end of file diff --git a/libs/language-server/src/test/assets/composite-blocktype-definition/invalid-unconnected-block.jv b/libs/language-server/src/test/assets/composite-blocktype-definition/invalid-unconnected-block.jv new file mode 100644 index 000000000..252667e32 --- /dev/null +++ b/libs/language-server/src/test/assets/composite-blocktype-definition/invalid-unconnected-block.jv @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +composite blocktype TestBlockType { + input inPort oftype Table; + output outPort oftype File; + + block TestExtractor1 oftype TestFileExtractor { } + block TestExtractor2 oftype TestFileExtractor { } + + inPort -> TestExtractor1 -> outPort; +} + +builtin blocktype TestFileExtractor { + input inPort oftype Table; + output outPort oftype File; +} diff --git a/libs/language-server/src/test/assets/block-definition/invalid-output-block-as-input.jv b/libs/language-server/src/test/assets/pipe-definition/invalid-output-block-as-input.jv similarity index 79% rename from libs/language-server/src/test/assets/block-definition/invalid-output-block-as-input.jv rename to libs/language-server/src/test/assets/pipe-definition/invalid-output-block-as-input.jv index 40f836c9b..1bec9407d 100644 --- a/libs/language-server/src/test/assets/block-definition/invalid-output-block-as-input.jv +++ b/libs/language-server/src/test/assets/pipe-definition/invalid-output-block-as-input.jv @@ -3,14 +3,12 @@ // SPDX-License-Identifier: AGPL-3.0-only pipeline Pipeline { - block BlockTo oftype TestTableLoader { + block BlockTo oftype TestFileLoader { } block BlockFrom oftype TestFileExtractor { } - BlockFrom -> BlockTo; - BlockTo -> BlockFrom; } @@ -19,7 +17,7 @@ builtin blocktype TestFileExtractor { output outPort oftype File; } -builtin blocktype TestTableLoader { +builtin blocktype TestFileLoader { input inPort oftype File; output outPort oftype None; } diff --git a/libs/language-server/src/test/assets/pipe-definition/chained/invalid-pipe-between-blocktypes.jv b/libs/language-server/src/test/assets/pipe-definition/invalid-pipe-between-blocktypes.jv similarity index 100% rename from libs/language-server/src/test/assets/pipe-definition/chained/invalid-pipe-between-blocktypes.jv rename to libs/language-server/src/test/assets/pipe-definition/invalid-pipe-between-blocktypes.jv diff --git a/libs/language-server/src/test/assets/pipe-definition/single/invalid-pipe-between-blocktypes.jv b/libs/language-server/src/test/assets/pipe-definition/single/invalid-pipe-between-blocktypes.jv deleted file mode 100644 index 2494fb23a..000000000 --- a/libs/language-server/src/test/assets/pipe-definition/single/invalid-pipe-between-blocktypes.jv +++ /dev/null @@ -1,26 +0,0 @@ -// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg -// -// SPDX-License-Identifier: AGPL-3.0-only - -pipeline Pipeline { - block TestExtractor oftype TestFileExtractor { - } - - block TestLoader oftype TestTableLoader { - } - - pipe { - from: TestExtractor; - to: TestLoader; - } -} - -builtin blocktype TestFileExtractor { - input inPort oftype None; - output outPort oftype File; -} - -builtin blocktype TestTableLoader { - input inPort oftype Table; - output outPort oftype None; -} \ No newline at end of file diff --git a/libs/language-server/src/test/assets/pipe-definition/single/valid-unknown-blocktype.jv b/libs/language-server/src/test/assets/pipe-definition/single/valid-unknown-blocktype.jv deleted file mode 100644 index 786638dec..000000000 --- a/libs/language-server/src/test/assets/pipe-definition/single/valid-unknown-blocktype.jv +++ /dev/null @@ -1,16 +0,0 @@ -// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg -// -// SPDX-License-Identifier: AGPL-3.0-only - -pipeline Pipeline { - block UnknownOutput oftype UnknownOutputType { - } - - block UnknownInput oftype UnknownInputType { - } - - pipe { - from: UnknownOutput; - to: UnknownInput; - } -} diff --git a/libs/language-server/src/test/assets/pipe-definition/chained/valid-undefined-block.jv b/libs/language-server/src/test/assets/pipe-definition/valid-undefined-block.jv similarity index 100% rename from libs/language-server/src/test/assets/pipe-definition/chained/valid-undefined-block.jv rename to libs/language-server/src/test/assets/pipe-definition/valid-undefined-block.jv diff --git a/libs/language-server/src/test/assets/pipe-definition/chained/valid-unknown-blocktype.jv b/libs/language-server/src/test/assets/pipe-definition/valid-unknown-blocktype.jv similarity index 100% rename from libs/language-server/src/test/assets/pipe-definition/chained/valid-unknown-blocktype.jv rename to libs/language-server/src/test/assets/pipe-definition/valid-unknown-blocktype.jv diff --git a/libs/language-server/src/test/assets/block-definition/invalid-block-as-multiple-pipe-inputs.jv b/libs/language-server/src/test/assets/pipeline-definition/invalid-block-as-multiple-pipe-inputs.jv similarity index 64% rename from libs/language-server/src/test/assets/block-definition/invalid-block-as-multiple-pipe-inputs.jv rename to libs/language-server/src/test/assets/pipeline-definition/invalid-block-as-multiple-pipe-inputs.jv index 5529c0ef1..5347e081a 100644 --- a/libs/language-server/src/test/assets/block-definition/invalid-block-as-multiple-pipe-inputs.jv +++ b/libs/language-server/src/test/assets/pipeline-definition/invalid-block-as-multiple-pipe-inputs.jv @@ -3,15 +3,15 @@ // SPDX-License-Identifier: AGPL-3.0-only pipeline Pipeline { - block BlockTo oftype TestTableLoader { - } + block BlockTo oftype TestTableLoader { } - block BlockFrom oftype TestFileExtractor { - } + block BlockFrom1 oftype TestFileExtractor { } - BlockFrom -> BlockTo; + block BlockFrom2 oftype TestFileExtractor { } - BlockFrom -> BlockTo; + BlockFrom1 -> BlockTo; + + BlockFrom2 -> BlockTo; } builtin blocktype TestFileExtractor { diff --git a/libs/language-server/src/test/assets/block-definition/invalid-missing-pipe.jv b/libs/language-server/src/test/assets/pipeline-definition/invalid-missing-pipe.jv similarity index 100% rename from libs/language-server/src/test/assets/block-definition/invalid-missing-pipe.jv rename to libs/language-server/src/test/assets/pipeline-definition/invalid-missing-pipe.jv diff --git a/libs/language-server/src/test/assets/block-definition/valid-block-definition.jv b/libs/language-server/src/test/assets/pipeline-definition/invalid-pipeline-only-blocks.jv similarity index 51% rename from libs/language-server/src/test/assets/block-definition/valid-block-definition.jv rename to libs/language-server/src/test/assets/pipeline-definition/invalid-pipeline-only-blocks.jv index 272e0f9f7..3c6115bfd 100644 --- a/libs/language-server/src/test/assets/block-definition/valid-block-definition.jv +++ b/libs/language-server/src/test/assets/pipeline-definition/invalid-pipeline-only-blocks.jv @@ -3,21 +3,10 @@ // SPDX-License-Identifier: AGPL-3.0-only pipeline Pipeline { - block TestExtractor oftype TestFileExtractor { - } - - block TestLoader oftype TestTableLoader { - } - - TestExtractor -> TestLoader; + block TestExtractor oftype TestFileExtractor {} } builtin blocktype TestFileExtractor { input inPort oftype None; output outPort oftype File; -} - -builtin blocktype TestTableLoader { - input inPort oftype File; - output outPort oftype None; } \ No newline at end of file diff --git a/libs/language-server/src/test/assets/pipeline-definition/valid-pipeline.jv b/libs/language-server/src/test/assets/pipeline-definition/valid-pipeline.jv index 26af22d9e..c63c7e306 100644 --- a/libs/language-server/src/test/assets/pipeline-definition/valid-pipeline.jv +++ b/libs/language-server/src/test/assets/pipeline-definition/valid-pipeline.jv @@ -3,11 +3,18 @@ // SPDX-License-Identifier: AGPL-3.0-only pipeline Pipeline { - block TestExtractor oftype TestFileExtractor { - } + TestExtractor -> TestLoader; + + block TestExtractor oftype TestFileExtractor {} + block TestLoader oftype TestFileLoader {} } builtin blocktype TestFileExtractor { input inPort oftype None; output outPort oftype File; +} + +builtin blocktype TestFileLoader { + input inPort oftype File; + output outPort oftype None; } \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 32134f5d1..2ee7c6822 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "jayvee", - "version": "0.2.0", + "version": "0.3.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "jayvee", - "version": "0.2.0", + "version": "0.3.0", "dependencies": { "@docusaurus/core": "2.4.1", "@docusaurus/preset-classic": "2.4.1", diff --git a/package.json b/package.json index 061b9a889..4b782b940 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "jayvee", - "version": "0.2.0", + "version": "0.3.0", "scripts": { "nx": "nx", "format": "nx format:write", diff --git a/rfc/0014-attribute-based-valuetypes/README.md b/rfc/0014-attribute-based-valuetypes/README.md new file mode 100644 index 000000000..96c530284 --- /dev/null +++ b/rfc/0014-attribute-based-valuetypes/README.md @@ -0,0 +1,88 @@ + + +# RFC 0014: Attributed-based syntax for defining value types + +| | | +|---|---| +| Feature Tag | `attribute-based value types` | +| Status | `DRAFT` | +| Responsible | `dirkriehle` | + + +## Summary + +I'd like to allow the definition of user-defined single-attribute value types. In addition to the current syntax, in which an underlying value type is referenced through 'oftype', it does not use instantiation/inheritance but rather composition. + +## Motivation + +The purpose of using composition is that it is + +1. more similar to traditional approaches and +2. is needed anyway for multi-attribute value types. + +## Explanation + +To define a value type like CorrelationCoefficient and its range of -1 to +1, we have to write: + +```jayvee +valuetype CorrelationCoefficient oftype decimal { + constraints: [ + MinusOneToPlusOneRange, + ]; +} +``` + +This syntax is smart in that you don't have to list and name an attribute but rather rely on an implicit 'value' attribute. Still, I propose to make that attribute explicit. New syntax would be: + +```jayvee +valuetype CorrelationCoefficient { + attributes: [ + value oftype decimal; + ]; + constraints: [ + MinusOneToPlusOneRange; // Don't know how to attach this to value attribute + ]; +} +``` + +While more verbose, it prepares the way for + +```jayvee +valuetype Money { + attributes: [ + amount oftype decimal; + currency oftype Currency; + ]; +} +``` +which we'll need anyway. Conceivably, the step to multi-attribute value types could be merged with this one, but I simply wanted to try the RFC process rather than keep sending email ;-) + + + +## Drawbacks + +1. Introduces a redundant syntax, +2. Creates extra work/may be too difficult, and +3. May be disruptive to how the language currently works. + +## Alternatives + +Conceivably, we could use multiple inheritance for multi-attribute value types... just joking. + +## Possible Future Changes/Enhancements + +This proposal is to prepare the way for multi-attribute value types. diff --git a/rfc/0015-multi-file-jayvee/0015-multi-file-jayvee.md b/rfc/0015-multi-file-jayvee/0015-multi-file-jayvee.md new file mode 100644 index 000000000..949d1d0aa --- /dev/null +++ b/rfc/0015-multi-file-jayvee/0015-multi-file-jayvee.md @@ -0,0 +1,200 @@ + + +# RFC 0015: Multi-file Jayvee + +| | | +| ----------- | --------------- | --------------------------------------------------------------- | +| Feature Tag | `multi-file` | +| Status | `ACCEPTED` | | +| Responsible | `georg-schwarz` | | + + + +## Summary + +This RFC introduces the possibility of distributing a Jayvee program over multiple files. +This feature will foster reuse of valuetypes, blocks, and other elements. +Inherent to this feature is a concept of how scoping and naming is handled for nested structures. +This RFC introduces two concepts: + +- Element usage from other files, and +- Packages + +## Motivation + +Right now, Jayvee users can only pack their whole Jayvee model into one file. +The challenge is two-fold: + +1. Larger projects will become unmaintainable quite quickly, as the elements cannot be organized into multiple files. +2. Without distribution to multiple files, there is no possibility to later reuse models of other projects. + +## Explanation + +### Element visibility + +We distinguish different kinds of visibilities of elements: + +- `file-private`: usable only within the same file +- `file-published`: usable also in other files of same project +- `package-private`: usable only within the package +- `package-published`: usable also in other locations (since `package` is always `file-published`) + +By introducing the concept of `packages`, most elements can be defined on three scope levels in a Jayvee file: + +1. file-scope (not contained in a further structure, like a pipeline or a package) +2. pipeline-scope (contained in a pipeline) +3. (new) package-scope (contained in a package) + +The name of an element is given by its definition. +The **qualified name** is constructed by prepending container structures in this pattern: `.`, e.g., `MyDomainPackage.MyDomainSpecificValuetype`. + +**Access paths:** + +- _file-scope_ elements can access + - ✅ _file-scope_ elements by their name + - ✅ _package-scope_ elements by their qualified name (`packageName.elementName`) + - ❌ no _pipeline-scope_ elements +- _package-scope_ elements can access + - ✅ _file-scope_ elements by their name + - ✅ elements of the same _package-scope_ by their name + - ✅ elements of another _package-scope_ by their qualified name + - ❌ no _pipeline-scope_ elements +- _pipeline-scope_ elements can access + - ✅ _file-scope_ elements by their name + - ✅ elements within the same _pipeline-scope_ by their name + - ❌ no elements of another _pipeline-scope_ + - ✅ _package-scope_ elements by their qualified name + +**Used elements** of different files are handled **as if they were defined at file scope level**. + +### Publishing elements for usage elsewhere (within the project) + +For publishing single elements, the RFC introduces the keyword `publish` to indicate the visibility `file-published`. +All elements within a file are not published per default, visibility `file-private`. +Explicitly declaring an element as published allows for usage elsewhere. + +**Example publish** + +``` +// define and publish in one syntax +publish valuetype MyValueType1 { + // ... details +} + +// define first +valuetype MyValueType2 { + // ... details +} + +// publish later +publish MyValueType2; + +// publish later under a different name +publish MyValueType2 as MyValueType3; +``` + +### Packages: bundling elements to a package for decoupled usage + +For bundling and publishing elements, the RFC introduces a new concept called `packages`. +A `package` can include `Valuetype`s, `Block`s, `BlockType`s, `Constraint`s, `Transform`s, and further `Package`s. +A package must be of visibility `file-published` and, thus, requires the keyword `publish`. +Elements within a package can be of visibility `package-published` by using the `publish` keyword, or are `package-private` per default. +`package`s within `package`s must be `publish`ed as a consequence. + +**Example package** + +``` +publish package MyDomainPackage { + // definition of a new valuetype as part of the package + publish valuetype MyDomainSpecificValuetype1 { + // ... details of valuetype + } + + // reference to an existing valuetype to make it part of the package + publish MyDomainSpecificValuetype2; + + // ... possibly more elements +} +``` + +The advantage of bundling elements into a `package` is the decoupling from the internal file system structure and logically grouping related elements into one namespace. +Rather than accessing files directly (and needing knowledge what is element is located in which files) users can simply use a whole package with all its elements (and don't need to know in which file the element is originally defined). + +### Using elements + +Only `file-published` (with keyword `publish`) elements can be `use`d in other files. + +#### Usage paths + +When using elements of a file or a package, we have to define where the elements are located. +Jayvee provides the following possibilities: + +- a relative file path, e.g., `use * from './path/to/file.jv';` + +The `use` of elements via a file path decouples from the file system structure by using the element name or a defined alias instead of the file path within the file. + +#### Using published elements of a file (within the same project) + +``` +use * from './path/to/location.jv'; // use all published elements from the file, access via qualified name as if it would be defined at the _file-scope_ +use * as MyWrappingNamespace from './path/to/location.jv'; // use all published elements from the file, access via qualified name as if it was defined in an artificial _package-scope_ (adding a prefix to the qualified name) +use { MyDomainSpecificValuetype1 } from './path/to/location.jv'; // only use the published elements from the file, access via qualified name as if it would be defined at the _file-scope_ +use { MyDomainSpecificValuetype1 as Vt1} from './path/to/location.jv'; // only use the published elements from the file, access via qualified name using the alias +``` + +References to these used elements is by their qualified name (unless altered by an alias). + +#### Using a package + +The `use` syntax is similar to importing an element of a file. +Packages can only be used as a whole. + +``` +use { MyDomainPackage } from './path/to/location.jv'; // only use the named package, access via qualified name +use { + MyDomainPackage1, + MyDomainPackage2 +} from './path/to/location.jv'; // only use the named packages, access via qualified name +use { MyDomainPackage as MyPackageAlias} from './path/to/location.jv'; // only use the named package, access via qualified name using the alias +``` + +References to these used elements is by their qualified name (unless altered by an alias). + +## Decision rationale + +- keywords `publish` and `use` over `export` and `import` since they are less technical and better understandable for subject-matter experts +- concept of "namespaces" is separate from publishing them for reuse in other projects (this RFC only introduces the concept of "namespaces" by the keyword `package`) + +## Drawbacks + +- Two different sharing mechanisms (`publish` keyword, `package` concept`) +- Elements of a pipeline cannot be reused, leading to potentially more slim pipelines and a parallel package +- Langium might not support this scoping mechanism out-of-the-box (more complex implementation) + +## Alternatives + +- "use" syntax without braces, etc., `use MyDomainPackage1, MyDomainPackage2 from './path/to/file.jv';` +- switch around `use` and `from`: `from 'location' use { Element };` +- different keyword for publishing files and packages (e.g., `export`) +- different keyword for using files and packages (e.g., `import`) +- different keyword for renaming published / used elements (e.g., `called`) +- Rather call it `module` or `component` instead of `package` + +## Possible Future Changes/Enhancements + +- build out to use packages of other projects via a package-manager mechanism, e.g., by using an URL as location of a "use" statement + - alternative: introduce new concept `library` +- allow "using" single elements of a package instead of "using" the whole package +- allow additional usage paths, like + - absolute file paths, and + - org/repo combination at a central package registry, e.g., `use * from 'jv:my-org/my-repo';` diff --git a/tools/scripts/check-for-invalid-windows-paths.mjs b/tools/scripts/check-for-invalid-windows-paths.mjs new file mode 100644 index 000000000..c4b996b5b --- /dev/null +++ b/tools/scripts/check-for-invalid-windows-paths.mjs @@ -0,0 +1,56 @@ +// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg +// +// SPDX-License-Identifier: AGPL-3.0-only + +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +/** + * Checks if a path contains characters not valid for windows. + * @param filePath {string} + * @returns {boolean} + */ +function containsInvalidCharactersForWindows(filePath) { + const invalidCharsRegex = /[<>:"/\\|?*]/; + return invalidCharsRegex.test(filePath); +} + +/** + * Checks a directory for paths that are invalid in windows. + * @param currentDirectory {string} + * @return string[] + */ +function checkPaths(currentDirectory) { + /** @type {string[]} */ + const invalidPaths = []; + const files = fs.readdirSync(currentDirectory); + + files.forEach(file => { + const filePath = path.join(currentDirectory, file); + + if (containsInvalidCharactersForWindows(file)) { + // If new part of a path contains invalid characters, add the full relative path. + invalidPaths.push(filePath); + } + + if (fs.statSync(filePath).isDirectory()) { + invalidPaths.push(...checkPaths(filePath)); + } + }); + + return invalidPaths; +} + +// Main script +const currentDirectory = process.cwd(); +const invalidPaths = checkPaths(currentDirectory); + +if (invalidPaths.length > 0) { + console.log(`${invalidPaths.length} invalid path(s) for Windows found.`); + for (const invalidPath of invalidPaths) { + console.log(`- ${invalidPath}`) + } + process.exit(1); +} else { + console.log(`The path ${currentDirectory} only contains valid windows file names.`) +}