Feature/396 handle xlsx files

libs/execution/src/lib/debugging/debug-log-visitor.ts

@@ -5,6 +5,7 @@
 import { internalValueToString } from '@jvalue/jayvee-language-server';
 
 import { Logger } from '../logger';
+import { Workbook } from '../types';
 import { FileSystem } from '../types/io-types/filesystem';
 import { BinaryFile } from '../types/io-types/filesystem-node-file-binary';
 import { TextFile } from '../types/io-types/filesystem-node-file-text';
@@ -134,6 +135,12 @@ export class DebugLogVisitor implements IoTypeVisitor<void> {
 
     this.log('... (omitted in peek mode)');
   }
+  visitWorkbook(workbook: Workbook): void {
+    this.log(`WorkSheets in WorkBook:`);
+    workbook
+      .getSheets()
+      .forEach((sheet) => console.log(`WorkSheet: ${sheet.getSheetName()}`));
+  }
 
   private log(text: string): void {
     this.logger.logDebug(`[${this.logPrefix}] ${text}`);

libs/execution/src/lib/types/io-types/index.ts

@@ -6,6 +6,7 @@ export * from './filesystem';
 export * from './filesystem-inmemory';
 export * from './io-type-implementation';
 export * from './none';
+export * from './workbook';
 export * from './sheet';
 export * from './table';
 export * from './filesystem-node-file-binary';

libs/execution/src/lib/types/io-types/io-type-implementation.ts

@@ -10,6 +10,7 @@ import { type TextFile } from './filesystem-node-file-text';
 import { type None } from './none';
 import { type Sheet } from './sheet';
 import { type Table } from './table';
+import { type Workbook } from './workbook';
 
 export interface IOTypeImplementation<T extends IOType = IOType> {
   ioType: T;
@@ -20,6 +21,7 @@ export interface IOTypeImplementation<T extends IOType = IOType> {
 export interface IoTypeVisitor<R = unknown> {
   visitTable(table: Table): R;
   visitSheet(sheet: Sheet): R;
+  visitWorkbook(workbook: Workbook): R;
   visitNone(none: None): R;
   visitFileSystem(fileSystem: FileSystem): R;
   visitBinaryFile(binaryFile: BinaryFile): R;

libs/execution/src/lib/types/io-types/sheet.ts

@@ -22,13 +22,17 @@ export class Sheet implements IOTypeImplementation<IOType.SHEET> {
   public readonly ioType = IOType.SHEET;
   private numberOfRows: number;
   private numberOfColumns: number;
-  constructor(private data: string[][]) {
+  constructor(private data: string[][], private sheetName?: string) {
     this.numberOfRows = data.length;
     this.numberOfColumns = data.reduce((prev, curr) => {
       return curr.length > prev ? curr.length : prev;
     }, 0);
   }
 
+  getSheetName(): string {
+    return this.sheetName ?? '';
+  }
+
   getData(): ReadonlyArray<ReadonlyArray<string>> {
     return this.data;
   }

libs/execution/src/lib/types/io-types/workbook.ts

@@ -0,0 +1,47 @@
+// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { strict as assert } from 'assert';
+
+import { IOType } from '@jvalue/jayvee-language-server';
+
+import { IOTypeImplementation, IoTypeVisitor } from './io-type-implementation';
+import { Sheet } from './sheet';
+
+export class Workbook implements IOTypeImplementation<IOType.WORKBOOK> {
+  public readonly ioType = IOType.WORKBOOK;
+  private sheets: Sheet[];
+  constructor() {
+    this.sheets = [];
+  }
+
+  getSheets(): ReadonlyArray<Sheet> {
+    return this.sheets;
+  }
+
+  getSheetByName(sheetName: string): Sheet {
+    const sheet = this.sheets.filter(
+      (sheet) => sheet.getSheetName() === sheetName,
+    )[0];
+    assert(sheet instanceof Sheet);
+    return sheet;
+  }
+
+  addSheet(sheet: Sheet) {
+    this.sheets.push(sheet);
+  }
+
+  addNewSheet(data: string[][], sheetName?: string) {
+    const sheetNameOrDefault = sheetName ?? `Sheet${this.sheets.length + 1}`;
+    if (
+      this.sheets.some((sheet) => sheet.getSheetName() === sheetNameOrDefault)
+    )
+      return;
+    this.addSheet(new Sheet(data, sheetNameOrDefault));
+  }
+
+  acceptVisitor<R>(visitor: IoTypeVisitor<R>): R {
+    return visitor.visitWorkbook(this);
+  }
+}

libs/extensions/tabular/exec/src/extension.ts

@@ -12,8 +12,10 @@ import { CellWriterExecutor } from './lib/cell-writer-executor';
 import { ColumnDeleterExecutor } from './lib/column-deleter-executor';
 import { CSVInterpreterExecutor } from './lib/csv-interpreter-executor';
 import { RowDeleterExecutor } from './lib/row-deleter-executor';
+import { SheetPickerExecutor } from './lib/sheet-picker-executor';
 import { TableInterpreterExecutor } from './lib/table-interpreter-executor';
 import { TableTransformerExecutor } from './lib/table-transformer-executor';
+import { XLSXInterpreterExecutor } from './lib/xlsx-interpreter-executor';
 
 export class TabularExecExtension implements JayveeExecExtension {
   getBlockExecutors(): BlockExecutorClass[] {
@@ -25,6 +27,8 @@ export class TabularExecExtension implements JayveeExecExtension {
       TableInterpreterExecutor,
       CSVInterpreterExecutor,
       TableTransformerExecutor,
+      XLSXInterpreterExecutor,
+      SheetPickerExecutor,
     ];
   }
 }

libs/extensions/tabular/exec/src/lib/sheet-picker-executor.ts

@@ -0,0 +1,39 @@
+// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import * as R from '@jvalue/jayvee-execution';
+import {
+  AbstractBlockExecutor,
+  BlockExecutorClass,
+  ExecutionContext,
+  Sheet,
+  Workbook,
+  implementsStatic,
+} from '@jvalue/jayvee-execution';
+import { IOType, PrimitiveValuetypes } from '@jvalue/jayvee-language-server';
+
+@implementsStatic<BlockExecutorClass>()
+export class SheetPickerExecutor extends AbstractBlockExecutor<
+  IOType.WORKBOOK,
+  IOType.SHEET
+> {
+  public static readonly type = 'SheetPicker';
+
+  constructor() {
+    super(IOType.WORKBOOK, IOType.SHEET);
+  }
+
+  // eslint-disable-next-line @typescript-eslint/require-await
+  async doExecute(
+    workbook: Workbook,
+    context: ExecutionContext,
+  ): Promise<R.Result<Sheet | null>> {
+    const sheetName = context.getPropertyValue(
+      'sheetName',
+      PrimitiveValuetypes.Text,
+    );
+    const sheet = workbook.getSheetByName(sheetName);
+    return R.ok(sheet);
+  }
+}

libs/extensions/tabular/exec/src/lib/xlsx-interpreter-executor.ts

@@ -0,0 +1,61 @@
+// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { strict as assert } from 'assert';
+
+import * as R from '@jvalue/jayvee-execution';
+import {
+  AbstractBlockExecutor,
+  BinaryFile,
+  BlockExecutorClass,
+  ExecutionContext,
+  Workbook,
+  implementsStatic,
+} from '@jvalue/jayvee-execution';
+import { IOType } from '@jvalue/jayvee-language-server';
+import * as xlsx from 'xlsx';
+
+@implementsStatic<BlockExecutorClass>()
+export class XLSXInterpreterExecutor extends AbstractBlockExecutor<
+  IOType.FILE,
+  IOType.WORKBOOK
+> {
+  public static readonly type = 'XLSXInterpreter';
+
+  constructor() {
+    super(IOType.FILE, IOType.WORKBOOK);
+  }
+
+  async doExecute(
+    file: BinaryFile,
+    context: ExecutionContext,
+  ): Promise<R.Result<Workbook>> {
+    context.logger.logDebug(`reading from xlsx file`);
+    const workBookFromFile = xlsx.read(file.content, { dense: true });
+    const workbook = new Workbook();
+    for (const workSheetName of workBookFromFile.SheetNames) {
+      const workSheet = workBookFromFile.Sheets[workSheetName];
+      assert(workSheet !== undefined);
+
+      const workSheetDataArray = Array.prototype.map.call<
+        xlsx.WorkSheet,
+        [
+          callbackfn: (
+            value: xlsx.CellObject[],
+            index: number,
+            array: xlsx.WorkSheet[],
+          ) => string[],
+        ],
+        string[][]
+      >(workSheet, (row: xlsx.CellObject[]): string[] => {
+        return row.map<string>((cell: xlsx.CellObject) => {
+          return cell.v?.toString() ?? '';
+        });
+      });
+
+      workbook.addNewSheet(workSheetDataArray, workSheetName);
+    }
+    return Promise.resolve(R.ok(workbook));
+  }
+}

libs/extensions/tabular/lang/src/extension.ts

@@ -13,8 +13,10 @@ import { CellWriterMetaInformation } from './lib/cell-writer-meta-inf';
 import { ColumnDeleterMetaInformation } from './lib/column-deleter-meta-inf';
 import { CSVInterpreterMetaInformation } from './lib/csv-interpreter-meta-inf';
 import { RowDeleterMetaInformation } from './lib/row-deleter-meta-inf';
+import { SheetPickerMetaInformation } from './lib/sheet-picker-meta-inf';
 import { TableInterpreterMetaInformation } from './lib/table-interpreter-meta-inf';
 import { TableTransformerMetaInformation } from './lib/table-transformer-meta-inf';
+import { XLSXInterpreterMetaInformation } from './lib/xlsx-interpreter-meta-inf';
 
 export class TabularLangExtension implements JayveeLangExtension {
   getBlockMetaInf(): Array<ConstructorClass<BlockMetaInformation>> {
@@ -26,6 +28,8 @@ export class TabularLangExtension implements JayveeLangExtension {
       TableInterpreterMetaInformation,
       CSVInterpreterMetaInformation,
       TableTransformerMetaInformation,
+      XLSXInterpreterMetaInformation,
+      SheetPickerMetaInformation,
     ];
   }
 }

libs/extensions/tabular/lang/src/lib/sheet-picker-meta-inf.ts

@@ -0,0 +1,44 @@
+// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import {
+  BlockMetaInformation,
+  IOType,
+  PrimitiveValuetypes,
+} from '@jvalue/jayvee-language-server';
+
+export class SheetPickerMetaInformation extends BlockMetaInformation {
+  constructor() {
+    super(
+      // How the block type should be called:
+      'SheetPicker',
+      // Property definitions:
+      {
+        sheetName: {
+          type: PrimitiveValuetypes.Text,
+          docs: {
+            description: 'The name of the sheet to select.',
+          },
+        },
+      },
+      // Input type:
+      IOType.WORKBOOK,
+
+      // Output type:
+      IOType.SHEET,
+    );
+
+    this.docs.description =
+      'Selects one `Sheet` from a `Workbook` based on its `sheetName`. If no sheet matches the name, no output is created and the execution of the pipeline is aborted.';
+    this.docs.examples = [
+      {
+        code: `block AgencySheetPicker oftype SheetPicker {
+  sheetName: "AgencyNames";
+}`,
+        description:
+          'Tries to pick the sheet `AgencyNames` from the provided `Workbook`. If `AgencyNames` exists it is passed on as `Sheet`, if it does not exist the execution of the pipeline is aborted.',
+      },
+    ];
+  }
+}

libs/extensions/tabular/lang/src/lib/xlsx-interpreter-meta-inf.ts

@@ -0,0 +1,33 @@
+// SPDX-FileCopyrightText: 2023 Friedrich-Alexander-Universitat Erlangen-Nurnberg
+//
+// SPDX-License-Identifier: AGPL-3.0-only
+
+import { BlockMetaInformation, IOType } from '@jvalue/jayvee-language-server';
+
+export class XLSXInterpreterMetaInformation extends BlockMetaInformation {
+  constructor() {
+    super(
+      // How the block type should be called:
+      'XLSXInterpreter',
+      // Property definitions:
+      {},
+      // Input type:
+      IOType.FILE,
+
+      // Output type:
+      IOType.WORKBOOK,
+    );
+
+    this.docs.description =
+      "Interprets an input file as a xlsx-file and outputs a `Workbook` containing `Sheet`'s.";
+    this.docs.examples = [
+      {
+        code: blockExample,
+        description:
+          "Interprets an input file as a xlsx-file and outputs a `Workbook` containing `Sheet`'s.",
+      },
+    ];
+  }
+}
+const blockExample = `block AgencyXLSXInterpreter oftype XLSXInterpreter {  
+  }`;

libs/language-server/src/lib/ast/model-util.ts

@@ -145,6 +145,7 @@ export enum IOType {
   TEXT_FILE = 'TextFile',
   FILE_SYSTEM = 'FileSystem',
   SHEET = 'Sheet',
+  WORKBOOK = 'Workbook',
   TABLE = 'Table',
 }