diff --git a/src/lib/Generator.class.ts b/src/lib/Generator.class.ts index de2f6d4..e3f652f 100644 --- a/src/lib/Generator.class.ts +++ b/src/lib/Generator.class.ts @@ -78,7 +78,6 @@ class Generator extends EventEmitter { if ($this !== undefined) this.$thisList.push($this) const union: UnionPattern = { type: 'union', patterns: [] } const error = (e: any): Error => new Error(`The Generator did not run succesfully, it could not get the results from the endpoint ${this.source}: ${(e as Error).message}`) - // @mightymax problem - what if the this.$thisList length is smaller than the batchsize? => and cleanup$thisList() (like the end() function previously) is needed if (this.$thisList.length >= (batchSize ?? this.batchSize)) { if (this.source === '') this.source = getEngineSource(this.endpoint) const unionQuery = getSPARQLQuery(getSPARQLQueryString(this.query), "construct"); diff --git a/src/lib/tests/Generator.class.test.ts b/src/lib/tests/Generator.class.test.ts index 2145c87..a81932f 100644 --- a/src/lib/tests/Generator.class.test.ts +++ b/src/lib/tests/Generator.class.test.ts @@ -6,7 +6,6 @@ import * as chai from 'chai' import chaiAsPromised from 'chai-as-promised' import { NamedNode, Store } from "n3"; import type { LDWorkbenchConfiguration } from "../LDWorkbenchConfiguration.js"; -import * as fs from "fs" chai.use(chaiAsPromised) const expect = chai.expect @@ -54,63 +53,11 @@ describe('Generator Class', () => { }); // BUG when both the generator and iterator tests are running, it seems the iterator will never terminate describe('run', () => { - // BUG File class seems to create files of inconsistent filesizes due to buffer size difference - it.only('Should work in batchSize for pipeline\'s generator - test with output file', async function () { - // when using local test files timeout should be removed - this.timeout(4000) - const filePath = 'src/lib/tests/data/example-pipelineBatch.nt'; - - - const batchConfiguration: LDWorkbenchConfiguration = { - name: 'Example Pipeline Batch', - description: 'This is an example pipeline. It uses files that are available in this repository and SPARQL endpoints that should work.\n', - destination: "file://" + filePath, - stages: [ - { - name: 'Stage 1', - iterator: { - query: 'file://static/example/iterator-stage-1.rq', - endpoint: 'https://api.triplydb.com/datasets/Triply/iris/services/demo-service/sparql' - }, - generator: { - query: 'file://static/example/generator-stage-1.rq', - // adjust batchsize for test here - batchSize: 4 - } - } - ] - } - - const pipelineBatch = new Pipeline(batchConfiguration) - async function runPipelineWithPromise(): Promise { - let batchPipelineEnd = false - return new Promise((resolve, reject) => { - pipelineBatch.run().then(_ => { - // waiting for the "end" event to be emitted - try { - const fileContent = fs.readFileSync(filePath, { encoding: 'utf8' }); - const fileLines = fileContent.split('\n').sort(); - // @mightymax it seems the resulting file can vary between 460-458 lines -> bug in File class - console.log('🪵 | file: Generator.class.test.ts:106 | pipelineBatch.addListener | fileLines:', fileLines.length) - batchPipelineEnd = true - if (batchPipelineEnd) { - resolve(true) - } - } catch (error) { - console.error(error) - } - }).catch(e => { reject(e) }) - }); - } - await runPipelineWithPromise() - - - }) it('Should work in batchSize for pipeline\'s generator - test with store', async function () { // when using local test files timeout should be removed this.timeout(50000) const N3Store = new Store() - const filePath = 'src/lib/tests/data/example-pipelineBatch.nt'; + const filePath = 'pipelines/data/example-pipelineBatch.nt'; const batchConfiguration: LDWorkbenchConfiguration = { @@ -133,30 +80,28 @@ describe('Generator Class', () => { ] } const pipelineBatch = new Pipeline(batchConfiguration) - const stageConfig = batchConfiguration.stages[0] - const stage = new Stage(pipelineBatch, stageConfig) - const generatorBatch = new Generator(stage); - const testNamedNode = new NamedNode('https://triplydb.com/triply/iris/id/floweringPlant/00106'); - - + pipelineBatch.validate() + const stage = pipelineBatch.stages.get('Stage 1') async function runGeneratorWithPromise(): Promise { return new Promise((resolve, reject) => { - generatorBatch.addListener('data', (quad) => { - N3Store.addQuad(quad) - }); - generatorBatch.addListener('end', (_numResults) => { - resolve(true); + pipelineBatch.run().then(() => { + + }).catch(error => {reject(error)}); + stage?.generator.addListener('data', (quad) => { + N3Store.addQuad(quad) + }); + stage?.generator.addListener('end', (_numResults) => { + resolve(true) }); - generatorBatch.addListener('error', (error) => { + stage?.generator.addListener('error', (error) => { reject(error); }); - // BUG current implementation for batch processing does not account for the case where the remaining/given inputsize of this.$thisList length is smaller than the batchsize - generatorBatch.run(testNamedNode); }); } await runGeneratorWithPromise() - console.log(N3Store.getQuads(null, null, null, null)) + expect(N3Store.size).to.equal(459) + expect(N3Store.getQuads(null,null,null,null)[458].subject.id).to.equal('https://triplydb.com/triply/iris/id/floweringPlant/00150') })