Fixes test and removed comments

netwerk-digitaal-erfgoed · Dec 27, 2023 · b8b77ed · b8b77ed
1 parent 25466cd
commit b8b77ed
Show file tree

Hide file tree

Showing 2 changed files with 14 additions and 70 deletions.
diff --git a/src/lib/Generator.class.ts b/src/lib/Generator.class.ts
@@ -78,7 +78,6 @@ class Generator extends EventEmitter {
     if ($this !== undefined) this.$thisList.push($this)
     const union: UnionPattern = { type: 'union', patterns: [] }
     const error = (e: any): Error => new Error(`The Generator did not run succesfully, it could not get the results from the endpoint ${this.source}: ${(e as Error).message}`)
-    // @mightymax problem - what  if the this.$thisList length is smaller than the batchsize? => and cleanup$thisList() (like the end() function previously) is needed
     if (this.$thisList.length >= (batchSize ?? this.batchSize)) {
       if (this.source === '') this.source = getEngineSource(this.endpoint)
       const unionQuery = getSPARQLQuery(getSPARQLQueryString(this.query), "construct");

diff --git a/src/lib/tests/Generator.class.test.ts b/src/lib/tests/Generator.class.test.ts
@@ -7,7 +7,6 @@ import * as chai from 'chai'
 import chaiAsPromised from 'chai-as-promised'
 import { NamedNode, Store } from "n3";
 import type { LDWorkbenchConfiguration } from "../LDWorkbenchConfiguration.js";
-import * as fs from "fs"
 chai.use(chaiAsPromised)
 const expect = chai.expect
 
@@ -29,63 +28,11 @@ describe('Generator Class', () => {
     });
     // BUG when both the generator and iterator tests are running, it seems the iterator will never terminate
     describe('run', () => {
-        // BUG File class seems to create files of inconsistent filesizes due to buffer size difference
-        it.only('Should work in batchSize for pipeline\'s generator - test with output file', async function () {
-            // when using local test files timeout should be removed
-            this.timeout(4000)
-            const filePath = 'src/lib/tests/data/example-pipelineBatch.nt';
-
-
-            const batchConfiguration: LDWorkbenchConfiguration = {
-                name: 'Example Pipeline Batch',
-                description: 'This is an example pipeline. It uses files that are available in this repository  and SPARQL endpoints that should work.\n',
-                destination: "file://" + filePath,
-                stages: [
-                    {
-                        name: 'Stage 1',
-                        iterator: {
-                            query: 'file://static/example/iterator-stage-1.rq',
-                            endpoint: 'https://api.triplydb.com/datasets/Triply/iris/services/demo-service/sparql'
-                        },
-                        generator: {
-                            query: 'file://static/example/generator-stage-1.rq',
-                            // adjust batchsize for test here
-                            batchSize: 4
-                        }
-                    }
-                ]
-            }
-
-            const pipelineBatch = new Pipeline(batchConfiguration)
-            async function runPipelineWithPromise(): Promise<boolean> {
-                let batchPipelineEnd = false
-                return new Promise((resolve, reject) => {
-                    pipelineBatch.run().then(_ => {
-                        // waiting for the "end" event to be emitted
-                        try {
-                            const fileContent = fs.readFileSync(filePath, { encoding: 'utf8' });
-                            const fileLines = fileContent.split('\n').sort();
-                            // @mightymax it seems the resulting file can vary between 460-458 lines -> bug in File class
-                            console.log('🪵  | file: Generator.class.test.ts:106 | pipelineBatch.addListener | fileLines:', fileLines.length)
-                            batchPipelineEnd = true
-                            if (batchPipelineEnd) {
-                                resolve(true)
-                            }
-                        } catch (error) {
-                            console.error(error)
-                        }
-                    }).catch(e => { reject(e) })
-                });
-            }
-            await runPipelineWithPromise()
-
-
-        })
         it('Should work in batchSize for pipeline\'s generator - test with store', async function () {
             // when using local test files timeout should be removed
             this.timeout(50000)
             const N3Store = new Store()
-            const filePath = 'src/lib/tests/data/example-pipelineBatch.nt';
+            const filePath = 'pipelines/data/example-pipelineBatch.nt';
 
 
             const batchConfiguration: LDWorkbenchConfiguration = {
@@ -108,30 +55,28 @@ describe('Generator Class', () => {
                 ]
             }
             const pipelineBatch = new Pipeline(batchConfiguration)
-            const stageConfig = batchConfiguration.stages[0]
-            const stage = new Stage(pipelineBatch, stageConfig)
-            const generatorBatch = new Generator(stage);
-            const testNamedNode = new NamedNode('https://triplydb.com/triply/iris/id/floweringPlant/00106');
-
-
+            pipelineBatch.validate()
+            const stage = pipelineBatch.stages.get('Stage 1')
 
             async function runGeneratorWithPromise(): Promise<boolean> {
                 return new Promise((resolve, reject) => {
-                    generatorBatch.addListener('data', (quad) => {
-                        N3Store.addQuad(quad)
-                    });
-                    generatorBatch.addListener('end', (_numResults) => {
-                        resolve(true);
+                  pipelineBatch.run().then(() => {
+
+                  }).catch(error => {reject(error)});
+                  stage?.generator.addListener('data', (quad) => {
+                    N3Store.addQuad(quad)
+                  });
+                  stage?.generator.addListener('end', (_numResults) => {
+                      resolve(true)
                     });
-                    generatorBatch.addListener('error', (error) => {
+                    stage?.generator.addListener('error', (error) => {
                         reject(error);
                     });
-                    // BUG current implementation for batch processing does not account for the case where the remaining/given inputsize of this.$thisList length is smaller than the batchsize
-                    generatorBatch.run(testNamedNode);
                 });
             }
             await runGeneratorWithPromise()
-            console.log(N3Store.getQuads(null, null, null, null))
+            expect(N3Store.size).to.equal(459)
+            expect(N3Store.getQuads(null,null,null,null)[458].subject.id).to.equal('https://triplydb.com/triply/iris/id/floweringPlant/00150')
 
 
         })