Skip to content

Commit

Permalink
Reverted #17 and #16 to seperate branches and removed implementation …
Browse files Browse the repository at this point in the history 
…from current branch
  • Loading branch information
Philippe Renzen committed Dec 22, 2023
1 parent 0aa74fc commit 80aa950
Show file tree
Hide file tree
Showing 16 changed files with 89 additions and 522 deletions.
58 changes: 26 additions & 32 deletions src/lib/Pipeline.class.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,7 @@ import path from "node:path";
import * as fs from "node:fs";
import { isFilePathString, isTriplyDBPathString } from '../utils/guards.js';
import TriplyDB from './TriplyDB.class.js';
interface RunnerOptions {
startFromStageName?: string
silent?: boolean
}

class Pipeline {
public readonly stages = new Map<string, Stage>();
public dataDir: string;
Expand Down Expand Up @@ -92,43 +89,42 @@ class Pipeline {
return this.$configuration;
}

public async run(opts?: RunnerOptions): Promise<void> {
public async run(startFromStageName?: string): Promise<void> {
this.now = new Date();
if (!(opts?.silent === true)) console.info(chalk.cyan(`🏁 starting pipeline "${chalk.bold(this.name)}"`));
const spinner = ora("validating pipeline")
if (!(opts?.silent === true)) spinner.start();
console.info(chalk.cyan(`🏁 starting pipeline "${chalk.bold(this.name)}"`));
const spinner = ora("validating pipeline").start();
let startFromStage = 0;
try {
this.validate();
if (opts?.startFromStageName !== undefined) {
if (/^\d+$/.test(opts.startFromStageName)) {
const ix = parseInt(opts.startFromStageName);
if (startFromStageName !== undefined) {
if (/^\d+$/.test(startFromStageName)) {
const ix = parseInt(startFromStageName);
if (Array.from(this.stages.keys()).length < ix) {
const e = new Error(
`Pipeline ${chalk.italic(
this.name
)} does not have stage #${chalk.italic(opts.startFromStageName)}.`
)} does not have stage #${chalk.italic(startFromStageName)}.`
);
if (!(opts?.silent === true)) spinner.fail(e.message);
spinner.fail(e.message);
this.error(e);
} else {
startFromStage = ix - 1;
}
} else if (!this.stages.has(opts.startFromStageName)) {
} else if (!this.stages.has(startFromStageName)) {
const e = new Error(
`Pipeline ${chalk.italic(
this.name
)} does not have stage ${chalk.italic(opts.startFromStageName)}.`
)} does not have stage ${chalk.italic(startFromStageName)}.`
);
if (!(opts?.silent === true)) spinner.fail(e.message);
spinner.fail(e.message);
this.error(e);
} else {
startFromStage = Array.from(this.stages.keys()).findIndex(
(value) => value === opts.startFromStageName
(value) => value === startFromStageName
);
}
}
if (!(opts?.silent === true)) spinner.succeed();
spinner.succeed();
} catch (e) {
spinner.fail((e as Error).message);
this.error(e as Error);
Expand All @@ -139,22 +135,21 @@ class Pipeline {
Array.from(this.stages.keys()).slice(0, startFromStage).forEach(stagename => {
ora().start().info(`stage "${chalk.bold(stagename)}" was skipped`).stop();
})
this.runRecursive(opts);
this.runRecursive();
}

private runRecursive(opts?: RunnerOptions): void {
private runRecursive(): void {
const stage = this.stages.get(this.stageNames.shift()!)!;
const spinner = ora("Loading results from Iterator")
if (!(opts?.silent === true)) spinner.start();
const spinner = ora("Loading results from Iterator").start();
stage.on("iteratorResult", ($this) => {
if (!(opts?.silent === true)) spinner.text = $this.value;
spinner.text = $this.value;
});
stage.on('error', (e) => {
spinner.fail()
this.error(e)
})
stage.on("end", (iris, statements) => {
if (!(opts?.silent === true)) spinner.succeed(
spinner.succeed(
`stage "${chalk.bold(
stage.name
)}" resulted in ${statements.toLocaleString()} statement${
Expand All @@ -164,9 +159,9 @@ class Pipeline {
if (this.stageNames.length !== 0) {
this.runRecursive();
} else {
this.writeResult(opts)
this.writeResult()
.then(_ => {
if (!(opts?.silent === true)) console.info(
console.info(
chalk.green(
`✔ your pipeline "${chalk.bold(
this.name
Expand All @@ -186,12 +181,11 @@ class Pipeline {
}
}

private async writeResult(opts?: RunnerOptions): Promise<void> {
const spinner = ora('Writing results to destination')
if (!(opts?.silent === true)) spinner.start();
private async writeResult(): Promise<void> {
const spinner = ora('Writing results to destination').start();
await this.destination.write(this, spinner)
if (!(opts?.silent === true)) spinner.suffixText = this.destination.path
if (!(opts?.silent === true)) spinner.succeed()
spinner.suffixText = this.destination.path
spinner.succeed()
}

get name(): string {
Expand All @@ -203,4 +197,4 @@ class Pipeline {
}
}

export default Pipeline;
export default Pipeline;
12 changes: 6 additions & 6 deletions src/lib/tests/Generator.class.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ describe('Generator Class', () => {
name: 'Stage 1',
iterator: {
query: 'file://static/example/iterator-stage-1.rq',
endpoint: 'file://static/tests/iris.nt'
endpoint: 'https://api.triplydb.com/datasets/Triply/iris/services/demo-service/sparql'
},
generator:
[{
Expand All @@ -36,7 +36,7 @@ describe('Generator Class', () => {
generator:
[{
query: 'file://static/example/generator-stage-2.rq',
endpoint: 'file://static/tests/wikidata.nt'
endpoint: 'https://query.wikidata.org/sparql'
}]
}
]
Expand Down Expand Up @@ -67,7 +67,7 @@ describe('Generator Class', () => {
name: 'Stage 1',
iterator: {
query: 'file://static/example/iterator-stage-1.rq',
endpoint: 'file://static/tests/iris.nt'
endpoint: 'https://api.triplydb.com/datasets/Triply/iris/services/demo-service/sparql'
},
generator:
[{
Expand All @@ -83,7 +83,7 @@ describe('Generator Class', () => {
generator:
[{
query: 'file://static/example/generator-stage-2.rq',
endpoint: 'file://static/tests/wikidata.nt'
endpoint: 'https://query.wikidata.org/sparql'
}]
}
]
Expand All @@ -109,7 +109,7 @@ describe('Generator Class', () => {
name: 'Stage 1',
iterator: {
query: 'file://static/example/iterator-stage-1.rq',
endpoint: 'file://static/tests/iris.nt'
endpoint: 'https://api.triplydb.com/datasets/Triply/iris/services/demo-service/sparql'
},
generator:
[{
Expand All @@ -123,7 +123,7 @@ describe('Generator Class', () => {
generator:
[{
query: 'file://static/example/generator-stage-2.rq',
endpoint: 'file://static/tests/wikidata.nt'
endpoint: 'https://query.wikidata.org/sparql'
}]
}
]
Expand Down
8 changes: 4 additions & 4 deletions src/lib/tests/Iterator.class.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ describe('Iterator Class', () => {
name: 'Stage 1',
iterator: {
query: 'file://static/example/iterator-stage-1.rq',
endpoint: 'file://static/tests/iris.nt'
endpoint: 'https://api.triplydb.com/datasets/Triply/iris/services/demo-service/sparql'
},
generator:
[{
Expand All @@ -34,7 +34,7 @@ describe('Iterator Class', () => {
generator:
[{
query: 'file://static/example/generator-stage-2.rq',
endpoint: 'file://static/tests/wikidata.nt'
endpoint: 'https://query.wikidata.org/sparql'
}]
}
]
Expand Down Expand Up @@ -65,7 +65,7 @@ describe('Iterator Class', () => {
name: 'Stage 1',
iterator: {
query: 'file://static/example/iterator-stage-1.rq',
endpoint: 'file://static/tests/iris.nt'
endpoint: 'https://api.triplydb.com/datasets/Triply/iris/services/demo-service/sparql'
},
generator:
[{
Expand All @@ -79,7 +79,7 @@ describe('Iterator Class', () => {
generator:
[{
query: 'file://static/example/generator-stage-2.rq',
endpoint: 'file://static/tests/wikidata.nt'
endpoint: 'https://query.wikidata.org/sparql'
}]
}
]
Expand Down
24 changes: 12 additions & 12 deletions src/lib/tests/Pipeline.class.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ describe('Pipeline Class', () => {
name: 'Stage 1',
iterator: {
query: 'file://static/example/iterator-stage-1.rq',
endpoint: 'file://static/tests/iris.nt'
endpoint: 'https://api.triplydb.com/datasets/Triply/iris/services/demo-service/sparql'
},
generator:
[{
Expand All @@ -33,7 +33,7 @@ describe('Pipeline Class', () => {
generator:
[{
query: 'file://static/example/generator-stage-2.rq',
endpoint: 'file://static/tests/wikidata.nt'
endpoint: 'https://query.wikidata.org/sparql'
}]
}
]
Expand All @@ -60,7 +60,7 @@ describe('Pipeline Class', () => {
name: 'Stage 1',
iterator: {
query: 'file://static/example/iterator-stage-1.rq',
endpoint: 'file://static/tests/iris.nt'
endpoint: 'https://api.triplydb.com/datasets/Triply/iris/services/demo-service/sparql'
},
generator:
[{
Expand All @@ -74,7 +74,7 @@ describe('Pipeline Class', () => {
generator:
[{
query: 'file://static/example/generator-stage-2.rq',
endpoint: 'file://static/tests/wikidata.nt'
endpoint: 'https://query.wikidata.org/sparql'
}]
}
]
Expand All @@ -99,7 +99,7 @@ describe('Pipeline Class', () => {
{
iterator: {
query: 'file://static/example/iterator-stage-1.rq',
endpoint: 'file://static/tests/iris.nt'
endpoint: 'https://api.triplydb.com/datasets/Triply/iris/services/demo-service/sparql'
},
generator: {
query: 'file://static/example/generator-stage-1-1.rq'
Expand All @@ -108,11 +108,11 @@ describe('Pipeline Class', () => {
{
iterator: {
query: 'file://static/example/iterator-stage-2.rq',
endpoint: 'file://static/tests/iris.nt'
endpoint: 'https://api.triplydb.com/datasets/Triply/iris/services/demo-service/sparql'
},
generator: {
query: 'file://static/example/generator-stage-2.rq',
endpoint: 'file://static/tests/wikidata.nt'
endpoint: 'https://query.wikidata.org/sparql'
}
}
]
Expand Down Expand Up @@ -211,7 +211,7 @@ describe('Pipeline Class', () => {
},
generator: [{
query: 'file://static/example/generator-stage-2.rq',
endpoint: 'file://static/tests/wikidata.nt'
endpoint: 'https://query.wikidata.org/sparql'
}]
}
]
Expand Down Expand Up @@ -257,7 +257,7 @@ describe('Pipeline Class', () => {
},
generator: [{
query: 'file://static/example/generator-stage-2.rq',
endpoint: 'file://static/tests/wikidata.nt'
endpoint: 'https://query.wikidata.org/sparql'
}]
}
]
Expand Down Expand Up @@ -287,7 +287,7 @@ describe('Pipeline Class', () => {
name: 'Stage 1',
iterator: {
query: 'file://static/example/iterator-stage-1.rq',
endpoint: 'file://static/tests/iris.nt'
endpoint: 'https://api.triplydb.com/datasets/Triply/iris/services/demo-service/sparql'
},
generator:
[{
Expand All @@ -301,14 +301,14 @@ describe('Pipeline Class', () => {
generator:
[{
query: 'file://static/example/generator-stage-2.rq',
endpoint: 'file://static/tests/wikidata.nt'
endpoint: 'https://query.wikidata.org/sparql'
}]
}
]
}
const pipeline = new Pipeline(configuration)

await expect(Promise.resolve(pipeline.run({silent: true}))).to.eventually.fulfilled
await expect(Promise.resolve(pipeline.run())).to.eventually.fulfilled

});
});
Expand Down
Loading

0 comments on commit 80aa950

Please sign in to comment.