feat: Import dumps to SPARQL endpoint

README.md

@@ -20,6 +20,7 @@ A pipeline consists of one or more **stages**. Each stage has:
 - one or more **generators**, which generate triples about each URI using SPARQL CONSTRUCT queries.
 
 Stages can be chained together, with the output of one stage becoming the input of the next.
+The output of each stage combined becomes the final output of the pipeline.
 
 ### Design principles
 
@@ -133,18 +134,31 @@ To query large local files, you may need to load them into a SPARQL store first.
 for example Oxigraph:
 
 ```shell
-docker run --rm -v $PWD/data:/data -p 7878:7878 oxigraph/oxigraph --location /data serve
+docker run --rm -v $PWD/data:/data -p 7878:7878 oxigraph/oxigraph --location /data serve --bind 0.0.0.0:7878
 ```
 
-Then configure the store in your pipeline:
+Then configure the store in your pipeline, configuring at least one store under `stores`
+and using the `importTo` parameter to import the `endpoint`’s data to the store,
+referencing the store’s `queryUrl`:
 
 ```yaml
 # config.yml
 stores:
-  - queryUrl: "http://localhost:7878/query"
-    storeUrl: "http://localhost:7878/store"
+  - queryUrl: "http://localhost:7878/query" # SPARQL endpoint for read queries.
+    storeUrl: "http://localhost:7878/store" # SPARQL Graph Store HTTP Protocol endpoint. 
+
+stages:
+  - name: ...
+    iterator:
+      query: ...
+      endpoint: data.nt
+      importTo: http://localhost:7878/query
+    generator:
+      - query: ...
 ```
 
+The data is loaded into a named graph `<import:filename>`, so in this case `<import:data.nt>`.
+
 #### Example configuration
 
 ```yaml

jest.config.js

@@ -11,8 +11,8 @@ export default {
   coverageReporters: ['json-summary', 'text'],
   coverageThreshold: {
     global: {
-      lines: 71.15,
-      statements: 71.25,
+      lines: 70.9,
+      statements: 71.01,
       branches: 66.66,
       functions: 76.59,
     },

package.json

@@ -83,8 +83,8 @@
     "typescript": "~5.4.3"
   },
   "dependencies": {
-    "@comunica/query-sparql": "^3.1.2",
-    "@comunica/query-sparql-file": "^3.1.2",
+    "@comunica/query-sparql": "^3.2.0",
+    "@comunica/query-sparql-file": "^3.2.0",
     "@triply/triplydb": "^8.0.6",
     "ajv": "^8.16.0",
     "chalk": "^5.3.0",
@@ -98,6 +98,7 @@
     "millify": "^6.1.0",
     "n3": "^1.17.4",
     "ora": "^7.0.1",
+    "p-retry": "^6.2.0",
     "parse-duration": "^1.1.0",
     "pretty-ms": "^8.0.0",
     "rdf-dereference": "^2.2.0",

src/pipeline.ts

@@ -207,6 +207,7 @@ class Pipeline {
         this.increaseProgress(progress, stage, iterationsProcessed, count);
       });
       stage.on('error', e => {
+        progress.fail('Error: ' + (e as Error).message);
         reject(e);
       });
       stage.on('end', (iris, statements) => {

src/stage.ts

@@ -106,6 +106,7 @@ export default class Stage extends EventEmitter<Events> {
         await this.importer.run();
       } catch (e) {
         this.emit('error', e as Error);
+        return;
       }
     }
 

test/import.test.ts

@@ -39,6 +39,6 @@ describe('Import', () => {
   }, 15_000);
 
   afterAll(async () => {
-    // await sparqlStore.stop();
+    await sparqlStore.stop();
   });
 });