From 03cce11d861e6d3b13a020cbcb17248074fefea9 Mon Sep 17 00:00:00 2001
From: Stefan Peters <Stefan.Peters@gbv.de>
Date: Fri, 29 Nov 2024 11:54:00 +0100
Subject: [PATCH] Add support for multiple JSKOS API backends (#55)

---
 README.md                     |  2 +-
 src/client/store.js           | 45 ++++++++++++++++++++++++++++-------
 src/client/views/HomeView.vue |  8 ++++---
 src/client/views/ItemView.vue |  3 ++-
 src/server/backend.js         | 38 ++++++++++++++++++++++-------
 src/server/server.js          |  7 ++++++
 6 files changed, 81 insertions(+), 22 deletions(-)

diff --git a/README.md b/README.md
index e409e41..96d0017 100644
--- a/README.md
+++ b/README.md
@@ -39,7 +39,7 @@ Instances of jskos-proxy are configured with environment variables, in local fil
 - `NAMESPACE` - URI namespace of all objects served via this proxy. Must end with a slash (default: `http://example.org/`)
 - `BASE` - Path under which the application will be hosted on. Must end with a slash. (Default: `/`)
   - If `NAMESPACE` is `http://example.org/some-path/`, but `http://example.org/` itself is not served by jskos-proxy, you need to set `BASE` to `/some-path/`.
-- `BACKEND` - JSKOS API base URL
+- `BACKEND` - JSKOS API base URLs (seperated by `,`)
 - `TITLE` - Title of the service (default `JSKOS Proxy`)
 <!-- - `LOGO` - optional logo image file, must be placed in `public` directory -->
 - `QUICK_SELECTION` - comma separated list of vocabulary URIs to prominently show at the start page
diff --git a/src/client/store.js b/src/client/store.js
index c7482f0..3e11364 100644
--- a/src/client/store.js
+++ b/src/client/store.js
@@ -44,6 +44,13 @@ export function setLocale(value) {
   }
 }
 
+import { cdk } from "cocoda-sdk"
+
+export const registries = config.backend.split(",").map(base => cdk.initializeRegistry({
+  provider: "ConceptApi",
+  status: `${base}status`,
+}))
+
 export const schemeFetchPromise = fetch(
   config.namespace.pathname, 
   { 
@@ -51,7 +58,18 @@ export const schemeFetchPromise = fetch(
     signal: AbortSignal.timeout(20000),
   },
 ).then(res => res.json()).then(data => {
-  state.schemes = data
+  state.schemes = data.map(scheme => {
+    scheme = new jskos.ConceptScheme(scheme)
+    // Add _registry to scheme (via special REGISTRY field provided by backend)
+    scheme._registry = registries.find(registry => registry._api?.status === scheme.REGISTRY?.status)
+    // If there's an identifier with the current namespace, use it as the main identifier
+    const identifier = (scheme.identifier || []).find(i => i.startsWith(config.namespace))
+    if (identifier) {
+      scheme.identifier.push(scheme.uri)
+      scheme.uri = identifier
+    }
+    return scheme
+  })
 }).catch(() => {
   console.error("Error loading schemes from backend.")
   // TODO: Add retry mechanism
@@ -61,13 +79,16 @@ export const schemeFetchPromise = fetch(
 export const schemes = computed(() => state.schemes)
 export const schemesAsConceptSchemes = computed(() => state.schemes?.map(scheme => new jskos.ConceptScheme(scheme)) || [])
 
-import { cdk } from "cocoda-sdk"
+export async function getRegistryForScheme(scheme) {
+  await schemeFetchPromise
+  return state.schemes.find(s => jskos.compare(s, scheme))?._registry
+}
 
-export const registry = cdk.initializeRegistry({
-  provider: "ConceptApi",
-  // ? Does "config.backend" always have a trailing slash?
-  status: `${config.backend}status`,
-})
+export async function getRegistryForUri(uri) {
+  await schemeFetchPromise
+  // Find scheme where URI matches namespace
+  return await getRegistryForScheme(state.schemes.find(scheme => scheme.notationFromUri(uri)))
+}
 
 export function getConcept(concept) {
   for (const uri of jskos.getAllUris(concept)) {
@@ -142,6 +163,10 @@ export function saveConceptsWithOptions(options) {
 
 let properties
 export async function loadConcept(uri) {
+  const registry = await getRegistryForUri(uri)
+  if (!registry) {
+    return null
+  }
   if (!properties) {
     // Adjust properties for concept details
     properties = registry._defaultParams?.properties || ""
@@ -169,7 +194,11 @@ export async function loadTop(scheme) {
     return scheme?.topConcepts
   }
   console.time(`loadTop ${scheme.uri}`)
-  const topConcepts = await registry.getTop({ scheme: { uri: scheme.uri }, params: { properties: "" } })
+  const registry = await getRegistryForScheme(scheme)
+  if (!registry) {
+    return null
+  }
+  const topConcepts = await registry.getTop({ scheme: { uri: scheme.uri, identifier: scheme.identifier }, params: { properties: "" } })
   topConcepts.forEach(concept => {
     concept.ancestors = []
   })
diff --git a/src/client/views/HomeView.vue b/src/client/views/HomeView.vue
index 7274a32..1998bbc 100644
--- a/src/client/views/HomeView.vue
+++ b/src/client/views/HomeView.vue
@@ -1,7 +1,7 @@
 <script setup>
 import { useRoute, useRouter } from "vue-router"
 import * as jskos from "jskos-tools"
-import { schemes, schemesAsConceptSchemes, quickSelection, publisherSelection, typeSelection, registry, loadConcept } from "@/store.js"
+import { schemes, schemesAsConceptSchemes, quickSelection, publisherSelection, typeSelection, loadConcept, registries } from "@/store.js"
 import { ref, computed, watch } from "vue"
 import { getRouterUrl } from "@/utils.js"
 import CategoryButton from "@/components/CategoryButton.vue"
@@ -95,12 +95,14 @@ watch(() => route.query?.conceptSearch, async (value) => {
   }
   console.time(`concept search ${value}`)
   conceptSearchResults.value = [null]
-  const results = await registry.search({ search: value.trim() })
+  // Load concept results from all configured registries
+  const results = (await Promise.all(registries.map(registry => registry.search({ search: value.trim() }).catch(() => ([]))))).reduce((all, cur) => all.concat(cur), [])
   console.timeEnd(`concept search ${value}`)
   if (value === route.query?.conceptSearch) {
     const groupedResults = []
     for (const result of results) {
-      const scheme = result.inScheme[0]
+      let scheme = result.inScheme[0]
+      scheme = schemes.value?.find(s => jskos.compare(s, scheme)) || scheme
       const existingGroup = groupedResults.find(g => jskos.compare(g.scheme, scheme))
       if (existingGroup) {
         existingGroup.results.push(result)
diff --git a/src/client/views/ItemView.vue b/src/client/views/ItemView.vue
index 09db0f7..522506f 100644
--- a/src/client/views/ItemView.vue
+++ b/src/client/views/ItemView.vue
@@ -2,7 +2,7 @@
 import config from "@/config.js"
 import * as jskos from "jskos-tools"
 import { AutoLink, LicenseInfo } from "jskos-vue"
-import { schemes, registry, loadTop, loadNarrower, loadConcept, loadAncestors, saveConcept, formats, getConceptByUri, detailsLoadedKey, detailsLoadedStates } from "@/store.js"
+import { schemes, loadTop, loadNarrower, loadConcept, loadAncestors, saveConcept, formats, getConceptByUri, detailsLoadedKey, detailsLoadedStates } from "@/store.js"
 import { computed, ref, reactive, watch } from "vue"
 import { useRoute, useRouter } from "vue-router"
 import { utils } from "jskos-vue"
@@ -25,6 +25,7 @@ const schemeUri = computed(() => route.params.voc.match(/^https?:\/\//) ? route.
 const scheme = computed(() => {
   return schemes.value?.find(s => jskos.compare(s, { uri: schemeUri.value }))
 })
+const registry = computed(() => scheme.value?._registry)
 
 const uri = computed(() => scheme.value && (route.params.id && `${config.namespace}${route.params.voc}/${route.params.id}` || route.query.uri))
 const concept = computed({
diff --git a/src/server/backend.js b/src/server/backend.js
index c289c44..35e85ce 100644
--- a/src/server/backend.js
+++ b/src/server/backend.js
@@ -50,15 +50,28 @@ export class ApiBackend {
     let previouslyErrored = false
     this.getSchemesPromise = new Promise(resolve => {
       cdk.repeat({
-        function: () => {
-          if (!this.registry?._api?.api) {
-            this.registry = cdk.initializeRegistry({
+        function: async () => {
+          if (!this.registries?.length) {
+            this.registries = this.base.split(",").map(base => cdk.initializeRegistry({
               provider: "ConceptApi",
               // ? Does "base" always have a trailing slash?
-              status: `${this.base}status`,
+              status: `${base}status`,
+            }))
+            // Explicitly set cdk instance for each registry to null so that `registryForScheme` won't be used
+            this.registries.forEach(registry => {
+              registry.cdk = null
             })
           }
-          return this.registry.getSchemes({ params: { limit: 10000 } })
+          let schemes = []
+          const results = (await Promise.all(this.registries.map(registry => registry.getSchemes({ params: { limit: 10000 } })))).reduce((all, cur) => all.concat(cur), [])
+          for (const result of results) {
+            // Only add to schemes if not there yet = backends specified first have priority
+            // Also don't add schemes that explicitly do not provide concepts
+            if (!schemes.find(scheme => jskos.compare(scheme, result)) && result.concepts?.length !== 0) {
+              schemes.push(result)
+            }
+          }
+          return schemes.map(scheme => new jskos.ConceptScheme(scheme))
         },
         callback: (error, result) => {
           if (error) {
@@ -70,8 +83,7 @@ export class ApiBackend {
             log(`Loaded ${result.length} schemes for backend.`)
             previouslyErrored = false
           }
-          // Only return schemes that have concepts
-          this.schemes = result.filter(s => !s.concepts || s.concepts?.length > 0)
+          this.schemes = result
           resolve()
         },
         interval: 6 * 1000,
@@ -96,9 +108,17 @@ export class ApiBackend {
   }
 
   async getConcept(uri) {
-    let properties = this.registry._defaultParams?.properties || ""
+    if (!this.schemes) {
+      await this.getSchemesPromise
+    }
+    // Determine registry
+    const registry = this.schemes.find(scheme => scheme.notationFromUri(uri))?._registry
+    if (!registry) {
+      return null
+    }
+    let properties = registry._defaultParams?.properties || ""
     properties += (properties ? "," : "") + "broader,ancestors,narrower"
-    return (await this.registry.getConcepts({ concepts: [{ uri }], params: { properties } }))?.[0]
+    return (await registry.getConcepts({ concepts: [{ uri }], params: { properties } }))?.[0]
   }
 
   toString() {
diff --git a/src/server/server.js b/src/server/server.js
index a6b3c91..d95fd3b 100644
--- a/src/server/server.js
+++ b/src/server/server.js
@@ -77,6 +77,13 @@ app.get(`${config.namespace.pathname}:voc?/:id?`, async (req, res, next) => {
 
     // Clean data
     for (const _item of Array.isArray(item) ? item : [item]) {
+      // Add REGISTRY field so that frontend knows which registry this item belongs to
+      if (_item._registry) {
+        _item.REGISTRY = {
+          provider: _item._registry.constructor.providerName,
+          status: _item._registry._api.status || `${_item._registry._api.api}status`,
+        }
+      }
       // Remove keys started with _
       Object.keys(_item).filter(key => key.startsWith("_")).forEach(key => delete _item[key])
       // Replace certain subsets with URI only objects