Merge branch 'main' of https://github.com/biothings/bte-server into dev

config/api_list.yaml

@@ -145,4 +145,8 @@ include:
   # not accessible by team or api-specific endpoints
   - id: 412af63e15b73e5a30778aac84ce313f
     name: Connections Hypothesis Provider API
+  # # TRAPI (Translator standard) APIs: Multiomics (made with RTX team Plover)
+  # # not accessible by team or api-specific endpoints
+  # - id: e51073371d7049b9643e1edbdd61bcbd
+  #   name: Clinical Trials KP - TRAPI 1.5.0
 exclude: [] # if adding exclude remove these square brackets

config/smartapi_overrides.yaml

@@ -1,4 +1,7 @@
 ---
 config:
   only_overrides: false
-apis: {}
+apis: {
+  # "8f08d1446e0bb9c2b323713ce83e2bd3": "https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/refs/heads/remove-clinical-trials/mychem.info/openapi_full.yml",
+  # "1138c3297e8e403b6ac10cff5609b319": "https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/refs/heads/remove-clinical-trials/repodb/smartapi.yaml"
+}

src/config/index.ts

@@ -74,23 +74,23 @@ export default class Config {
   setLimiter() {
     const slowLimiter = rateLimit({
       windowMs: 1 * 60 * 1000, //1min
-      max: parseInt(process.env.MAX_QUERIES_PER_MIN || "20"),
+      max: parseInt(process.env.MAX_QUERIES_PER_MIN || "50"),
     });
     const medLimiter = rateLimit({
       windowMs: 1 * 60 * 1000, //1min
-      max: parseInt(process.env.MAX_QUERIES_PER_MIN || "30"),
+      max: parseInt(process.env.MAX_QUERIES_PER_MIN || "100"),
     });
     const fastLimiter = rateLimit({
       windowMs: 1 * 60 * 1000, //1min
-      max: parseInt(process.env.MAX_QUERIES_PER_MIN || "6000"),
+      max: parseInt(process.env.MAX_QUERIES_PER_MIN || "10000"),
     });
     this.app.use("/", fastLimiter);
-    this.app.use("/v1/query", slowLimiter);
+    this.app.use("/v1/query", medLimiter);
     this.app.use("/v1/team/:team_name/query", medLimiter);
-    this.app.use("/v1/team/:smartapiID/query", medLimiter);
-    this.app.use("/v1/meta_knowledge_graph", medLimiter);
-    this.app.use("/v1/team/:teamName/meta_knowledge_graph", medLimiter);
-    this.app.use("/v1/smartapi/:smartapiID/meta_knowledge_graph", medLimiter);
+    this.app.use("/v1/team/:smartapiID/query", fastLimiter);
+    this.app.use("/v1/meta_knowledge_graph", fastLimiter);
+    this.app.use("/v1/team/:teamName/meta_knowledge_graph", fastLimiter);
+    this.app.use("/v1/smartapi/:smartapiID/meta_knowledge_graph", fastLimiter);
     this.app.use("/v1/asyncquery", fastLimiter);
     this.app.use("/v1/team/:teamName/asyncquery", fastLimiter);
     this.app.use("/v1/smartapi/:smartapiID/asyncquery", fastLimiter);

src/controllers/threading/threadHandler.ts

@@ -22,10 +22,10 @@ import { Queue } from "bull";
 const SYNC_MIN_CONCURRENCY = 2;
 const ASYNC_MIN_CONCURRENCY = 3;
 
-// On most instances, there are two nodes, one for Service Provider endpoints and one for everything else
-// On Dev and local instances, this isn't the case, so a lower concurrency is needed
-const CORE_CONCURRENCY_RATIO = parseInt(process.env.CORE_CONCURRENCY_RATIO) || 0.25;
-const MEM_CONCURRENCY_RATIO = parseFloat(process.env.MEM_CONCURRENCY_RATIO) || 0.6;
+// Ratio of Queryies per core
+const CORE_CONCURRENCY_RATIO = parseInt(process.env.CORE_CONCURRENCY_RATIO) || 1;
+// Ratio of Queries per GB of memory
+const MEM_CONCURRENCY_RATIO = parseFloat(process.env.MEM_CONCURRENCY_RATIO) || 2;
 
 const CORE_LIMIT = Math.ceil(os.cpus().length * CORE_CONCURRENCY_RATIO);
 

src/routes/index.ts

@@ -50,10 +50,11 @@ export const tasks: TaskByRoute = {
   asyncquery_v1: (taskInfo: TaskInfo) => V1AsyncQuery.task(taskInfo),
   asyncquery_v1_by_api: (taskInfo: TaskInfo) => V1AsyncQueryByAPI.task(taskInfo),
   asyncquery_v1_by_team: (taskInfo: TaskInfo) => V1AsyncQueryByTeam.task(taskInfo),
+  // load MetaKG from global
+  meta_knowledge_graph_v1: (taskInfo: TaskInfo) => V1MetaKG.task(taskInfo),
   // Not threaded due to being lightweight/speed being higher priority
   // performance: (taskInfo: TaskInfo) => Performance.task(taskInfo),
   // metakg: (taskInfo: TaskInfo) => MetaKG.task(taskInfo),
-  // meta_knowledge_graph_v1: (taskInfo: TaskInfo) => V1MetaKG.task(taskInfo),
   // meta_knowledge_graph_v1_by_api: (taskInfo: TaskInfo) => V1MetaKGByAPI.task(taskInfo),
   // meta_knowledge_graph_v1_by_team: (taskInfo: TaskInfo) => V1MetaKGByTeam.task(taskInfo),
 };

src/routes/v1/meta_knowledge_graph_v1.ts

@@ -1,5 +1,8 @@
+import path from "path";
+import { TaskInfo } from "@biothings-explorer/types";
 import handler from "../../controllers/meta_knowledge_graph";
 import * as utils from "../../utils/common";
+import { runTask, taskResponse, taskError } from "../../controllers/threading/threadHandler";
 import { Express, NextFunction, Request, Response, RequestHandler } from "express";
 
 class MetaKG {
@@ -8,16 +11,33 @@ class MetaKG {
       .route("/v1/meta_knowledge_graph")
       .get((async (_req: Request, res: Response, next: NextFunction) => {
         try {
-          const metaKGHandler = new handler(undefined);
-          const kg = await metaKGHandler.getKG();
+          const response = await runTask(_req, res, path.parse(__filename).name);
           res.setHeader("Content-Type", "application/json");
-          res.end(JSON.stringify(kg));
+          res.end(JSON.stringify(response));
         } catch (error) {
           next(error);
         }
       }) as RequestHandler)
       .all(utils.methodNotAllowed);
   }
+
+  async task(taskInfo: TaskInfo) {
+    try {
+      let kg = undefined;
+
+      // read metakg from files if not globally defined
+      if(!taskInfo.data.options.metakg) {
+        const metaKGHandler = new handler(undefined);
+        kg = await metaKGHandler.getKG(); 
+      } else {
+        kg = taskInfo.data.options.metakg;
+      }
+      // response.logs = utils.filterForLogLevel(response.logs, options.logLevel);
+      return taskResponse(kg);
+    } catch (error) {
+      taskError(error as Error);
+    }
+  }
 }
 
 export default new MetaKG();