Merge branch 'main' of https://github.com/biothings/bte-server into r…

…etriever-backport

.github/workflows/test_ws_codecov.yml

@@ -13,11 +13,21 @@ jobs:
       id: branch-name
       uses: tj-actions/branch-names@v6
 
-    - uses: actions/checkout@v3
+    - name: Checkout to specific branch
+      uses: actions/checkout@v3
+      id: specific-checkout
+      continue-on-error: true
       with:
         repository: biothings/biothings_explorer
         ref: ${{ steps.branch-name.outputs.current_branch }}
 
+    - name: Checkout to main if above failed
+      if: steps.specific-checkout.outcome == 'failure'
+      uses: actions/checkout@v3
+      with:
+        repository: biothings/biothings_explorer
+        ref: main
+
     - name: Use Node.js 18.x
       uses: actions/setup-node@v3
       with:

__test__/integration/TRAPIv1.1.test.ts

@@ -99,7 +99,7 @@ describe("Testing v1.1 endpoints", () => {
           expect.arrayContaining([
             expect.objectContaining({
               subject: "biolink:SmallMolecule",
-              predicate: "biolink:treats",
+              predicate: "biolink:contributes_to",
               object: "biolink:Disease",
             }),
           ]),
@@ -133,7 +133,7 @@ describe("Testing v1.1 endpoints", () => {
           expect.arrayContaining([
             expect.objectContaining({
               subject: "biolink:SmallMolecule",
-              predicate: "biolink:treats",
+              predicate: "biolink:contributes_to",
               object: "biolink:Disease",
             }),
           ]),
@@ -155,7 +155,7 @@ describe("Testing v1.1 endpoints", () => {
       });
   });
 
-  test("POST /v1/query with gene2chemical query", async () => {
+  test.skip("POST /v1/query with gene2chemical query", async () => {
     const mychem_query_3_input = JSON.parse(
       fs.readFileSync(path.resolve(__dirname, "../data/api_results/mychem_query_3_input.json"), { encoding: "utf8" }),
     );
@@ -437,7 +437,7 @@ describe("Testing v1.1 endpoints", () => {
       });
   });
 
-  test("POST /v1/query with disease2gene query", async () => {
+  test.skip("POST /v1/query with disease2gene query", async () => {
     const big_sri_input_3 = JSON.parse(
       fs.readFileSync(path.resolve(__dirname, "../data/api_results/big_sri_input_3.json"), { encoding: "utf8" }),
     );
@@ -1038,7 +1038,7 @@ describe("Testing v1.1 endpoints", () => {
       });
   });
 
-  test("POST /v1/query with query that doesn't provide input category", async () => {
+  test.skip("POST /v1/query with query that doesn't provide input category", async () => {
     // @ts-expect-error mocked axios handles typing poorly
     mockedAxios.mockImplementation(async (qData: any) => {
       let res: unknown = undefined;

config/api_list.yaml

@@ -91,8 +91,6 @@ include:
   # also accessible by v1/team/Service Provider/ and v1/team/Multiomics Provider endpoints, and by api-specific endpoints
   - id: adf20dd6ff23dfe18e8e012bde686e31
     name: Multiomics BigGIM-DrugResponse KP API
-  - id: 08a5ddcde71b4bf838327ef469076acd
-    name: Multiomics ClinicalTrials KP
   - id: d86a24f6027ffe778f84ba10a7a1861a
     name: Multiomics EHR Risk KP API
   - id: 02af7d098ab304e80d6f4806c3527027
@@ -147,4 +145,8 @@ include:
   # not accessible by team or api-specific endpoints
   - id: 412af63e15b73e5a30778aac84ce313f
     name: Connections Hypothesis Provider API
+  # TRAPI (Translator standard) APIs: Multiomics (made with RTX team Plover)
+  # not accessible by team or api-specific endpoints
+  - id: e51073371d7049b9643e1edbdd61bcbd
+    name: Clinical Trials KP - TRAPI 1.5.0
 exclude: [] # if adding exclude remove these square brackets

config/smartapi_exclusions.yaml

@@ -3,8 +3,3 @@
   name: BioThings Explorer (BTE) TRAPI
 - id: 36f82f05705c317bac17ddae3a0ea2f0
   name: Service Provider TRAPI
-## temp separate registrations for TRAPI 1.5
-- id: 8d97ee37572eaaf9e74df447763fd0c5
-  name: BioThings Explorer (BTE) TRAPI
-- id: a0595049c9a250003c2eced0946d84f6
-  name: Service Provider TRAPI

config/smartapi_overrides.yaml

@@ -1,14 +1,7 @@
 ---
 config:
   only_overrides: false
-apis:
-    32f36164fabed5d3abe6c2fd899c9418: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/biolink-4-update/idisk/smartapi.yaml
-    b48c34df08d16311e3bca06b135b828d: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/biolink-4-update/suppkg/suppkg.yaml
-    e481efd21f8e8c1deac05662439c2294: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/biolink-4-update/ttd/smartapi.yaml
-    1d288b3a3caf75d541ffaae3aab386c8: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/biolink-4-update/semmeddb/smartapi.yaml
-    8f08d1446e0bb9c2b323713ce83e2bd3: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/biolink-4-update/mychem.info/openapi_full.yml
-    1138c3297e8e403b6ac10cff5609b319: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/biolink-4-update/repodb/smartapi.yaml
-    978fe380a147a8641caf72320862697b: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/treats-refactor-tmkp/text_mining/smartapi.yaml
-    38e9e5169a72aee3659c9ddba956790d: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/source_record_urls/bindingdb/smartapi.yaml
-    b772ebfbfa536bba37764d7fddb11d6f: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/source_record_urls/ncats_rare_source/smartapi.yaml
-    59dce17363dce279d389100834e43648: https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/source_record_urls/mygene.info/openapi_full.yml
+apis: {
+  "8f08d1446e0bb9c2b323713ce83e2bd3": "https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/refs/heads/remove-clinical-trials/mychem.info/openapi_full.yml",
+  "1138c3297e8e403b6ac10cff5609b319": "https://raw.githubusercontent.com/NCATS-Tangerine/translator-api-registry/refs/heads/remove-clinical-trials/repodb/smartapi.yaml"
+}

docs/smartapi.yaml

@@ -12,10 +12,10 @@ info:
     component: ARA
     team:
       - Exploring Agent
-    biolink-version: "4.1.6"
+    biolink-version: "4.2.2"
     infores: "infores:biothings-explorer"
   x-trapi:
-    version: "1.5.0"
+    version: 1.5.0
     multicuriequery: false
     pathfinderquery: true  ## dev instance only
     asyncquery: true

src/config/index.ts

@@ -74,23 +74,23 @@ export default class Config {
   setLimiter() {
     const slowLimiter = rateLimit({
       windowMs: 1 * 60 * 1000, //1min
-      max: parseInt(process.env.MAX_QUERIES_PER_MIN || "20"),
+      max: parseInt(process.env.MAX_QUERIES_PER_MIN || "50"),
     });
     const medLimiter = rateLimit({
       windowMs: 1 * 60 * 1000, //1min
-      max: parseInt(process.env.MAX_QUERIES_PER_MIN || "30"),
+      max: parseInt(process.env.MAX_QUERIES_PER_MIN || "100"),
     });
     const fastLimiter = rateLimit({
       windowMs: 1 * 60 * 1000, //1min
-      max: parseInt(process.env.MAX_QUERIES_PER_MIN || "6000"),
+      max: parseInt(process.env.MAX_QUERIES_PER_MIN || "10000"),
     });
     this.app.use("/", fastLimiter);
-    this.app.use("/v1/query", slowLimiter);
+    this.app.use("/v1/query", medLimiter);
     this.app.use("/v1/team/:team_name/query", medLimiter);
-    this.app.use("/v1/team/:smartapiID/query", medLimiter);
-    this.app.use("/v1/meta_knowledge_graph", medLimiter);
-    this.app.use("/v1/team/:teamName/meta_knowledge_graph", medLimiter);
-    this.app.use("/v1/smartapi/:smartapiID/meta_knowledge_graph", medLimiter);
+    this.app.use("/v1/team/:smartapiID/query", fastLimiter);
+    this.app.use("/v1/meta_knowledge_graph", fastLimiter);
+    this.app.use("/v1/team/:teamName/meta_knowledge_graph", fastLimiter);
+    this.app.use("/v1/smartapi/:smartapiID/meta_knowledge_graph", fastLimiter);
     this.app.use("/v1/asyncquery", fastLimiter);
     this.app.use("/v1/team/:teamName/asyncquery", fastLimiter);
     this.app.use("/v1/smartapi/:smartapiID/asyncquery", fastLimiter);

src/controllers/threading/threadHandler.ts

@@ -23,10 +23,10 @@ import { Queue } from "bull";
 const SYNC_MIN_CONCURRENCY = 2;
 const ASYNC_MIN_CONCURRENCY = 3;
 
-// On most instances, there are two nodes, one for Service Provider endpoints and one for everything else
-// On Dev and local instances, this isn't the case, so a lower concurrency is needed
-const CORE_CONCURRENCY_RATIO = parseInt(process.env.CORE_CONCURRENCY_RATIO) || 0.25;
-const MEM_CONCURRENCY_RATIO = parseFloat(process.env.MEM_CONCURRENCY_RATIO) || 0.6;
+// Ratio of Queryies per core
+const CORE_CONCURRENCY_RATIO = parseInt(process.env.CORE_CONCURRENCY_RATIO) || 1;
+// Ratio of Queries per GB of memory
+const MEM_CONCURRENCY_RATIO = parseFloat(process.env.MEM_CONCURRENCY_RATIO) || 2;
 
 const CORE_LIMIT = Math.ceil(os.cpus().length * CORE_CONCURRENCY_RATIO);
 

src/middlewares/error.ts

@@ -7,7 +7,7 @@ const debug = Debug("bte:biothings-explorer-trapi:error_handler");
 import * as Sentry from "@sentry/node";
 import { Express, NextFunction, Request, Response } from "express";
 import StatusError from "../utils/errors/status_error";
-import { TrapiResponse, InvalidQueryGraphError } from "@biothings-explorer/types";
+import { TrapiResponse, InvalidQueryGraphError, NotImplementedError } from "@biothings-explorer/types";
 
 class ErrorHandler {
   shouldHandleError(error: Error) {
@@ -98,6 +98,13 @@ class ErrorHandler {
           .set("Retry-After", String((error as ServerOverloadedError).retryAfter))
           .json(json);
       }
+
+      if (error instanceof NotImplementedError || error.name === 'NotImplementedError') {
+        json.status = "NotImplementedError"
+        json.description = "The feature you are trying to use is not yet implemented."
+        return res.status(501).json(json)
+      }
+
       if (!(error as StatusError).statusCode) (error as StatusError).statusCode = 500;
 
       if ((error as StatusError).statusCode === 301) {