From eb61af176adc7c46b2b46ce93dd029f1843a8f0a Mon Sep 17 00:00:00 2001 From: NeuralFlux <40491005+NeuralFlux@users.noreply.github.com> Date: Thu, 31 Oct 2024 13:04:39 -0400 Subject: [PATCH 1/3] chore: refactor route var names to camel case --- src/config.ts | 4 ++-- src/load/single_spec_async_loader.ts | 6 +++--- src/load/team_specs_async_loader.ts | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/config.ts b/src/config.ts index eabb585..358135c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -20,10 +20,10 @@ export const FILTER_FIELDS: FIELDS_FOR_FILTER[] = [ export const SMARTAPI_URL = "https://smart-api.info/api/query?q=tags.name:translator&size=1000&fields=paths,servers,tags,components.x-bte*,info,_meta"; -export const SINGLE_API_SMARTAPI_QUERY_TEMPLATE = "https://smart-api.info/api/metadata/{smartapi_id}"; +export const SINGLE_API_SMARTAPI_QUERY_TEMPLATE = "https://smart-api.info/api/metadata/{smartAPIID}"; export const TEAM_SMARTAPI_QUERY_TEMPLATE = - 'https://smart-api.info/api/query?q=info.x-translator.team:"{team_name}"&size=1000&fields=paths,servers,tags,components.x-bte*,info,_meta'; + 'https://smart-api.info/api/query?q=info.x-translator.team:"{teamName}"&size=1000&fields=paths,servers,tags,components.x-bte*,info,_meta'; export const COMPONENT_SMARTAPI_QUERY_TEMPLATE = 'https://smart-api.info/api/query?q=info.x-translator.component:"{component_name}"&size=1000&fields=paths,servers,tags,components.x-bte*,info,_meta'; diff --git a/src/load/single_spec_async_loader.ts b/src/load/single_spec_async_loader.ts index 34110b3..d13e4df 100644 --- a/src/load/single_spec_async_loader.ts +++ b/src/load/single_spec_async_loader.ts @@ -4,9 +4,9 @@ import { SmartAPISpec } from "../parser/types"; export default class SingleSpecAsyncLoader extends BaseAsyncLoader { private _smartapi_id: string; - constructor(smartapiID: string) { - super(SINGLE_API_SMARTAPI_QUERY_TEMPLATE.replace("{smartapi_id}", smartapiID)); - this._smartapi_id = smartapiID; + constructor(smartAPIID: string) { + super(SINGLE_API_SMARTAPI_QUERY_TEMPLATE.replace("{smartAPIID}", smartAPIID)); + this._smartapi_id = smartAPIID; } protected async fetch(): Promise { diff --git a/src/load/team_specs_async_loader.ts b/src/load/team_specs_async_loader.ts index cc44cbe..d78d562 100644 --- a/src/load/team_specs_async_loader.ts +++ b/src/load/team_specs_async_loader.ts @@ -5,7 +5,7 @@ import { SmartAPISpec } from "../parser/types"; export default class TeamSpecsAsyncLoader extends BaseAsyncLoader { constructor(teamName: string) { - super(TEAM_SMARTAPI_QUERY_TEMPLATE.replace("{team_name}", teamName)); + super(TEAM_SMARTAPI_QUERY_TEMPLATE.replace("{teamName}", teamName)); } protected async fetch(): Promise { From 0dd10e559dd908759cc8659b8822b16b6e341730 Mon Sep 17 00:00:00 2001 From: NeuralFlux <40491005+NeuralFlux@users.noreply.github.com> Date: Tue, 5 Nov 2024 12:50:13 -0500 Subject: [PATCH 2/3] fix: add file locking while building ops --- package.json | 4 +- .../sync_operations_builder.ts | 24 +++--- .../sync_operations_builder_with_reasoner.ts | 80 +++++++++++-------- 3 files changed, 65 insertions(+), 43 deletions(-) diff --git a/package.json b/package.json index 810457b..ce8210e 100644 --- a/package.json +++ b/package.json @@ -43,9 +43,9 @@ "@babel/preset-typescript": "^7.23.2", "@types/debug": "^4.1.10", "@types/express": "^4.17.20", - "@types/node": "^14.18.63", "@types/jest": "^26.0.24", "@types/lodash": "^4.14.200", + "@types/node": "^14.18.63", "@typescript-eslint/eslint-plugin": "^6.8.0", "@typescript-eslint/parser": "^6.8.0", "babel-loader": "^8.3.0", @@ -71,9 +71,9 @@ }, "dependencies": { "@babel/runtime": "^7.23.2", + "@biothings-explorer/utils": "workspace:../utils", "@commitlint/cli": "^17.8.1", "@commitlint/config-conventional": "^11.0.0", - "@biothings-explorer/utils": "workspace:../utils", "axios": "^0.21.4", "camelcase": "^6.3.0", "core-js": "^3.33.1", diff --git a/src/operations_builder/sync_operations_builder.ts b/src/operations_builder/sync_operations_builder.ts index bd31788..240ccea 100644 --- a/src/operations_builder/sync_operations_builder.ts +++ b/src/operations_builder/sync_operations_builder.ts @@ -1,6 +1,9 @@ import BaseOperationsBuilder from "./base_operations_builder"; import { syncLoaderFactory } from "../load/sync_loader_factory"; import { BuilderOptions } from "../types"; +import { lockWithActionSync } from "@biothings-explorer/utils"; +import { SmartAPISpec } from "../parser/types"; +const debug = require("debug")("bte:smartapi-kg:SyncOperationsBuilder"); export default class SyncOperationsBuilder extends BaseOperationsBuilder { private _file_path: string; @@ -11,15 +14,18 @@ export default class SyncOperationsBuilder extends BaseOperationsBuilder { } build() { - const specs = syncLoaderFactory( - this._options.smartAPIID, - this._options.teamName, - this._options.tag, - this._options.component, - this._options.apiList, - this._file_path, - this._options.smartapiSpecs - ); + let specs: SmartAPISpec[] = lockWithActionSync(this._file_path, () => { + return syncLoaderFactory( + this._options.smartAPIID, + this._options.teamName, + this._options.tag, + this._options.component, + this._options.apiList, + this._file_path, + this._options.smartapiSpecs + ); + }, debug); + return this.loadOpsFromSpecs(specs); } } diff --git a/src/operations_builder/sync_operations_builder_with_reasoner.ts b/src/operations_builder/sync_operations_builder_with_reasoner.ts index 586115a..8b51b08 100644 --- a/src/operations_builder/sync_operations_builder_with_reasoner.ts +++ b/src/operations_builder/sync_operations_builder_with_reasoner.ts @@ -7,7 +7,7 @@ import { PredicatesMetadata } from "../types"; import Debug from "debug"; const debug = Debug("bte:smartapi-kg:SyncOperationsBuilderWithReasoner"); import { SmartAPISpec } from "../parser/types"; -import { biolink } from "@biothings-explorer/utils"; +import { biolink, lockWithActionSync } from "@biothings-explorer/utils"; declare global { var missingAPIs: SmartAPISpec[]; @@ -220,39 +220,55 @@ export default class SyncOperationsBuilderWithReasoner extends BaseOperationsBui } build() { - const specs = syncLoaderFactory( - this._options.smartAPIID, - this._options.teamName, - this._options.tag, - this._options.component, - this._options.apiList, - this._file_path, - this._options.smartapiSpecs, - ); - const nonTRAPIOps = this.loadOpsFromSpecs(specs); - const predicatesMetadata = this.fetch(); - global.missingAPIs = syncLoaderFactory( - undefined, - undefined, - undefined, - undefined, - undefined, + const predicatesMetadata: PredicatesMetadata[] = lockWithActionSync( + this._predicates_file_path, + () => this.fetch(), + debug + ) + + let specs: SmartAPISpec[]; + let nonTRAPIOps: SmartAPIKGOperationObject[]; + + lockWithActionSync( this._file_path, - this._options.smartapiSpecs, - ).filter( - spec => - "info" in spec && - "x-translator" in spec.info && - spec.info["x-translator"].component === "KP" && - "paths" in spec && - "/query" in spec.paths && - "x-trapi" in spec.info && - spec.servers.length && - "/meta_knowledge_graph" in spec.paths && - !predicatesMetadata - .map(m => m.association.smartapi.id) - .includes(spec._id), + () => { + specs = syncLoaderFactory( + this._options.smartAPIID, + this._options.teamName, + this._options.tag, + this._options.component, + this._options.apiList, + this._file_path, + this._options.smartapiSpecs, + ); + + nonTRAPIOps = this.loadOpsFromSpecs(specs); + global.missingAPIs = syncLoaderFactory( + undefined, + undefined, + undefined, + undefined, + undefined, + this._file_path, + this._options.smartapiSpecs, + ).filter( + spec => + "info" in spec && + "x-translator" in spec.info && + spec.info["x-translator"].component === "KP" && + "paths" in spec && + "/query" in spec.paths && + "x-trapi" in spec.info && + spec.servers.length && + "/meta_knowledge_graph" in spec.paths && + !predicatesMetadata + .map(m => m.association.smartapi.id) + .includes(spec._id), + ); + }, + debug ); + let TRAPIOps = [] as SmartAPIKGOperationObject[]; predicatesMetadata.map(metadata => { TRAPIOps.push.apply(TRAPIOps, this.parsePredicateEndpoint(metadata)); From bc4e020125da57935e83301c46dabdcebc24530a Mon Sep 17 00:00:00 2001 From: NeuralFlux <40491005+NeuralFlux@users.noreply.github.com> Date: Tue, 5 Nov 2024 17:34:13 -0500 Subject: [PATCH 3/3] fix: make sync kg construction support async file locks --- src/index.ts | 13 +++ .../sync_operations_builder.ts | 24 +++--- .../sync_operations_builder_with_reasoner.ts | 80 ++++++++----------- 3 files changed, 54 insertions(+), 63 deletions(-) diff --git a/src/index.ts b/src/index.ts index 999ebeb..62815e8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,6 +6,7 @@ import { ft } from "./filter"; import path from "path"; import Debug from "debug"; import QueryOperationObject from "./parser/query_operation"; +import { lockWithActionAsync } from "@biothings-explorer/utils"; const debug = Debug("bte:smartapi-kg:MetaKG"); export * from "./types"; @@ -66,6 +67,18 @@ export default class MetaKG { return this.ops; } + /* Async wrapper for using constructMetaKGSync to enable using async file locking */ + async constructMetaKGWithFileLock(includeReasoner = false, options: BuilderOptions = {}): Promise { + this._ops = await lockWithActionAsync( + [this._file_path, this._predicates_path], + async () => { + return syncBuilderFactory(options, includeReasoner, this._file_path, this._predicates_path); + }, + debug + ); + return this.ops; + } + /** * Filters the whole meta kg based on apiList, teamName, tag or component * @param {Object} options - filtering options diff --git a/src/operations_builder/sync_operations_builder.ts b/src/operations_builder/sync_operations_builder.ts index 240ccea..bd31788 100644 --- a/src/operations_builder/sync_operations_builder.ts +++ b/src/operations_builder/sync_operations_builder.ts @@ -1,9 +1,6 @@ import BaseOperationsBuilder from "./base_operations_builder"; import { syncLoaderFactory } from "../load/sync_loader_factory"; import { BuilderOptions } from "../types"; -import { lockWithActionSync } from "@biothings-explorer/utils"; -import { SmartAPISpec } from "../parser/types"; -const debug = require("debug")("bte:smartapi-kg:SyncOperationsBuilder"); export default class SyncOperationsBuilder extends BaseOperationsBuilder { private _file_path: string; @@ -14,18 +11,15 @@ export default class SyncOperationsBuilder extends BaseOperationsBuilder { } build() { - let specs: SmartAPISpec[] = lockWithActionSync(this._file_path, () => { - return syncLoaderFactory( - this._options.smartAPIID, - this._options.teamName, - this._options.tag, - this._options.component, - this._options.apiList, - this._file_path, - this._options.smartapiSpecs - ); - }, debug); - + const specs = syncLoaderFactory( + this._options.smartAPIID, + this._options.teamName, + this._options.tag, + this._options.component, + this._options.apiList, + this._file_path, + this._options.smartapiSpecs + ); return this.loadOpsFromSpecs(specs); } } diff --git a/src/operations_builder/sync_operations_builder_with_reasoner.ts b/src/operations_builder/sync_operations_builder_with_reasoner.ts index 8b51b08..586115a 100644 --- a/src/operations_builder/sync_operations_builder_with_reasoner.ts +++ b/src/operations_builder/sync_operations_builder_with_reasoner.ts @@ -7,7 +7,7 @@ import { PredicatesMetadata } from "../types"; import Debug from "debug"; const debug = Debug("bte:smartapi-kg:SyncOperationsBuilderWithReasoner"); import { SmartAPISpec } from "../parser/types"; -import { biolink, lockWithActionSync } from "@biothings-explorer/utils"; +import { biolink } from "@biothings-explorer/utils"; declare global { var missingAPIs: SmartAPISpec[]; @@ -220,55 +220,39 @@ export default class SyncOperationsBuilderWithReasoner extends BaseOperationsBui } build() { - const predicatesMetadata: PredicatesMetadata[] = lockWithActionSync( - this._predicates_file_path, - () => this.fetch(), - debug - ) - - let specs: SmartAPISpec[]; - let nonTRAPIOps: SmartAPIKGOperationObject[]; - - lockWithActionSync( + const specs = syncLoaderFactory( + this._options.smartAPIID, + this._options.teamName, + this._options.tag, + this._options.component, + this._options.apiList, this._file_path, - () => { - specs = syncLoaderFactory( - this._options.smartAPIID, - this._options.teamName, - this._options.tag, - this._options.component, - this._options.apiList, - this._file_path, - this._options.smartapiSpecs, - ); - - nonTRAPIOps = this.loadOpsFromSpecs(specs); - global.missingAPIs = syncLoaderFactory( - undefined, - undefined, - undefined, - undefined, - undefined, - this._file_path, - this._options.smartapiSpecs, - ).filter( - spec => - "info" in spec && - "x-translator" in spec.info && - spec.info["x-translator"].component === "KP" && - "paths" in spec && - "/query" in spec.paths && - "x-trapi" in spec.info && - spec.servers.length && - "/meta_knowledge_graph" in spec.paths && - !predicatesMetadata - .map(m => m.association.smartapi.id) - .includes(spec._id), - ); - }, - debug + this._options.smartapiSpecs, + ); + const nonTRAPIOps = this.loadOpsFromSpecs(specs); + const predicatesMetadata = this.fetch(); + global.missingAPIs = syncLoaderFactory( + undefined, + undefined, + undefined, + undefined, + undefined, + this._file_path, + this._options.smartapiSpecs, + ).filter( + spec => + "info" in spec && + "x-translator" in spec.info && + spec.info["x-translator"].component === "KP" && + "paths" in spec && + "/query" in spec.paths && + "x-trapi" in spec.info && + spec.servers.length && + "/meta_knowledge_graph" in spec.paths && + !predicatesMetadata + .map(m => m.association.smartapi.id) + .includes(spec._id), ); - let TRAPIOps = [] as SmartAPIKGOperationObject[]; predicatesMetadata.map(metadata => { TRAPIOps.push.apply(TRAPIOps, this.parsePredicateEndpoint(metadata));