From a3d554ea2b41b2a51ec17eeaa8845220fdcedef5 Mon Sep 17 00:00:00 2001 From: patrick blume Date: Tue, 26 Nov 2024 12:16:34 +0100 Subject: [PATCH 1/3] implement working updated version of findAllChildTerms --- .../SharedComponents/TermSearchInput.fs | 3 +- src/Server/Api/IOntologyAPI.fs | 28 +++++++++++++++---- src/Server/Database/Term.fs | 18 ++++++++++++ src/Shared/DTO.fs | 1 - src/Shared/Shared.fs | 10 ++++--- 5 files changed, 49 insertions(+), 11 deletions(-) diff --git a/src/Client/SharedComponents/TermSearchInput.fs b/src/Client/SharedComponents/TermSearchInput.fs index 993b1df4..bc0f1123 100644 --- a/src/Client/SharedComponents/TermSearchInput.fs +++ b/src/Client/SharedComponents/TermSearchInput.fs @@ -44,7 +44,8 @@ module TermSearchAux = let searchAllByParent(parentTAN: string, setResults: Term [] -> unit) = async { - let! terms = Api.api.getAllTermsByParentTerm <| Shared.SwateObsolete.TermMinimal.create "" parentTAN + let query = TermQuery.create("", 50, parentTAN) + let! terms = Api.ontology.findAllChildTerms query setResults terms } diff --git a/src/Server/Api/IOntologyAPI.fs b/src/Server/Api/IOntologyAPI.fs index 461605c9..5aa9bdf5 100644 --- a/src/Server/Api/IOntologyAPI.fs +++ b/src/Server/Api/IOntologyAPI.fs @@ -46,12 +46,25 @@ module Helper = return dbSearchRes } + let searchChildTerms credentials (content: TermQuery) = + async { + let dbSearchRes = + let searchmode = content.searchMode |> Option.defaultWith (fun () -> getSearchModeFromQuery content.query) + let ontologies = content.ontologies |> Option.defaultValue [] |> getOntologiesModeFromList + if content.parentTermId.IsSome then + Term.Term(credentials).findAllChildTerms(content.parentTermId.Value, ?limit=content.limit) |> Array.ofSeq + else + Term.Term(credentials).searchByName(content.query.Trim(), searchmode, ?limit=content.limit, ?sourceOntologyName = ontologies) |> Array.ofSeq + + return dbSearchRes + } + open Helper [] module V3 = - let ontologyApi (credentials : Helper.Neo4JCredentials) : IOntologyAPIv3 = + let ontologyApi (credentials: Helper.Neo4JCredentials) : IOntologyAPIv3 = { //Development getTestNumber = @@ -80,6 +93,11 @@ module V3 = | 0 -> None | _ -> failwith $"Found multiple terms with the same accession: {id}" // must be multiples as negative cannot exist for length } + findAllChildTerms = fun content -> + async { + let! results = Helper.V3.searchChildTerms credentials content + return results + } } let createIOntologyApi credentials = @@ -97,7 +115,7 @@ open Shared.SwateObsolete.Regex module V1 = /// Deprecated - let ontologyApi (credentials : Helper.Neo4JCredentials) : IOntologyAPIv1 = + let ontologyApi (credentials: Helper.Neo4JCredentials) : IOntologyAPIv1 = /// We use sorensen dice to avoid scoring mutliple occassions of the same word (Issue https://github.com/nfdi4plants/Swate/issues/247) let sorensenDiceSortTerms (searchStr:string) (terms: Term []) = terms |> SorensenDice.sortBySimilarity searchStr (fun term -> term.Name) @@ -193,7 +211,7 @@ module V1 = return filteredResult } - getUnitTermSuggestions = fun (max:int,typedSoFar:string) -> + getUnitTermSuggestions = fun (max:int, typedSoFar:string) -> async { let dbSearchRes = match typedSoFar with @@ -207,7 +225,7 @@ module V1 = return res } - getTermsByNames = fun (queryArr) -> + getTermsByNames = fun queryArr -> async { // check if search string is empty. This case should delete TAN- and TSR- values in table let filteredQueries = queryArr |> Array.filter (fun x -> x.Term.Name <> "" || x.Term.TermAccession <> "") @@ -224,7 +242,7 @@ module V1 = Term.TermQuery.getByName(searchTerm.Term.Name, searchType=FullTextSearch.Exact) ) let result = - Helper.Neo4j.runQueries(queries,credentials) + Helper.Neo4j.runQueries(queries, credentials) |> Array.map2 (fun termSearchable dbResults -> // replicate if..elif..else conditions from 'queries' if termSearchable.Term.TermAccession <> "" then diff --git a/src/Server/Database/Term.fs b/src/Server/Database/Term.fs index 6f3e56aa..bff8290e 100644 --- a/src/Server/Database/Term.fs +++ b/src/Server/Database/Term.fs @@ -185,6 +185,24 @@ type Term(?credentials:Neo4JCredentials, ?session:IAsyncSession) = printfn "%s" exn.Message [||] + /// + /// This is a more complete implementation, which should be abstracted more later. + /// + member this.findAllChildTerms(parentId: string, ?limit: int) = + let query = + sprintf + """MATCH (child)-[*1..]->(:Term {accession: $Accession}) + RETURN child.accession, child.name, child.definition, child.is_obsolete + %s""" + (if limit.IsSome then "LIMIT $Limit" else "") + let param = + Map [ + /// need to box values, because limit.Value will error if parsed as string + "Accession", box parentId + if limit.IsSome then "Limit", box limit.Value + ] |> Some + Neo4j.runQuery(query,param,(Term.asTerm("child")),?session=session,?credentials=credentials) + /// This function will allow for raw apache lucene input. It is possible to search either term name or description or both. /// The function will error if both term name and term description are None. member this.getByAdvancedTermSearch(advancedSearchOptions:Shared.AdvancedSearchTypes.AdvancedSearchOptions) = diff --git a/src/Shared/DTO.fs b/src/Shared/DTO.fs index a3878daf..9c837547 100644 --- a/src/Shared/DTO.fs +++ b/src/Shared/DTO.fs @@ -27,4 +27,3 @@ type TermQueryResults = { query = query results = results } - diff --git a/src/Shared/Shared.fs b/src/Shared/Shared.fs index 934811a8..ac89fdf6 100644 --- a/src/Shared/Shared.fs +++ b/src/Shared/Shared.fs @@ -46,10 +46,12 @@ module SorensenDice = type IOntologyAPIv3 = { // Development - getTestNumber : unit -> Async - searchTerm: TermQuery -> Async - searchTerms: TermQuery[] -> Async - getTermById: string -> Async + getTestNumber : unit -> Async + searchTerm : TermQuery -> Async + searchTerms : TermQuery[] -> Async + getTermById : string -> Async + findAllChildTerms : TermQuery -> Async + } /// Development api From be8da185970010086cdfb5c64c9b652c45d7aa90 Mon Sep 17 00:00:00 2001 From: patrick blume Date: Tue, 26 Nov 2024 14:23:46 +0100 Subject: [PATCH 2/3] optimized code --- src/Server/Database/Term.fs | 61 +++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/src/Server/Database/Term.fs b/src/Server/Database/Term.fs index bff8290e..66e7377f 100644 --- a/src/Server/Database/Term.fs +++ b/src/Server/Database/Term.fs @@ -103,14 +103,14 @@ type Term(?credentials:Neo4JCredentials, ?session:IAsyncSession) = let limit = defaultArg limit 5 let searchNameQuery = Queries.NameQueryFullText ("node", limit=true) let searchTreeQuery = - """MATCH (node:Term) - WHERE node.accession IN $AccessionList - MATCH (endNode:Term {accession: $Parent}) - MATCH (node) - WHERE EXISTS ( - (endNode)<-[:is_a*]-(node) - ) - RETURN node.accession, node.name, node.definition, node.is_obsolete""" + """MATCH (node:Term) + WHERE node.accession IN $AccessionList + MATCH (endNode:Term {accession: $Parent}) + MATCH (node) + WHERE EXISTS ( + (endNode)<-[:is_a*]-(node) + ) + RETURN node.accession, node.name, node.definition, node.is_obsolete""" // These two examples can be used to check function for efficiency coming from both node directions. // Some searches are better optimized starting from child and checking for parent. For other queries it is the other way around, // it depends on the relationship complexity of the parent and/or child node. @@ -189,19 +189,40 @@ type Term(?credentials:Neo4JCredentials, ?session:IAsyncSession) = /// This is a more complete implementation, which should be abstracted more later. /// member this.findAllChildTerms(parentId: string, ?limit: int) = + let limit = defaultArg limit 5 + let query = - sprintf - """MATCH (child)-[*1..]->(:Term {accession: $Accession}) - RETURN child.accession, child.name, child.definition, child.is_obsolete - %s""" - (if limit.IsSome then "LIMIT $Limit" else "") - let param = - Map [ - /// need to box values, because limit.Value will error if parsed as string - "Accession", box parentId - if limit.IsSome then "Limit", box limit.Value - ] |> Some - Neo4j.runQuery(query,param,(Term.asTerm("child")),?session=session,?credentials=credentials) + """MATCH (child)-[:is_a*]->(:Term {accession: $Parent}) + RETURN child.accession, child.name, child.definition, child.is_obsolete + LIMIT $Limit""" + + let config = Action(fun (config : TransactionConfigBuilder) -> config.WithTimeout(TimeSpan.FromSeconds(0.5)) |> ignore) + use session = if session.IsSome then session.Value else Neo4j.establishConnection(credentials.Value) + let main = + task { + let! tree_query = + let parameters = System.Collections.Generic.Dictionary([ + KeyValuePair("Parent", box parentId); + KeyValuePair("Limit", box limit) + ]) + session.RunAsync(query, parameters, config) + + let! tree_records = tree_query.ToListAsync() + let tree_results = + [| + for record in tree_records do + yield Term.asTerm("child") record + |] + return tree_results + } + try + main.Result + with + | exn -> + printfn "%s" exn.Message + [||] + + //Neo4j.runQuery(query, param, (Term.asTerm("child")), ?session=session, ?credentials=credentials) /// This function will allow for raw apache lucene input. It is possible to search either term name or description or both. /// The function will error if both term name and term description are None. From 6e37d7cab457a96bb68b402c6f153fe2339d4964 Mon Sep 17 00:00:00 2001 From: patrick blume Date: Wed, 27 Nov 2024 09:11:30 +0100 Subject: [PATCH 3/3] Implement review changes --- src/Client/OfficeInterop/OfficeInterop.fs | 6 +++--- .../SharedComponents/TermSearchInput.fs | 15 ++++++------- src/Server/Api/IOntologyAPI.fs | 19 +++++++---------- src/Server/Database/Term.fs | 19 +++++++++-------- src/Server/Database/TreeSearch.fs | 2 +- src/Shared/DTOs/ParentTermQueryDto.fs | 21 +++++++++++++++++++ src/Shared/{DTO.fs => DTOs/TermQueryDto.fs} | 16 +++++++------- src/Shared/Shared.fs | 13 ++++++------ src/Shared/Shared.fsproj | 3 ++- 9 files changed, 67 insertions(+), 47 deletions(-) create mode 100644 src/Shared/DTOs/ParentTermQueryDto.fs rename src/Shared/{DTO.fs => DTOs/TermQueryDto.fs} (76%) diff --git a/src/Client/OfficeInterop/OfficeInterop.fs b/src/Client/OfficeInterop/OfficeInterop.fs index 302e5336..a42fca99 100644 --- a/src/Client/OfficeInterop/OfficeInterop.fs +++ b/src/Client/OfficeInterop/OfficeInterop.fs @@ -9,7 +9,7 @@ open GlobalBindings open Shared open Database -open DTO +open DTOs.TermQuery open OfficeInterop open OfficeInterop.ExcelUtil @@ -1635,7 +1635,7 @@ let validateSelectedAndNeighbouringBuildingBlocks () = /// let searchTermInDatabase name = promise { - let term = TermQuery.create(name, searchMode=Database.FullTextSearch.Exact) + let term = TermQueryDto.create(name, searchMode=Database.FullTextSearch.Exact) let! results = Async.StartAsPromise(Api.ontology.searchTerm term) let result = Array.tryHead results return result @@ -1650,7 +1650,7 @@ let searchTermsInDatabase names = let terms = names |> List.map (fun name -> - TermQuery.create(name, searchMode=Database.FullTextSearch.Exact) + TermQueryDto.create(name, searchMode=Database.FullTextSearch.Exact) ) |> Array.ofSeq let! result = Async.StartAsPromise(Api.ontology.searchTerms terms) diff --git a/src/Client/SharedComponents/TermSearchInput.fs b/src/Client/SharedComponents/TermSearchInput.fs index bc0f1123..22d136c6 100644 --- a/src/Client/SharedComponents/TermSearchInput.fs +++ b/src/Client/SharedComponents/TermSearchInput.fs @@ -6,7 +6,8 @@ open Browser.Types open ARCtrl open Shared open Shared.Database -open DTO +open Shared.DTOs.TermQuery +open Shared.DTOs.ParentTermQuery open Fable.Core.JsInterop module TermSearchAux = @@ -30,23 +31,23 @@ module TermSearchAux = let searchByName(query: string, setResults: Term [] -> unit) = async { - let query = TermQuery.create(query, 10) + let query = TermQueryDto.create(query, 10) let! terms = Api.ontology.searchTerm query setResults terms } let searchByParent(query: string, parentTAN: string, setResults: Term [] -> unit) = async { - let query = TermQuery.create(query, 50, parentTAN) + let query = TermQueryDto.create(query, 50, parentTAN) let! terms = Api.ontology.searchTerm query setResults terms } - let searchAllByParent(parentTAN: string, setResults: Term [] -> unit) = + let findAllChildTerms(parentTAN: string, setResults: Term [] -> unit) = async { - let query = TermQuery.create("", 50, parentTAN) + let query = ParentTermQueryDto.create(parentTAN, 50) let! terms = Api.ontology.findAllChildTerms query - setResults terms + setResults terms.results } let allByParentSearch ( @@ -62,7 +63,7 @@ module TermSearchAux = async { ClickOutsideHandler.AddListener(SelectAreaID, fun e -> stopSearch()) } - searchAllByParent(parent.TermAccessionShort, fun terms -> setSearchTreeState {Results = terms; SearchIs = SearchIs.Done}) + findAllChildTerms(parent.TermAccessionShort, fun terms -> setSearchTreeState {Results = terms; SearchIs = SearchIs.Done}) ] |> Async.Parallel |> Async.Ignore diff --git a/src/Server/Api/IOntologyAPI.fs b/src/Server/Api/IOntologyAPI.fs index 5aa9bdf5..83a969bc 100644 --- a/src/Server/Api/IOntologyAPI.fs +++ b/src/Server/Api/IOntologyAPI.fs @@ -3,7 +3,8 @@ module API.IOntologyAPI open Shared open Database -open DTO +open DTOs.TermQuery +open DTOs.ParentTermQuery open Fable.Remoting.Server open Fable.Remoting.Giraffe open ARCtrl @@ -26,7 +27,7 @@ module Helper = module V3 = open ARCtrl.Helper.Regex.ActivePatterns - let searchSingleTerm credentials (content: TermQuery) = + let searchSingleTerm credentials (content: TermQueryDto) = async { let dbSearchRes = match content.query.Trim() with @@ -46,15 +47,10 @@ module Helper = return dbSearchRes } - let searchChildTerms credentials (content: TermQuery) = + let searchChildTerms credentials (content: ParentTermQueryDto) = async { let dbSearchRes = - let searchmode = content.searchMode |> Option.defaultWith (fun () -> getSearchModeFromQuery content.query) - let ontologies = content.ontologies |> Option.defaultValue [] |> getOntologiesModeFromList - if content.parentTermId.IsSome then - Term.Term(credentials).findAllChildTerms(content.parentTermId.Value, ?limit=content.limit) |> Array.ofSeq - else - Term.Term(credentials).searchByName(content.query.Trim(), searchmode, ?limit=content.limit, ?sourceOntologyName = ontologies) |> Array.ofSeq + Term.Term(credentials).findAllChildTerms(content.parentTermId, ?limit=content.limit) |> Array.ofSeq return dbSearchRes } @@ -81,7 +77,7 @@ module V3 = Helper.V3.searchSingleTerm credentials query ] let! results = asyncQueries |> Async.Parallel - let zipped = Array.map2 (fun a b -> TermQueryResults.create(a,b)) queries results + let zipped = Array.map2 (fun a b -> TermQueryDtoResults.create(a,b)) queries results return zipped } getTermById = fun id -> @@ -96,7 +92,8 @@ module V3 = findAllChildTerms = fun content -> async { let! results = Helper.V3.searchChildTerms credentials content - return results + let zipped = ParentTermQueryDtoResults.create(content, results) + return zipped } } diff --git a/src/Server/Database/Term.fs b/src/Server/Database/Term.fs index 66e7377f..3a9f89b9 100644 --- a/src/Server/Database/Term.fs +++ b/src/Server/Database/Term.fs @@ -1,7 +1,6 @@ module Database.Term open Neo4j.Driver -open Shared.DTO open Shared.Database open Shared.SwateObsolete open Helper @@ -104,13 +103,13 @@ type Term(?credentials:Neo4JCredentials, ?session:IAsyncSession) = let searchNameQuery = Queries.NameQueryFullText ("node", limit=true) let searchTreeQuery = """MATCH (node:Term) - WHERE node.accession IN $AccessionList - MATCH (endNode:Term {accession: $Parent}) - MATCH (node) - WHERE EXISTS ( - (endNode)<-[:is_a*]-(node) - ) - RETURN node.accession, node.name, node.definition, node.is_obsolete""" + WHERE node.accession IN $AccessionList + MATCH (endNode:Term {accession: $Parent}) + MATCH (node) + WHERE EXISTS ( + (endNode)<-[:is_a*]-(node) + ) + RETURN node.accession, node.name, node.definition, node.is_obsolete""" // These two examples can be used to check function for efficiency coming from both node directions. // Some searches are better optimized starting from child and checking for parent. For other queries it is the other way around, // it depends on the relationship complexity of the parent and/or child node. @@ -186,8 +185,10 @@ type Term(?credentials:Neo4JCredentials, ?session:IAsyncSession) = [||] /// - /// This is a more complete implementation, which should be abstracted more later. + /// Find all child terms of a given parent /// + /// + /// member this.findAllChildTerms(parentId: string, ?limit: int) = let limit = defaultArg limit 5 diff --git a/src/Server/Database/TreeSearch.fs b/src/Server/Database/TreeSearch.fs index ac68aefd..f8bbae7e 100644 --- a/src/Server/Database/TreeSearch.fs +++ b/src/Server/Database/TreeSearch.fs @@ -3,7 +3,7 @@ module Database.TreeSearch open System open Neo4j.Driver -open Shared.DTO +open Shared.DTOs open Shared.Database open Shared.Database.TreeTypes diff --git a/src/Shared/DTOs/ParentTermQueryDto.fs b/src/Shared/DTOs/ParentTermQueryDto.fs new file mode 100644 index 00000000..b2854b27 --- /dev/null +++ b/src/Shared/DTOs/ParentTermQueryDto.fs @@ -0,0 +1,21 @@ +module Shared.DTOs.ParentTermQuery + +open Shared + +type ParentTermQueryDto = { + parentTermId: string + limit: int option +} with + static member create(parentTermId, ?limit) = { + parentTermId = parentTermId + limit = limit + } + +type ParentTermQueryDtoResults = { + query: ParentTermQueryDto + results: Database.Term [] +} with + static member create(query, results) = { + query = query + results = results + } \ No newline at end of file diff --git a/src/Shared/DTO.fs b/src/Shared/DTOs/TermQueryDto.fs similarity index 76% rename from src/Shared/DTO.fs rename to src/Shared/DTOs/TermQueryDto.fs index 9c837547..a3670452 100644 --- a/src/Shared/DTO.fs +++ b/src/Shared/DTOs/TermQueryDto.fs @@ -1,10 +1,8 @@ -module Shared.DTO +module Shared.DTOs.TermQuery -open ARCtrl +open Shared -open Database - -type TermQuery = { +type TermQueryDto = { query: string limit: int option parentTermId: string option @@ -19,11 +17,11 @@ type TermQuery = { searchMode = searchMode } -type TermQueryResults = { - query: TermQuery - results: Term [] +type TermQueryDtoResults = { + query: TermQueryDto + results: Database.Term [] } with static member create(query, results) = { query = query results = results - } + } \ No newline at end of file diff --git a/src/Shared/Shared.fs b/src/Shared/Shared.fs index ac89fdf6..028c701a 100644 --- a/src/Shared/Shared.fs +++ b/src/Shared/Shared.fs @@ -3,7 +3,8 @@ namespace Shared open System open Shared open Database -open DTO +open DTOs.TermQuery +open DTOs.ParentTermQuery [] module Regex = @@ -46,11 +47,11 @@ module SorensenDice = type IOntologyAPIv3 = { // Development - getTestNumber : unit -> Async - searchTerm : TermQuery -> Async - searchTerms : TermQuery[] -> Async - getTermById : string -> Async - findAllChildTerms : TermQuery -> Async + getTestNumber : unit -> Async + searchTerm : TermQueryDto -> Async + searchTerms : TermQueryDto[] -> Async + getTermById : string -> Async + findAllChildTerms : ParentTermQueryDto -> Async } diff --git a/src/Shared/Shared.fsproj b/src/Shared/Shared.fsproj index 31d37d4f..a426ba67 100644 --- a/src/Shared/Shared.fsproj +++ b/src/Shared/Shared.fsproj @@ -5,10 +5,11 @@ + + -