Skip to content

Commit

Permalink
Merge pull request #548 from nfdi4plants/ontology_api_v3_adjusted
Browse files Browse the repository at this point in the history 
adjust ontology v3 api
  • Loading branch information
@Freymaurer
Freymaurer authored Oct 28, 2024
2 parents d7e59a7 + cd182c3 commit d05b563
Show file tree
Hide file tree
Showing 10 changed files with 119 additions and 223 deletions.
7 changes: 5 additions & 2 deletions src/Client/SharedComponents/TermSearchInput.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ open Feliz.Bulma
open Browser.Types
open ARCtrl
open Shared
open Shared.TermTypes
open Fable.Core.JsInterop

module TermSearchAux =
Expand All @@ -28,13 +29,15 @@ module TermSearchAux =

let searchByName(query: string, setResults: TermTypes.Term [] -> unit) =
async {
let! terms = Api.ontology.searchTerms {|limit = 10; ontologies = []; query=query|}
let query = TermQuery.create(query, 10)
let! terms = Api.ontology.searchTerm query
setResults terms
}

let searchByParent(query: string, parentTAN: string, setResults: TermTypes.Term [] -> unit) =
async {
let! terms = Api.ontology.searchTermsByParent {|limit = 50; parentTAN = parentTAN; query = query|}
let query = TermQuery.create(query, 50, parentTAN)
let! terms = Api.ontology.searchTerm query
setResults terms
}

Expand Down
86 changes: 46 additions & 40 deletions src/Server/Api/IOntologyAPI.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ module Helper =

let getSearchModeFromQuery (query: string) =
let searchTextLength = query.Length
let searchmode = if searchTextLength < 3 then Database.Helper.FullTextSearch.Exact else Database.Helper.FullTextSearch.PerformanceComplete
let searchmode = if searchTextLength < 3 then Database.FullTextSearch.Exact else Database.FullTextSearch.PerformanceComplete
searchmode

let getOntologiesModeFromList (ontologiesList: string list) =
Expand All @@ -23,47 +23,53 @@ module Helper =
else
Term.AnyOfOntology.Multiples ontologiesList |> Some


module V3 =
open ARCtrl.Helper.Regex.ActivePatterns

let searchSingleTerm credentials (content: TermQuery) =
async {
let dbSearchRes =
match content.query.Trim() with
| TermAnnotationShort taninfo ->
Term.Term(credentials).getByAccession $"{taninfo.IDSpace}:{taninfo.LocalID}"
// This suggests we search for a term name
| notAnAccession ->
let searchmode = content.searchMode |> Option.defaultWith (fun () -> getSearchModeFromQuery content.query)
let ontologies = content.ontologies |> Option.defaultValue [] |> getOntologiesModeFromList
match content.parentTermId with
| None ->
Term.Term(credentials).getByName(notAnAccession, searchmode, ?limit=content.limit, ?sourceOntologyName = ontologies)
| Some tan ->
Term.Term(credentials).searchByParentStepwise(notAnAccession, tan, searchmode, ?limit=content.limit)
|> Array.ofSeq
return dbSearchRes
}

open Helper

[<RequireQualifiedAccess>]
module V3 =

open ARCtrl.Helper.Regex.ActivePatterns

let ontologyApi (credentials : Helper.Neo4JCredentials) : IOntologyAPIv3 =
{
//Development
getTestNumber =
fun () -> async { return 42 }
searchTerms = fun content ->
searchTerm = fun content ->
async {
let dbSearchRes =
match content.query with
| TermAnnotationShort taninfo ->
Term.Term(credentials).getByAccession $"{taninfo.IDSpace}:{taninfo.LocalID}"
// This suggests we search for a term name
| notAnAccession ->
let searchmode = getSearchModeFromQuery content.query
let ontologies = getOntologiesModeFromList content.ontologies
Term.Term(credentials).getByName(notAnAccession, searchmode, ?sourceOntologyName = ontologies, limit=content.limit)
|> Array.ofSeq
return dbSearchRes
let! results = Helper.V3.searchSingleTerm credentials content
return results
}
searchTermsByParent = fun content ->
searchTerms = fun queries ->
async {
let dbSearchRes =
match content.query.Trim() with
| TermAnnotationShort taninfo ->
Term.Term(credentials).getByAccession $"{taninfo.IDSpace}:{taninfo.LocalID}"
| notAnAccession ->
let searchmode = getSearchModeFromQuery notAnAccession
match content.parentTAN.Trim() with
| "" ->
Term.Term(credentials).getByName(notAnAccession, searchmode, limit=content.limit)
| parentTAN ->
Term.Term(credentials).searchByParentStepwise(notAnAccession, parentTAN, searchmode, limit=content.limit)
|> Array.ofSeq
return dbSearchRes
let asyncQueries = [
for query in queries do
Helper.V3.searchSingleTerm credentials query
]
let! results = asyncQueries |> Async.Parallel
let zipped = Array.map2 (fun a b -> TermQueryResults.create(a,b)) queries results
return zipped
}
getTermById = fun id ->
async {
Expand Down Expand Up @@ -117,7 +123,7 @@ module V1 =
// This suggests we search for a term name
| notAnAccession ->
let searchTextLength = typedSoFar.Length
let searchmode = if searchTextLength < 3 then Database.Helper.FullTextSearch.Exact else Database.Helper.FullTextSearch.PerformanceComplete
let searchmode = if searchTextLength < 3 then Database.FullTextSearch.Exact else Database.FullTextSearch.PerformanceComplete
Term.Term(credentials).getByName(notAnAccession, searchmode)
|> Array.ofSeq
let arr = if dbSearchRes.Length <= max then dbSearchRes else Array.take max dbSearchRes
Expand All @@ -131,7 +137,7 @@ module V1 =
| Regex.Aux.Regex Regex.Pattern.TermAnnotationShortPattern foundAccession ->
Database.Term.Term(credentials).getByAccession foundAccession.Value
| _ ->
let searchmode = if typedSoFar.Length < 3 then Database.Helper.FullTextSearch.Exact else Database.Helper.FullTextSearch.PerformanceComplete
let searchmode = if typedSoFar.Length < 3 then Database.FullTextSearch.Exact else Database.FullTextSearch.PerformanceComplete
if parentTerm.TermAccession = ""
then
Term.Term(credentials).getByNameAndParent_Name(typedSoFar, parentTerm.Name, searchmode)
Expand All @@ -158,9 +164,9 @@ module V1 =
| _ ->
if childTerm.TermAccession = ""
then
Term.Term(credentials).getByNameAndChild_Name (typedSoFar,childTerm.Name,Helper.FullTextSearch.PerformanceComplete)
Term.Term(credentials).getByNameAndChild_Name (typedSoFar,childTerm.Name,FullTextSearch.PerformanceComplete)
else
Term.Term(credentials).getByNameAndChild(typedSoFar,childTerm.TermAccession,Helper.FullTextSearch.PerformanceComplete)
Term.Term(credentials).getByNameAndChild(typedSoFar,childTerm.TermAccession,FullTextSearch.PerformanceComplete)
|> Array.ofSeq
//|> sorensenDiceSortTerms typedSoFar
let res = if dbSearchRes.Length <= max then dbSearchRes else Array.take max dbSearchRes
Expand Down Expand Up @@ -209,10 +215,10 @@ module V1 =
Term.TermQuery.getByAccession searchTerm.Term.TermAccession
// if term is a unit it should be contained inside the unit ontology, if not it is most likely free text input.
elif searchTerm.IsUnit then
Term.TermQuery.getByName(searchTerm.Term.Name, searchType=Helper.FullTextSearch.Exact, sourceOntologyName= Term.AnyOfOntology.Single "uo")
Term.TermQuery.getByName(searchTerm.Term.Name, searchType=FullTextSearch.Exact, sourceOntologyName= Term.AnyOfOntology.Single "uo")
// if none of the above apply we do a standard term search
else
Term.TermQuery.getByName(searchTerm.Term.Name, searchType=Helper.FullTextSearch.Exact)
Term.TermQuery.getByName(searchTerm.Term.Name, searchType=FullTextSearch.Exact)
)
let result =
Helper.Neo4j.runQueries(queries,credentials)
Expand Down Expand Up @@ -289,7 +295,7 @@ module V2 =
// This suggests we search for a term name
| notAnAccession ->
let searchTextLength = inp.query.Length
let searchmode = if searchTextLength < 3 then Database.Helper.FullTextSearch.Exact else Database.Helper.FullTextSearch.PerformanceComplete
let searchmode = if searchTextLength < 3 then Database.FullTextSearch.Exact else Database.FullTextSearch.PerformanceComplete
Term.Term(credentials).getByName(notAnAccession, searchmode, ?sourceOntologyName = Option.map Term.AnyOfOntology.Single inp.ontology)
|> Array.ofSeq
//|> sorensenDiceSortTerms typedSoFar
Expand All @@ -307,7 +313,7 @@ module V2 =
Database.Term.Term(credentials).getByAccession foundAccession.Value
| _ ->
printfn "[getTermSuggestionsByParentTerm] Hit default search"
let searchmode = if inp.query.Length < 3 then Database.Helper.FullTextSearch.Exact else Database.Helper.FullTextSearch.PerformanceComplete
let searchmode = if inp.query.Length < 3 then Database.FullTextSearch.Exact else Database.FullTextSearch.PerformanceComplete
printfn "[getTermSuggestionsByParentTerm] searchmode: %A" searchmode
if inp.parent_term.TermAccession = ""
then
Expand Down Expand Up @@ -337,7 +343,7 @@ module V2 =
| Regex.Aux.Regex Regex.Pattern.TermAnnotationShortPattern foundAccession ->
Term.Term(credentials).getByAccession foundAccession.Value
| _ ->
let searchmode = if inp.query.Length < 3 then Database.Helper.FullTextSearch.Exact else Database.Helper.FullTextSearch.PerformanceComplete
let searchmode = if inp.query.Length < 3 then Database.FullTextSearch.Exact else Database.FullTextSearch.PerformanceComplete
if inp.child_term.TermAccession = ""
then
Term.Term(credentials).getByNameAndChild_Name (inp.query,inp.child_term.Name,searchmode)
Expand Down Expand Up @@ -392,10 +398,10 @@ module V2 =
Term.TermQuery.getByAccession searchTerm.Term.TermAccession
// if term is a unit it should be contained inside the unit ontology, if not it is most likely free text input.
elif searchTerm.IsUnit then
Term.TermQuery.getByName(searchTerm.Term.Name, searchType=Helper.FullTextSearch.Exact, sourceOntologyName = Term.AnyOfOntology.Multiples ["uo"; "dpbo"])
Term.TermQuery.getByName(searchTerm.Term.Name, searchType=FullTextSearch.Exact, sourceOntologyName = Term.AnyOfOntology.Multiples ["uo"; "dpbo"])
// if none of the above apply we do a standard term search
else
Term.TermQuery.getByName(searchTerm.Term.Name, searchType=Helper.FullTextSearch.Exact)
Term.TermQuery.getByName(searchTerm.Term.Name, searchType=FullTextSearch.Exact)
)
let result =
Helper.Neo4j.runQueries(queries,credentials)
Expand Down
64 changes: 30 additions & 34 deletions src/Server/Database/Helper.fs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,58 +2,54 @@ module Database.Helper

open System
open Neo4j.Driver

module Regex =
[<Literal>]
let EscapePattern = @"(?<!\\)[\+\-\&\&\|\|\!\(\)\{\}\[\]\^""\~\*\?\:]{1}"
open Shared.Database

let defaultOutputWith<'a> (def:'a) (neo4jReturnVal:obj) =
if isNull neo4jReturnVal then def else neo4jReturnVal.As<'a>()

type Neo4JCredentials = {
User : string
Pw : string
BoltUrl : string
DatabaseName: string
} with
static member UserVarString = "DB_USER"
static member PwVarString = "DB_PASSWORD"
static member UriVarString = "DB_URL"
static member DBNameVarString = "DB_NAME"


/// <summary>
/// Cypher full text search is based on Apache lucene syntax. Which in turn uses certain special characters to apply advanced logic to the query.
/// This function escapes these special characters so they can be used in the query without being interpreted as special logic characters.
///
/// https://github.com/nfdi4plants/Swate/issues/491
/// </summary>
/// <param name="query"></param>
let escapeQuery (query: string) =
let eval = System.Text.RegularExpressions.MatchEvaluator(fun m -> "\\" + m.Value)
let regex = System.Text.RegularExpressions.Regex(Regex.EscapePattern)
regex.Replace(query, eval)
module Regex =
[<Literal>]
let EscapePattern = @"(?<!\\)[\+\-\&\&\|\|\!\(\)\{\}\[\]\^""\~\*\?\:]{1}"

/// <summary>
/// Cypher full text search is based on Apache lucene syntax. Which in turn uses certain special characters to apply advanced logic to the query.
/// This function escapes these special characters so they can be used in the query without being interpreted as special logic characters.
///
/// https://github.com/nfdi4plants/Swate/issues/491
/// </summary>
/// <param name="query"></param>
let escapeQuery (query: string) =
let eval = System.Text.RegularExpressions.MatchEvaluator(fun m -> "\\" + m.Value)
let regex = System.Text.RegularExpressions.Regex(EscapePattern)
regex.Replace(query, eval)

type FullTextSearch =
| Exact
| Complete
| PerformanceComplete
| Fuzzy
with
type FullTextSearch with
member this.ofQueryString(queryString:string) =
let escaped = escapeQuery queryString
let escaped = Regex.escapeQuery queryString
match this with
| Exact -> queryString
| Complete -> queryString + "*"
| PerformanceComplete ->
let singleWordArr = escaped.Split(" ", StringSplitOptions.RemoveEmptyEntries)
let singleWordArr = escaped.Split(" ", System.StringSplitOptions.RemoveEmptyEntries)
let count = singleWordArr.Length
singleWordArr
// add "+" to every word so the fulltext search must include the previous word, this highly improves search performance
|> Array.mapi (fun i str -> if i <> count-1 then "+" + str else str)
|> String.concat " "
| Fuzzy -> queryString.Replace(" ","~ ") + "~"
|> fun x -> escaped, x

type Neo4JCredentials = {
User : string
Pw : string
BoltUrl : string
DatabaseName: string
} with
static member UserVarString = "DB_USER"
static member PwVarString = "DB_PASSWORD"
static member UriVarString = "DB_URL"
static member DBNameVarString = "DB_NAME"

type Neo4j =

Expand Down
Loading

0 comments on commit d05b563

Please sign in to comment.