From 9becff22923592b2439ae48ef4fe99224e8aa15f Mon Sep 17 00:00:00 2001 From: Vedran Kasalica Date: Mon, 20 Nov 2023 13:11:36 +0100 Subject: [PATCH 1/8] Fix getJson for OEB benchmark --- .../nl/esciencecenter/models/OpenEBenchmark.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/nl/esciencecenter/models/OpenEBenchmark.java b/src/main/java/nl/esciencecenter/models/OpenEBenchmark.java index f4379cf..d90beff 100644 --- a/src/main/java/nl/esciencecenter/models/OpenEBenchmark.java +++ b/src/main/java/nl/esciencecenter/models/OpenEBenchmark.java @@ -1,15 +1,12 @@ package nl.esciencecenter.models; -import java.util.ArrayList; import java.util.List; -import org.json.JSONArray; -import org.json.JSONException; import org.json.JSONObject; import lombok.NonNull; import lombok.RequiredArgsConstructor; -import nl.esciencecenter.restape.ToolBenchmarkingAPIs; +import nl.uu.cs.ape.utils.APEUtils; @RequiredArgsConstructor /** @@ -29,7 +26,12 @@ public JSONObject getJson() { benchmarkJson.put("value", value); benchmarkJson.put("desirability_value", desirabilityValue); - benchmarkJson.put("workflow", workflow); + String workflowString = "["; + for (WorkflowStepBench step : workflow) { + workflowString += step.toString() + ","; + } + workflowString = APEUtils.removeLastChar(workflowString) + "]"; + benchmarkJson.put("workflow", workflowString); return benchmarkJson; } } From 3db07a25f7010773a70bd96712c0eac487ac20b5 Mon Sep 17 00:00:00 2001 From: Vedran Kasalica Date: Wed, 17 Jan 2024 20:41:07 +0100 Subject: [PATCH 2/8] Add Javadocs --- .../restape/ToolBenchmarkingAPIs.java | 19 ++++++++-- .../restape/RestApeControllerIT.java | 5 ++- .../restape/RestApeUtilsTest.java | 14 +++++++ .../restape/ToolBenchmarkingAPIsTest.java | 37 ++++++++++--------- 4 files changed, 52 insertions(+), 23 deletions(-) diff --git a/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java b/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java index d8bc792..cea4374 100644 --- a/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java +++ b/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java @@ -176,8 +176,17 @@ private static JSONObject computeOpenEBenchmarks(SolutionWorkflow workflow) { } /** - * Get the JSON annotations from bio.tools for the given tool IDs. + * Retrieve a JSON object corresponding to the tool from bio.tools for the given + * tool ID. * The method uses the bio.tools API to fetch the annotations. + * + * @param toolID - tool ID, not case sensitive. IDs are transformed into lower + * case as used in bio.tools, e.g., + * "comet", "blast", etc. + * @return JSONObject containing the metrics for the tool. + * @throws IOException In case the tool is not found in bio.tools. + * @throws JSONException In case the JSON object returned by bio.tools cannot be + * parsed. */ public static JSONObject fetchToolFromBioTools(String toolID) throws JSONException, IOException { JSONObject bioToolAnnotation; @@ -207,7 +216,7 @@ public static JSONObject fetchToolFromBioTools(String toolID) throws JSONExcepti * @param biotoolsExclusive - if true, only bio.tools URLs will be returned * @return JSONArray of JSONObjects, each containing the metrics for a tool * version. - * @throws IOException + * @throws IOException - In case the tool is not found in bio.tools. * @throws JSONException */ public static JSONArray fetchToolVersionsFromOEB(String toolID, boolean biotoolsExclusive) @@ -267,8 +276,10 @@ public static JSONObject getJSONfromURL(String url) { * Parse OpenEBench aggregated annotations to get the list of tool versions and * their URLs. * - * @param openEBenchAggregateAnnotation - * @return + * @param openEBenchAggregateAnnotation - JSONArray of JSONObjects, each of + * which contains a aggregated tool + * annotation. + * @return List of tool version URLs. */ static List getToolVersionsURLs(JSONArray openEBenchAggregateAnnotation) throws JSONException { diff --git a/src/test/java/nl/esciencecenter/restape/RestApeControllerIT.java b/src/test/java/nl/esciencecenter/restape/RestApeControllerIT.java index a4c40c9..eb36c52 100644 --- a/src/test/java/nl/esciencecenter/restape/RestApeControllerIT.java +++ b/src/test/java/nl/esciencecenter/restape/RestApeControllerIT.java @@ -15,8 +15,11 @@ class RestApeControllerIT { @Autowired private TestRestTemplate template; + /** + * Test if the server is running and returns the correct message. + */ @Test - void getGreetings() throws Exception { + void getGreetings() { ResponseEntity response = template.getForEntity("/", String.class); assertThat(response.getBody()).isEqualTo("Welcome to the RESTful APE API!"); } diff --git a/src/test/java/nl/esciencecenter/restape/RestApeUtilsTest.java b/src/test/java/nl/esciencecenter/restape/RestApeUtilsTest.java index 299a478..4eb5445 100644 --- a/src/test/java/nl/esciencecenter/restape/RestApeUtilsTest.java +++ b/src/test/java/nl/esciencecenter/restape/RestApeUtilsTest.java @@ -12,6 +12,9 @@ @SpringBootTest class RestApeUtilsTest { + /** + * Test whether the APE API can be setup with a URL path. + */ @Test void loadURLPath() { String urlPath = "https://raw.githubusercontent.com/Workflomics/domain-annotations/main/MassSpectometry/config.json"; @@ -20,6 +23,9 @@ void loadURLPath() { } + /** + * Test whether the APE API can be setup with a local path. + */ @Test void loadLocalPath() { @@ -31,6 +37,10 @@ void loadLocalPath() { } + /** + * Test whether the APE API retrieves data types within a domain configured in + * a URL path. + */ @Test void getTypesFromURLPath() { String urlPath = "https://raw.githubusercontent.com/Workflomics/domain-annotations/main/MassSpectometry/config.json"; @@ -39,6 +49,10 @@ void getTypesFromURLPath() { } + /** + * Test whether the APE API retrieves tools within a domain configured in + * a local path. + */ @Test void getToolsFromURLPath() { String urlPath = "https://raw.githubusercontent.com/Workflomics/domain-annotations/main/MassSpectometry/config.json"; diff --git a/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java b/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java index a775952..6ec7d6c 100644 --- a/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java +++ b/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java @@ -12,44 +12,45 @@ import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; +/** + * {@link ToolBenchmarkingAPIsTest} tests the methods in + * {@link ToolBenchmarkingAPIs}. + */ @SpringBootTest class ToolBenchmarkingAPIsTest { - @Test - void getToolsFromURLPath() { - String urlPath = "https://raw.githubusercontent.com/Workflomics/domain-annotations/main/MassSpectometry/config.json"; - - assertDoesNotThrow(() -> ApeAPI.getTools(urlPath)); - - } + private static final String TOOL_ID = "comet"; + /** + * Test whether tool annotations from the OpenEBench API can be retrieved. + */ @Test void fetchToolAggregateFromOEB() { - String toolID = "comet"; - JSONArray openEBenchAggregateAnnotation = null; try { - openEBenchAggregateAnnotation = ToolBenchmarkingAPIs.fetchToolAggregateFromOEB(toolID); + JSONArray openEBenchAggregateAnnotation = ToolBenchmarkingAPIs.fetchToolAggregateFromOEB(TOOL_ID); + assertFalse(openEBenchAggregateAnnotation == null || openEBenchAggregateAnnotation.isEmpty()); } catch (JSONException | IOException e) { e.printStackTrace(); fail(); } - assertFalse(openEBenchAggregateAnnotation == null || openEBenchAggregateAnnotation.isEmpty()); } + /** + * Test whether tool versions URLs can be computed from the retrieved OpenEBench + * API tool annotations. + */ @Test void getToolVersionsURLs() { - String toolID = "comet"; - List toolOEBVersionsURLs = null; try { - JSONArray openEBenchAggregateAnnotation = ToolBenchmarkingAPIs.fetchToolAggregateFromOEB(toolID); - toolOEBVersionsURLs = ToolBenchmarkingAPIs.getToolVersionsURLs(openEBenchAggregateAnnotation); + JSONArray openEBenchAggregateAnnotation = ToolBenchmarkingAPIs.fetchToolAggregateFromOEB(TOOL_ID); + List toolOEBVersionsURLs = ToolBenchmarkingAPIs.getToolVersionsURLs(openEBenchAggregateAnnotation); + + assertFalse(toolOEBVersionsURLs == null || toolOEBVersionsURLs.isEmpty() + || !toolOEBVersionsURLs.get(0).contains("https://openebench.bsc.es/")); } catch (JSONException | IOException e) { e.printStackTrace(); fail(); } - - assertFalse(toolOEBVersionsURLs == null || toolOEBVersionsURLs.isEmpty() - || !toolOEBVersionsURLs.get(0).contains("https://openebench.bsc.es/")); } } From 1c3efb75e7a5a3378d12c314b47cbdcc75053a4a Mon Sep 17 00:00:00 2001 From: Vedran Kasalica Date: Thu, 18 Jan 2024 15:34:31 +0100 Subject: [PATCH 3/8] Improve tests and fix retrieval from OEB --- .../esciencecenter/restape/RestApeUtils.java | 4 ++ .../restape/ToolBenchmarkingAPIs.java | 33 +++++++++----- .../restape/ToolBenchmarkingAPIsTest.java | 44 ++++++++++++++++--- 3 files changed, 65 insertions(+), 16 deletions(-) diff --git a/src/main/java/nl/esciencecenter/restape/RestApeUtils.java b/src/main/java/nl/esciencecenter/restape/RestApeUtils.java index d666839..b9ac365 100644 --- a/src/main/java/nl/esciencecenter/restape/RestApeUtils.java +++ b/src/main/java/nl/esciencecenter/restape/RestApeUtils.java @@ -5,13 +5,16 @@ import java.nio.file.Paths; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; +import java.util.function.Consumer; import org.json.JSONObject; import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; import lombok.AccessLevel; import lombok.Getter; +@Slf4j @NoArgsConstructor(access = AccessLevel.PRIVATE) public class RestApeUtils { @@ -126,4 +129,5 @@ public static JSONObject combineJSONObjects(JSONObject... jsonObjects) { } return combinedJson; } + } diff --git a/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java b/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java index cea4374..ef02d00 100644 --- a/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java +++ b/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java @@ -22,6 +22,7 @@ import lombok.AccessLevel; import lombok.NoArgsConstructor; + import nl.esciencecenter.models.BenchmarkBase; import nl.esciencecenter.models.BioToolsBenchmark; import nl.uu.cs.ape.solver.solutionStructure.SolutionWorkflow; @@ -217,7 +218,8 @@ public static JSONObject fetchToolFromBioTools(String toolID) throws JSONExcepti * @return JSONArray of JSONObjects, each containing the metrics for a tool * version. * @throws IOException - In case the tool is not found in bio.tools. - * @throws JSONException + * @throws JSONException - In case the JSON object returned by bio.tools or + * OpenEBench API cannot be parsed. */ public static JSONArray fetchToolVersionsFromOEB(String toolID, boolean biotoolsExclusive) throws JSONException, IOException { @@ -228,21 +230,28 @@ public static JSONArray fetchToolVersionsFromOEB(String toolID, boolean biotools if (biotoolsExclusive) { filterOutNonBioTools(toolOEBVersionsURLs); } - swapOEBCallTool2Metric(toolOEBVersionsURLs); + toolOEBVersionsURLs = swapOEBCallTool2Metric(toolOEBVersionsURLs); JSONArray openEBenchToolVersions = new JSONArray(); // retrieve the JSON metrics for each tool version toolOEBVersionsURLs.forEach(metricOEBenchURL -> { try { - openEBenchToolVersions.put(APEFiles.readPathToJSONObject(metricOEBenchURL)); - } catch (JSONException | IOException e) { + File file = APEFiles.readPathToFile(metricOEBenchURL); + String content = FileUtils.readFileToString(file, StandardCharsets.UTF_8); + openEBenchToolVersions.put(new JSONObject(content)); + } catch (JSONException e) { + log.error("Tool version metrics JSON provided by OEB could not be parsed."); + e.printStackTrace(); + } catch (IOException e) { + log.error("Tool version metrics file could not be created."); e.printStackTrace(); } }); log.debug("The list of tool versions was successfully fetched from OpenEBench."); return openEBenchToolVersions; + } /** @@ -309,15 +318,16 @@ static boolean filterOutNonBioTools(List openEBenchToolVersionURLs) { } /** - * Change the URL to retrieve detailed tool metrics rather than general tool - * information. + * Create a new list of URLs that references tool metrics rather than general + * tool + * information. In practice, the method replaces `/tool/` with `/metric/` in the + * URL. * * @param openEBenchToolVersionURLs - * @return + * @return List of URLs that reference tool metrics. */ static List swapOEBCallTool2Metric(List openEBenchToolVersionURLs) { - openEBenchToolVersionURLs.forEach(url -> url.replaceFirst("/tool/", "/metric/")); - return openEBenchToolVersionURLs; + return openEBenchToolVersionURLs.stream().map(url -> url.replaceFirst("/tool/", "/metrics/")).toList(); } /** @@ -329,8 +339,9 @@ static List swapOEBCallTool2Metric(List openEBenchToolVersionURL * * @param toolID * @return - * @throws JSONException - * @throws IOException + * @throws JSONException In case the JSON object returned by OpenEBench API + * cannot be parsed. + * @throws IOException In case a local file cannot be created. */ public static JSONArray fetchToolAggregateFromOEB(String toolID) throws JSONException, IOException { JSONArray openEBenchAnnotation; diff --git a/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java b/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java index 6ec7d6c..526d679 100644 --- a/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java +++ b/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java @@ -1,10 +1,12 @@ package nl.esciencecenter.restape; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import java.io.IOException; +import java.util.Arrays; import java.util.List; import org.json.JSONArray; @@ -21,11 +23,26 @@ class ToolBenchmarkingAPIsTest { private static final String TOOL_ID = "comet"; + @Test + void testSwapOEBCallTool2Metric() { + List input = Arrays.asList( + "http://example.com/api/tool/version1", + "http://example.com/api/test/version2"); + + List expected = Arrays.asList( + "http://example.com/api/metrics/version1", + "http://example.com/api/test/version2"); + + List result = ToolBenchmarkingAPIs.swapOEBCallTool2Metric(input); + + assertEquals(expected, result, "The URLs should be correctly transformed"); + } + /** * Test whether tool annotations from the OpenEBench API can be retrieved. */ @Test - void fetchToolAggregateFromOEB() { + void testFetchToolAggregateFromOEB() { try { JSONArray openEBenchAggregateAnnotation = ToolBenchmarkingAPIs.fetchToolAggregateFromOEB(TOOL_ID); assertFalse(openEBenchAggregateAnnotation == null || openEBenchAggregateAnnotation.isEmpty()); @@ -40,17 +57,34 @@ void fetchToolAggregateFromOEB() { * API tool annotations. */ @Test - void getToolVersionsURLs() { + void testGetToolVersionsURLs() { try { JSONArray openEBenchAggregateAnnotation = ToolBenchmarkingAPIs.fetchToolAggregateFromOEB(TOOL_ID); List toolOEBVersionsURLs = ToolBenchmarkingAPIs.getToolVersionsURLs(openEBenchAggregateAnnotation); - assertFalse(toolOEBVersionsURLs == null || toolOEBVersionsURLs.isEmpty() - || !toolOEBVersionsURLs.get(0).contains("https://openebench.bsc.es/")); + assertTrue(toolOEBVersionsURLs != null && !toolOEBVersionsURLs.isEmpty() + && toolOEBVersionsURLs.get(0).contains("https://openebench.bsc.es/")); + } catch (JSONException | IOException e) { + e.printStackTrace(); + fail(); + } + } + + @Test + void testFetchToolVersionsFromOEB() { + try { + JSONArray toolVersions = ToolBenchmarkingAPIs.fetchToolVersionsFromOEB(TOOL_ID, true); + + assertTrue(toolVersions != null && !toolVersions.isEmpty()); } catch (JSONException | IOException e) { e.printStackTrace(); fail(); } } + @Test + void test() { + String toolID = "comet"; + } + } From 6e597986acd4652dd75caa2e625988a33ca84926 Mon Sep 17 00:00:00 2001 From: Vedran Kasalica Date: Fri, 19 Jan 2024 18:30:38 +0100 Subject: [PATCH 4/8] Parse OpenEBench API to learn the license --- .../nl/esciencecenter/restape/ApeAPI.java | 2 - .../esciencecenter/restape/LicenseType.java | 34 +++ .../restape/ToolBenchmarkingAPIs.java | 197 ++++++++++++++---- .../restape/ToolBenchmarkingAPIsTest.java | 52 ++++- 4 files changed, 237 insertions(+), 48 deletions(-) create mode 100644 src/main/java/nl/esciencecenter/restape/LicenseType.java diff --git a/src/main/java/nl/esciencecenter/restape/ApeAPI.java b/src/main/java/nl/esciencecenter/restape/ApeAPI.java index 43771c8..b0ca945 100644 --- a/src/main/java/nl/esciencecenter/restape/ApeAPI.java +++ b/src/main/java/nl/esciencecenter/restape/ApeAPI.java @@ -28,8 +28,6 @@ import org.json.JSONObject; import org.semanticweb.owlapi.model.OWLOntologyCreationException; -import com.oracle.truffle.regex.tregex.util.json.JsonObject; - import guru.nidi.graphviz.attribute.Rank.RankDir; import guru.nidi.graphviz.engine.Format; import lombok.NoArgsConstructor; diff --git a/src/main/java/nl/esciencecenter/restape/LicenseType.java b/src/main/java/nl/esciencecenter/restape/LicenseType.java new file mode 100644 index 0000000..6d81b22 --- /dev/null +++ b/src/main/java/nl/esciencecenter/restape/LicenseType.java @@ -0,0 +1,34 @@ +package nl.esciencecenter.restape; + +/** + * Enumeration of various types of software licenses. + * This enum classifies licenses into categories based on their openness + * and OSI approval status. + */ +public enum LicenseType { + /** + * Represents an unknown license type. + * This value is used when the license type cannot be determined. + */ + Unknown, + + /** + * Represents a closed-source license. + * This indicates proprietary software where the source code is not publicly + * available. + */ + Closed, + + /** + * Represents an open-source license. + * This indicates software where the source code is publicly available, but it + * is not necessarily OSI-approved. + */ + Open, + + /** + * Represents a license that is approved by the Open Source Initiative (OSI). + * This indicates software that adheres to the OSI's definition of open source. + */ + OSI_Approved; +} \ No newline at end of file diff --git a/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java b/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java index ef02d00..0bc7d2f 100644 --- a/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java +++ b/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java @@ -6,20 +6,20 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; import org.apache.commons.io.FileUtils; +import org.checkerframework.checker.units.qual.t; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.oracle.truffle.regex.tregex.util.json.JsonObject; - import lombok.AccessLevel; import lombok.NoArgsConstructor; @@ -29,6 +29,23 @@ import nl.uu.cs.ape.solver.solutionStructure.SolutionsList; import nl.uu.cs.ape.utils.APEFiles; +/** + * The {@code ToolBenchmarkingAPIs} class provides methods to compute the tool + * metrics provided by OpenEBench API.
+ *
+ * + * Important: + * The OpenEBench API does not provide a well structured API at the moment. The + * current API interface is available at + * {@link https://openebench.bsc.es/monitor}. Therefore, some of the methods in + * this class are hardcoded to be able to utilize the current API interface.
+ *
+ * The new API version will in BioSchemas format, in JSON-LD, aligned with + * BioConda (see documentation at + * {@link https://openebench.bsc.es/bioschemas/}), however that API is not yet + * available. + * + */ @NoArgsConstructor(access = AccessLevel.PRIVATE) public class ToolBenchmarkingAPIs { @@ -142,34 +159,38 @@ private static JSONObject computeBiotoolsBenchmark(SolutionWorkflow workflow) { * @param workflow * @return */ - private static JSONObject computeOpenEBenchmarks(SolutionWorkflow workflow) { + static JSONObject computeOpenEBenchmarks(SolutionWorkflow workflow) { JSONObject benchmarkResult = new JSONObject(); - // for each tool in the workflow, get the openEBench annotations from bio.tool - // API - List openEBenchAnnotations = new ArrayList<>(); - - // TODO: uncoment - // workflow.getModuleNodes().forEach(toolNode -> { - // String toolID = toolNode.getUsedModule().getPredicateLabel(); - // try { - - // JSONArray openEBenchEntry = - // ToolBenchmarkingAPIs.fetchToolVersionsFromOEB(toolID); - // openEBenchEntry.put(ToolBenchmarkingAPIs.restAPEtoolID, - // toolNode.getUsedModule().getPredicateLabel()); - // openEBenchAnnotations.add(openEBenchEntry); - // } catch (JSONException | IOException e) { - // JSONObject openEBenchEntry = new JSONObject(); - // openEBenchEntry.put(ToolBenchmarkingAPIs.restAPEtoolID, - // toolNode.getUsedModule().getPredicateLabel()); - // openEBenchAnnotations.add(openEBenchEntry); - // e.printStackTrace(); - // } - // }); + /* + * For each tool in the workflow, get the OpenEBench annotations from OpenEBench + * API + */ + List openEBenchBiotoolsMetrics = new ArrayList<>(); + + workflow.getModuleNodes().forEach(toolNode -> { + String toolID = toolNode.getUsedModule().getPredicateLabel(); + try { + JSONObject openEBenchEntry = ToolBenchmarkingAPIs.fetchOEBMetricsForBiotoolsVersion(toolID); + openEBenchEntry.put(ToolBenchmarkingAPIs.restAPEtoolID, + toolNode.getUsedModule().getPredicateLabel()); + openEBenchBiotoolsMetrics.add(openEBenchEntry); + } catch (JSONException | IOException e) { + JSONObject openEBenchEntry = new JSONObject(); + openEBenchEntry.put(ToolBenchmarkingAPIs.restAPEtoolID, + toolNode.getUsedModule().getPredicateLabel()); + openEBenchBiotoolsMetrics.add(openEBenchEntry); + e.printStackTrace(); + } + }); JSONArray benchmarks = new JSONArray(); + BenchmarkBase license = new BenchmarkBase("License", "License information available", + "Number of tools which have a license specified.", "license", null); + // benchmarks.put(BioToolsBenchmark.countLicencedEntries(biotoolsAnnotations, + // licensedBenchmark).getJson()); + benchmarkResult.put("benchmarks", benchmarks); return benchmarkResult; @@ -209,37 +230,40 @@ public static JSONObject fetchToolFromBioTools(String toolID) throws JSONExcepti } /** - * Retrieve a list of JSON objects corresponding to tool versions from - * OpenEBench for the given tool ID. + * Retrieve a list of JSON objects containing the metrics for each tool version + * corresponding to the given tool ID from OpenEBench API. All version from + * BioConda will be included, as well as all copies of bio.tools entry, each + * representing a platform ("cdd", "cmd", "app", etc.). * - * @param toolID - tool ID, not case sensitive, (as used in - * bio.tools), e.g., "comet", "blast", etc. - * @param biotoolsExclusive - if true, only bio.tools URLs will be returned - * @return JSONArray of JSONObjects, each containing the metrics for a tool + * @param toolID - tool ID, not case sensitive, (as used in + * bio.tools), e.g., "comet", "blast", etc. + * @return List of JSONObjects, each containing the metrics for a tool * version. * @throws IOException - In case the tool is not found in bio.tools. * @throws JSONException - In case the JSON object returned by bio.tools or * OpenEBench API cannot be parsed. */ - public static JSONArray fetchToolVersionsFromOEB(String toolID, boolean biotoolsExclusive) + public static List fetchToolMetricsPerVersionFromOEB(String toolID) throws JSONException, IOException { toolID = toolID.toLowerCase(); JSONArray openEBenchAggregateAnnotation = fetchToolAggregateFromOEB(toolID); List toolOEBVersionsURLs = getToolVersionsURLs(openEBenchAggregateAnnotation); - if (biotoolsExclusive) { - filterOutNonBioTools(toolOEBVersionsURLs); - } + /* + * Correct the URLs to point to the metrics rather than general tool + * information. The OpenEBench API does not provide a more direct way to + * retrieve the metrics. + */ toolOEBVersionsURLs = swapOEBCallTool2Metric(toolOEBVersionsURLs); - JSONArray openEBenchToolVersions = new JSONArray(); + List openEBenchToolVersions = new ArrayList<>(); // retrieve the JSON metrics for each tool version toolOEBVersionsURLs.forEach(metricOEBenchURL -> { try { File file = APEFiles.readPathToFile(metricOEBenchURL); String content = FileUtils.readFileToString(file, StandardCharsets.UTF_8); - openEBenchToolVersions.put(new JSONObject(content)); + openEBenchToolVersions.add(new JSONObject(content)); } catch (JSONException e) { log.error("Tool version metrics JSON provided by OEB could not be parsed."); e.printStackTrace(); @@ -254,6 +278,55 @@ public static JSONArray fetchToolVersionsFromOEB(String toolID, boolean biotools } + /** + * Retrieve a JSON objects containing the metrics for the bio.tools entry for + * the given tool ID from OpenEBench API. + * + * @param toolID - tool ID, not case sensitive, (as used in + * bio.tools), e.g., "comet", "blast", etc. + * @return JSONObject, containing the metrics for the tool version. + * @throws IOException - In case the tool is not found in bio.tools. + * @throws JSONException - In case the JSON object returned by bio.tools or + * OpenEBench API cannot be parsed. + */ + public static JSONObject fetchOEBMetricsForBiotoolsVersion(String toolID) + throws JSONException, IOException { + toolID = toolID.toLowerCase(); + JSONArray openEBenchAggregateAnnotation = fetchToolAggregateFromOEB(toolID); + + String biotoolsVersionURL = getToolVersionsURLs(openEBenchAggregateAnnotation).stream() + .filter(url -> url.contains("biotools:")) + .findFirst().orElse(null); + + if (biotoolsVersionURL == null) { + return new JSONObject(); + } + + /* + * Correct the URL to point to the metrics rather than general tool + * information. The OpenEBench API does not provide a more direct way to + * retrieve the metrics. + */ + biotoolsVersionURL = swapOEBCallTool2Metric(biotoolsVersionURL); + + // retrieve the JSON metrics for each tool version + String metricsJson = ""; + try { + File metricsFile = APEFiles.readPathToFile(biotoolsVersionURL); + metricsJson = FileUtils.readFileToString(metricsFile, StandardCharsets.UTF_8); + } catch (JSONException e) { + log.error("Tool version metrics JSON provided by OEB could not be parsed."); + e.printStackTrace(); + } catch (IOException e) { + log.error("Tool version metrics file could not be created."); + e.printStackTrace(); + } + + log.debug("The list of tool versions was successfully fetched from OpenEBench."); + return new JSONObject(metricsJson); + + } + /** * Use the given URL to fetch the JSON object, using the OkHttp client. In case * of a failure, the method will return empty JSON object. @@ -327,7 +400,19 @@ static boolean filterOutNonBioTools(List openEBenchToolVersionURLs) { * @return List of URLs that reference tool metrics. */ static List swapOEBCallTool2Metric(List openEBenchToolVersionURLs) { - return openEBenchToolVersionURLs.stream().map(url -> url.replaceFirst("/tool/", "/metrics/")).toList(); + return openEBenchToolVersionURLs.stream().map(url -> swapOEBCallTool2Metric(url)).toList(); + } + + /** + * Create a new URL that references tool metrics rather than general tool + * information. In practice, the method replaces `/tool/` with `/metric/` in the + * URL. + * + * @param openEBenchToolVersionURL + * @return URL that references tool metrics. + */ + static String swapOEBCallTool2Metric(String openEBenchToolVersionURL) { + return openEBenchToolVersionURL.replaceFirst("/tool/", "/metrics/"); } /** @@ -335,7 +420,13 @@ static List swapOEBCallTool2Metric(List openEBenchToolVersionURL * contains general information about the tool version and a link (under "@id") * to * the detailed information. The same link, when `/tool/` is replaced with - * `/metrics/` can be used to retrieve the metrics for the tool version. + * `/metrics/` can be used to retrieve the metrics for the tool version.
+ *
+ * Important: + * The OpenEBench API does not provide a more direct way to + * retrieve the metrics, and thus, the URLs must be corrected to point to the + * metrics rather than general tool + * information. * * @param toolID * @return @@ -343,7 +434,7 @@ static List swapOEBCallTool2Metric(List openEBenchToolVersionURL * cannot be parsed. * @throws IOException In case a local file cannot be created. */ - public static JSONArray fetchToolAggregateFromOEB(String toolID) throws JSONException, IOException { + static JSONArray fetchToolAggregateFromOEB(String toolID) throws JSONException, IOException { JSONArray openEBenchAnnotation; String urlToAggregateOEB = "https://openebench.bsc.es/monitor/rest/aggregate?id=" + toolID; @@ -354,4 +445,30 @@ public static JSONArray fetchToolAggregateFromOEB(String toolID) throws JSONExce return openEBenchAnnotation; } + /** + * Parse the JSON object returned by OpenEBench API describing the tool metrics + * and return whether the tool has an OSI approved license. + * + * @param tootMetrics - JSON object returned by OpenEBench API describing the + * tool metrics. + * @return true if the tool has an OSI approved license, false otherwise. + */ + public static LicenseType isOSIFromOEBMetrics(JSONObject tootMetrics) throws JSONException { + JSONObject licenseJson; + try { + licenseJson = tootMetrics.getJSONObject("project").getJSONObject("license"); + } catch (JSONException e) { + return LicenseType.Unknown; + } + + boolean isOSI = licenseJson.getBoolean("osi"); + if (isOSI) { + return LicenseType.OSI_Approved; + } else if (licenseJson.getBoolean("open_source")) { + return LicenseType.Open; + } else { + return LicenseType.Closed; + } + } + } diff --git a/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java b/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java index 526d679..e113ffe 100644 --- a/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java +++ b/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java @@ -11,6 +11,7 @@ import org.json.JSONArray; import org.json.JSONException; +import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; @@ -70,12 +71,42 @@ void testGetToolVersionsURLs() { } } + /** + * Test for isOSIFromOEBMetrics method. + * It verifies that the method correctly identifies and returns the OSI status + * from the provided JSON object representing tool metrics. + */ @Test - void testFetchToolVersionsFromOEB() { - try { - JSONArray toolVersions = ToolBenchmarkingAPIs.fetchToolVersionsFromOEB(TOOL_ID, true); + void testIsOSIFromOEBMetrics() { + // Constructing the JSON structure inline for the tool metrics + JSONObject mockToolMetrics = new JSONObject(""" + { + "project": { + "license": { + "osi": true + } + } + } + """); + + // Testing the isOSIFromOEBMetrics method and asserting that the OSI status is + // true + assertTrue(ToolBenchmarkingAPIs.isOSIFromOEBMetrics(mockToolMetrics) == LicenseType.OSI_Approved, + "The method should return true for OSI license"); + } - assertTrue(toolVersions != null && !toolVersions.isEmpty()); + @Test + void testFetchToolMetricsPerVersionFromOEB() { + try { + List allToolMetrics = ToolBenchmarkingAPIs.fetchToolMetricsPerVersionFromOEB(TOOL_ID); + assertTrue(allToolMetrics != null && allToolMetrics.size() > 0); + allToolMetrics.forEach(tootMetrics -> { + try { + tootMetrics.get("project"); + } catch (JSONException e) { + fail("An exception occurred while checking for 'osi license' in 'project'. The JSON object could not be parsed correctly."); + } + }); } catch (JSONException | IOException e) { e.printStackTrace(); fail(); @@ -83,8 +114,17 @@ void testFetchToolVersionsFromOEB() { } @Test - void test() { - String toolID = "comet"; + void testFetchOEBMetricsForBiotoolsVersion() { + try { + JSONObject toolMetrics = ToolBenchmarkingAPIs.fetchOEBMetricsForBiotoolsVersion("shelx"); + ToolBenchmarkingAPIs.isOSIFromOEBMetrics(toolMetrics); + } catch (JSONException e) { + fail("An exception occurred while checking for 'osi license' in 'project'. The JSON object could not be parsed correctly."); + } catch (ClassCastException e) { + fail("The 'project' field is not a JSONObject"); + } catch (IOException e) { + fail("Failed to fetch metrics for bio.tools version for tool + " + TOOL_ID); + } } } From 6aded12fd4dfba7195675248574c4f963c507efb Mon Sep 17 00:00:00 2001 From: Vedran Kasalica Date: Mon, 22 Jan 2024 17:21:12 +0100 Subject: [PATCH 5/8] Include OpenEBench license benchmarks --- .../esciencecenter/models/BenchmarkBase.java | 33 ++++++++++ .../models/BioToolsBenchmark.java | 36 +++-------- .../esciencecenter/models/OpenEBenchmark.java | 61 +++++++++++++++++++ .../nl/esciencecenter/restape/ApeAPI.java | 4 -- .../restape/ToolBenchmarkingAPIs.java | 9 +-- src/main/resources/application.properties | 2 +- 6 files changed, 110 insertions(+), 35 deletions(-) diff --git a/src/main/java/nl/esciencecenter/models/BenchmarkBase.java b/src/main/java/nl/esciencecenter/models/BenchmarkBase.java index edbd96e..5a8d787 100644 --- a/src/main/java/nl/esciencecenter/models/BenchmarkBase.java +++ b/src/main/java/nl/esciencecenter/models/BenchmarkBase.java @@ -28,4 +28,37 @@ public JSONObject getTitleJson() { benchmarkJson.put("benchmark_description", benchmarkDescription); return benchmarkJson; } + + static String ratioString(int count, int length) { + return count + "/" + length; + } + + /** + * Calculate the desirability value for the given workflow, assuming that it + * increases linearly with the number of tools that satisfy the benchmark. + * + * @param count number of tools that satisfy the benchmark + * @param workflowLength length of the workflow + * @return Desirability value for the given workflow. + */ + static double normalDesirabilityDistribution(int count, int workflowLength) { + return 1.0 * count / workflowLength; + } + + /** + * Calculate the desirability value for the given workflow, assuming that it is + * desired that all tools satisfy the benchmark. Small desirability values are + * assigned to workflows where only a subset of tools satisfy the benchmark. + * + * @param count number of tools that satisfy the benchmark + * @param workflowLength length of the workflow + * @return Desirability value for the given workflow. + */ + static double strictDesirabilityDistribution(int count, int workflowLength) { + if (count == workflowLength) { + return 1; + } else { + return 1.0 * count / workflowLength / 10; + } + } } \ No newline at end of file diff --git a/src/main/java/nl/esciencecenter/models/BioToolsBenchmark.java b/src/main/java/nl/esciencecenter/models/BioToolsBenchmark.java index 8e6f564..69b0e8c 100644 --- a/src/main/java/nl/esciencecenter/models/BioToolsBenchmark.java +++ b/src/main/java/nl/esciencecenter/models/BioToolsBenchmark.java @@ -25,26 +25,6 @@ public class BioToolsBenchmark { private double desirabilityValue; private List workflow; - private static boolean emptyToolAnnotation(JSONObject toolAnnot) { - return !toolAnnot.has("biotoolsID"); - } - - private static String ratioString(int count, int length) { - return count + "/" + length; - } - - private static double normalDistribution(int count, int workflowLength) { - return 1.0 * count / workflowLength; - } - - private static double strictDistribution(int count, int workflowLength) { - if (count == workflowLength) { - return 1; - } else { - return 1.0 * count / workflowLength / 10; - } - } - public static BioToolsBenchmark countEntries(List biotoolsAnnotations, BenchmarkBase benchmarkTitle) { BioToolsBenchmark benchmark = new BioToolsBenchmark(benchmarkTitle); int workflowLength = biotoolsAnnotations.size(); @@ -52,8 +32,8 @@ public static BioToolsBenchmark countEntries(List biotoolsAnnotation benchmark.workflow = countEntries(biotoolsAnnotations); int count = (int) benchmark.workflow.stream().filter(tool -> tool.getDesirabilityValue() > 0).count(); - benchmark.desirabilityValue = strictDistribution(count, workflowLength); - benchmark.value = ratioString(count, workflowLength); + benchmark.desirabilityValue = BenchmarkBase.strictDesirabilityDistribution(count, workflowLength); + benchmark.value = BenchmarkBase.ratioString(count, workflowLength); return benchmark; } @@ -66,8 +46,8 @@ public static BioToolsBenchmark countLicencedEntries(List biotoolsAn benchmark.workflow = countField(biotoolsAnnotations, benchmarkTitle.getExpectedField()); int count = (int) benchmark.workflow.stream().filter(tool -> tool.getDesirabilityValue() > 0).count(); - benchmark.desirabilityValue = strictDistribution(count, workflowLength); - benchmark.value = ratioString(count, workflowLength); + benchmark.desirabilityValue = BenchmarkBase.strictDesirabilityDistribution(count, workflowLength); + benchmark.value = BenchmarkBase.ratioString(count, workflowLength); return benchmark; } @@ -81,9 +61,9 @@ public static BioToolsBenchmark countOSEntries(List biotoolsAnnotati benchmarkTitle.getExpectedValue()); int count = (int) benchmark.workflow.stream().filter(tool -> tool.getDesirabilityValue() > 0).count(); - benchmark.desirabilityValue = normalDistribution(count, workflowLength); + benchmark.desirabilityValue = BenchmarkBase.normalDesirabilityDistribution(count, workflowLength); - benchmark.value = ratioString(count, workflowLength); + benchmark.value = BenchmarkBase.ratioString(count, workflowLength); return benchmark; } @@ -183,6 +163,10 @@ private static int countExistanceOfFields(List biotoolsAnnotations, return (int) count; } + private static boolean emptyToolAnnotation(JSONObject toolAnnot) { + return !toolAnnot.has("biotoolsID"); + } + /** * Check whether the given value is in the set of given values. */ diff --git a/src/main/java/nl/esciencecenter/models/OpenEBenchmark.java b/src/main/java/nl/esciencecenter/models/OpenEBenchmark.java index d90beff..4a1e726 100644 --- a/src/main/java/nl/esciencecenter/models/OpenEBenchmark.java +++ b/src/main/java/nl/esciencecenter/models/OpenEBenchmark.java @@ -1,11 +1,14 @@ package nl.esciencecenter.models; +import java.util.ArrayList; import java.util.List; import org.json.JSONObject; import lombok.NonNull; import lombok.RequiredArgsConstructor; +import nl.esciencecenter.restape.LicenseType; +import nl.esciencecenter.restape.ToolBenchmarkingAPIs; import nl.uu.cs.ape.utils.APEUtils; @RequiredArgsConstructor @@ -21,6 +24,63 @@ public class OpenEBenchmark { private double desirabilityValue; private List workflow; + public static OpenEBenchmark countLicenceOpenness(List openEBenchAnnotations, + BenchmarkBase benchmarkTitle) { + OpenEBenchmark benchmark = new OpenEBenchmark(benchmarkTitle); + int workflowLength = openEBenchAnnotations.size(); + + benchmark.workflow = evaluateLicense(openEBenchAnnotations); + int count = (int) benchmark.workflow.stream().filter(tool -> tool.getDesirabilityValue() > 0).count(); + + benchmark.desirabilityValue = BenchmarkBase.strictDesirabilityDistribution(count, workflowLength); + benchmark.value = BenchmarkBase.ratioString(count, workflowLength); + + return benchmark; + } + + /** + * Count the number of tools which have the given field name in the bio.tools + * annotation JSON. + * + * @param biotoolsAnnotations + * @param fieldName + * @return + */ + private static List evaluateLicense(List biotoolsAnnotations) { + List biotoolsEntries = new ArrayList<>(); + + biotoolsAnnotations.stream().forEach(toolAnnot -> { + WorkflowStepBench biotoolsEntryBenchmark = new WorkflowStepBench(); + biotoolsEntryBenchmark.setDescription(toolAnnot.getString(ToolBenchmarkingAPIs.restAPEtoolID)); + LicenseType license = ToolBenchmarkingAPIs.isOSIFromOEBMetrics(toolAnnot); + // set case for each license type + switch (license) { + case Unknown: + biotoolsEntryBenchmark.setDesirabilityValue(0); + biotoolsEntryBenchmark.setValue("unknown"); + break; + case Closed: + biotoolsEntryBenchmark.setDesirabilityValue(0.1); + biotoolsEntryBenchmark.setValue("closed"); + break; + case Open: + biotoolsEntryBenchmark.setDesirabilityValue(0.8); + biotoolsEntryBenchmark.setValue("open"); + break; + case OSI_Approved: + biotoolsEntryBenchmark.setDesirabilityValue(1); + biotoolsEntryBenchmark.setValue("osi"); + break; + default: + throw new IllegalArgumentException(); + } + + biotoolsEntries.add(biotoolsEntryBenchmark); + }); + + return biotoolsEntries; + } + public JSONObject getJson() { JSONObject benchmarkJson = this.benchmarkTitle.getTitleJson(); @@ -34,4 +94,5 @@ public JSONObject getJson() { benchmarkJson.put("workflow", workflowString); return benchmarkJson; } + } diff --git a/src/main/java/nl/esciencecenter/restape/ApeAPI.java b/src/main/java/nl/esciencecenter/restape/ApeAPI.java index b0ca945..f89edf9 100644 --- a/src/main/java/nl/esciencecenter/restape/ApeAPI.java +++ b/src/main/java/nl/esciencecenter/restape/ApeAPI.java @@ -9,7 +9,6 @@ import java.util.List; import nl.esciencecenter.models.BenchmarkBase; -import nl.esciencecenter.models.BioToolsBenchmark; import nl.uu.cs.ape.APE; import nl.uu.cs.ape.configuration.APECoreConfig; import nl.uu.cs.ape.configuration.APERunConfig; @@ -22,13 +21,10 @@ import nl.uu.cs.ape.solver.solutionStructure.SolutionsList; import nl.uu.cs.ape.utils.APEUtils; -import org.apache.commons.io.FileUtils; import org.json.JSONArray; -import org.json.JSONException; import org.json.JSONObject; import org.semanticweb.owlapi.model.OWLOntologyCreationException; -import guru.nidi.graphviz.attribute.Rank.RankDir; import guru.nidi.graphviz.engine.Format; import lombok.NoArgsConstructor; import lombok.AccessLevel; diff --git a/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java b/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java index 0bc7d2f..12d220d 100644 --- a/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java +++ b/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java @@ -25,6 +25,7 @@ import nl.esciencecenter.models.BenchmarkBase; import nl.esciencecenter.models.BioToolsBenchmark; +import nl.esciencecenter.models.OpenEBenchmark; import nl.uu.cs.ape.solver.solutionStructure.SolutionWorkflow; import nl.uu.cs.ape.solver.solutionStructure.SolutionsList; import nl.uu.cs.ape.utils.APEFiles; @@ -65,7 +66,8 @@ static boolean computeBenchmarks(SolutionsList candidateSolutions, String runID) candidateSolutions.getParallelStream().forEach(workflow -> { JSONObject workflowBenchmarks = RestApeUtils.combineJSONObjects( computeWorkflowSpecificFields(workflow, runID), - computeBiotoolsBenchmark(workflow)); + computeBiotoolsBenchmark(workflow), + computeOpenEBenchmarks(workflow)); String titleBenchmark = workflow.getFileName() + ".json"; Path solFolder = candidateSolutions.getRunConfiguration().getSolutionDirPath2CWL(); File script = solFolder.resolve(titleBenchmark).toFile(); @@ -186,10 +188,9 @@ static JSONObject computeOpenEBenchmarks(SolutionWorkflow workflow) { JSONArray benchmarks = new JSONArray(); - BenchmarkBase license = new BenchmarkBase("License", "License information available", + BenchmarkBase licenseBenchmark = new BenchmarkBase("License", "License information available", "Number of tools which have a license specified.", "license", null); - // benchmarks.put(BioToolsBenchmark.countLicencedEntries(biotoolsAnnotations, - // licensedBenchmark).getJson()); + benchmarks.put(OpenEBenchmark.countLicenceOpenness(openEBenchBiotoolsMetrics, licenseBenchmark).getJson()); benchmarkResult.put("benchmarks", benchmarks); diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index c8cec29..9970a1e 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -5,4 +5,4 @@ spring.datasource.password=restape spring.jpa.database-platform=org.hibernate.dialect.H2Dialect #enabling the H2 console spring.h2.console.enabled=true -server.port=4444 \ No newline at end of file +server.port=8080 \ No newline at end of file From abeffd6f191e018258122604cde7fd6d79e0b0c3 Mon Sep 17 00:00:00 2001 From: Vedran Kasalica Date: Mon, 22 Jan 2024 19:23:49 +0100 Subject: [PATCH 6/8] Finish licensing and add citation benchmarks --- .../models/benchmarks/OpenEBenchmark.java | 113 ++++++++++++++++-- .../restape/ToolBenchmarkingAPIs.java | 60 +++++----- src/main/resources/application.properties | 2 +- .../restape/RestApeControllerTest.java | 1 + 4 files changed, 137 insertions(+), 39 deletions(-) diff --git a/src/main/java/nl/esciencecenter/models/benchmarks/OpenEBenchmark.java b/src/main/java/nl/esciencecenter/models/benchmarks/OpenEBenchmark.java index b43ab72..1f31309 100644 --- a/src/main/java/nl/esciencecenter/models/benchmarks/OpenEBenchmark.java +++ b/src/main/java/nl/esciencecenter/models/benchmarks/OpenEBenchmark.java @@ -1,15 +1,17 @@ package nl.esciencecenter.models.benchmarks; import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import lombok.NonNull; import lombok.RequiredArgsConstructor; import nl.esciencecenter.restape.LicenseType; import nl.esciencecenter.restape.ToolBenchmarkingAPIs; -import nl.uu.cs.ape.utils.APEUtils; @RequiredArgsConstructor /** @@ -24,12 +26,12 @@ public class OpenEBenchmark { private double desirabilityValue; private List workflow; - public static OpenEBenchmark countLicenceOpenness(List openEBenchAnnotations, + public static OpenEBenchmark countLicenceOpenness(List openEBenchBiotoolsMetrics, BenchmarkBase benchmarkTitle) { OpenEBenchmark benchmark = new OpenEBenchmark(benchmarkTitle); - int workflowLength = openEBenchAnnotations.size(); + int workflowLength = openEBenchBiotoolsMetrics.size(); - benchmark.workflow = evaluateLicense(openEBenchAnnotations); + benchmark.workflow = evaluateLicenseBenchmark(openEBenchBiotoolsMetrics); int count = (int) benchmark.workflow.stream().filter(tool -> tool.getDesirabilityValue() > 0).count(); benchmark.desirabilityValue = BenchmarkBase.strictDesirabilityDistribution(count, workflowLength); @@ -46,52 +48,141 @@ public static OpenEBenchmark countLicenceOpenness(List openEBenchAnn * @param fieldName * @return */ - private static List evaluateLicense(List biotoolsAnnotations) { + private static List evaluateLicenseBenchmark(List biotoolsAnnotations) { List biotoolsEntries = new ArrayList<>(); biotoolsAnnotations.stream().forEach(toolAnnot -> { WorkflowStepBench biotoolsEntryBenchmark = new WorkflowStepBench(); - biotoolsEntryBenchmark.setDescription(toolAnnot.getString(ToolBenchmarkingAPIs.restAPEtoolID)); LicenseType license = ToolBenchmarkingAPIs.isOSIFromOEBMetrics(toolAnnot); // set case for each license type switch (license) { case Unknown: biotoolsEntryBenchmark.setDesirabilityValue(0); biotoolsEntryBenchmark.setValue("unknown"); + biotoolsEntryBenchmark.setDescription("Unknown"); break; case Closed: biotoolsEntryBenchmark.setDesirabilityValue(0.1); biotoolsEntryBenchmark.setValue("closed"); + biotoolsEntryBenchmark.setDescription("Closed"); break; case Open: biotoolsEntryBenchmark.setDesirabilityValue(0.8); biotoolsEntryBenchmark.setValue("open"); + biotoolsEntryBenchmark.setDescription("Open"); break; case OSI_Approved: biotoolsEntryBenchmark.setDesirabilityValue(1); biotoolsEntryBenchmark.setValue("osi"); + biotoolsEntryBenchmark.setDescription("OSI approved"); break; default: throw new IllegalArgumentException(); } - biotoolsEntries.add(biotoolsEntryBenchmark); }); return biotoolsEntries; } + public static OpenEBenchmark countCitationsBenchmark(List openEBenchBiotoolsMetrics, + BenchmarkBase benchmarkTitle) { + OpenEBenchmark benchmark = new OpenEBenchmark(benchmarkTitle); + + benchmark.workflow = countCitationPerTool(openEBenchBiotoolsMetrics); + List counts = new ArrayList<>(); + benchmark.workflow.forEach(tool -> counts.add(Integer.parseInt(tool.getValue()))); + int median = findMedian(counts); + + benchmark.value = median + ""; + benchmark.desirabilityValue = computeCitationDesirability(median); + + return benchmark; + } + + /** + * Calculates the median of the given List of Integers. + * + * @param counts the List of Integer values + * @return the median value as a double + * @throws IllegalArgumentException if the input list is empty + */ + private static int findMedian(List counts) { + if (counts == null || counts.isEmpty()) { + throw new IllegalArgumentException("List of counts cannot be null or empty"); + } + + Collections.sort(counts); + + int size = counts.size(); + if (size % 2 == 1) { + // If the size is odd, return the middle element + return counts.get(size / 2); + } else { + // If the size is even, return the average of the two middle elements + double leftMiddle = counts.get(size / 2 - 1); + double rightMiddle = counts.get(size / 2); + return (int) (leftMiddle + rightMiddle) / 2; + } + } + + private static List countCitationPerTool(List openEBenchBiotoolsMetrics) { + List biotoolsEntries = new ArrayList<>(); + openEBenchBiotoolsMetrics.stream().forEach(toolAnnot -> { + WorkflowStepBench biotoolsEntryBenchmark = new WorkflowStepBench(); + int count = 0; + try { + JSONArray publications = toolAnnot.getJSONObject("project").getJSONArray("publications"); + for (int i = 0; i < publications.length(); i++) { + JSONObject publicationData = publications.getJSONObject(i); + count += publicationData.getJSONArray("entries").getJSONObject(0).getInt("cit_count"); + } + // set case for each license type + biotoolsEntryBenchmark.setDesirabilityValue(computeCitationDesirability(count)); + biotoolsEntryBenchmark.setValue(String.valueOf(count)); + biotoolsEntryBenchmark.setDescription(String.valueOf(count)); + biotoolsEntries.add(biotoolsEntryBenchmark); + } catch (JSONException e) { + e.printStackTrace(); + // set case for each license type + biotoolsEntryBenchmark.setDesirabilityValue(0); + biotoolsEntryBenchmark.setValue("Unknown"); + biotoolsEntryBenchmark.setDescription("Unknown"); + biotoolsEntries.add(biotoolsEntryBenchmark); + } + }); + + return biotoolsEntries; + } + + /* + * Citation desirability is computed according to a predefined set of rules. + * + */ + private static @NonNull double computeCitationDesirability(int count) { + if (count == 0) { + return 0; + } else if (count < 10) { + return 0.25; + } else if (count < 30) { + return 0.5; + } else if (count < 50) { + return 0.75; + } else { + return 1; + } + } + public JSONObject getJson() { JSONObject benchmarkJson = this.benchmarkTitle.getTitleJson(); benchmarkJson.put("value", value); benchmarkJson.put("desirability_value", desirabilityValue); - String workflowString = "["; + JSONArray workflowJson = new JSONArray(); for (WorkflowStepBench step : workflow) { - workflowString += step.toString() + ","; + workflowJson.put(step.toJSON()); } - workflowString = APEUtils.removeLastChar(workflowString) + "]"; - benchmarkJson.put("workflow", workflowString); + benchmarkJson.put("steps", workflowJson); return benchmarkJson; } diff --git a/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java b/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java index 51322a9..e34c7c5 100644 --- a/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java +++ b/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java @@ -63,10 +63,13 @@ public class ToolBenchmarkingAPIs { */ static boolean computeBenchmarks(SolutionsList candidateSolutions, String runID) { candidateSolutions.getParallelStream().forEach(workflow -> { - JSONObject workflowBenchmarks = RestApeUtils.combineJSONObjects( - computeWorkflowSpecificFields(workflow, runID), - computeBiotoolsBenchmark(workflow), - computeOpenEBenchmarks(workflow)); + JSONArray biotoolsbenchmark = computeBiotoolsBenchmark(workflow); + JSONArray openEBenchmarks = computeOpenEBenchmarks(workflow); + openEBenchmarks.forEach(biotoolsbenchmark::put); + + JSONObject workflowBenchmarks = computeWorkflowSpecificFields(workflow, runID); + workflowBenchmarks.put("benchmarks", biotoolsbenchmark); + String titleBenchmark = workflow.getFileName() + ".json"; Path solFolder = candidateSolutions.getRunConfiguration().getSolutionDirPath2CWL(); File script = solFolder.resolve(titleBenchmark).toFile(); @@ -98,8 +101,7 @@ private static JSONObject computeWorkflowSpecificFields(SolutionWorkflow workflo * @param workflow * @return */ - private static JSONObject computeBiotoolsBenchmark(SolutionWorkflow workflow) { - JSONObject benchmarkResult = new JSONObject(); + private static JSONArray computeBiotoolsBenchmark(SolutionWorkflow workflow) { // for each tool in the workflow, get the biotools annotations from bio.tool API List biotoolsAnnotations = new ArrayList<>(); @@ -121,10 +123,6 @@ private static JSONObject computeBiotoolsBenchmark(SolutionWorkflow workflow) { JSONArray benchmarks = new JSONArray(); - BenchmarkBase licensedBenchmark = new BenchmarkBase("Licensed", "Tools with a license", - "Number of tools which have a license specified.", "license", null); - benchmarks.put(BioToolsBenchmark.countLicencedEntries(biotoolsAnnotations, licensedBenchmark).getJson()); - BenchmarkBase linuxBenchmark = new BenchmarkBase("Linux", "Linux (OS) supported tools", "Number of tools which support Linux OS.", "operatingSystem", "Linux"); benchmarks.put(BioToolsBenchmark.countOSEntries(biotoolsAnnotations, linuxBenchmark).getJson()); @@ -137,19 +135,27 @@ private static JSONObject computeBiotoolsBenchmark(SolutionWorkflow workflow) { "Number of tools which support Windows OS.", "operatingSystem", "Windows"); benchmarks.put(BioToolsBenchmark.countOSEntries(biotoolsAnnotations, windowsBenchmark).getJson()); - BenchmarkBase bioToolBenchmark = new BenchmarkBase("In bio.tools", "Available in bio.tools", - "Number of tools annotated in bio.tools.", null, null); - benchmarks.put(BioToolsBenchmark.countEntries(biotoolsAnnotations, bioToolBenchmark).getJson()); - - // BenchmarkBase openEBenchmark = new BenchmarkBase("In OpenEBench", "Available - // in OpenEBench", - // "Number of tools tracked in OpenEBench.", null, null); - // benchmarks.put(BioToolsBenchmark.countEntries(biotoolsAnnotations, - // bioToolBenchmark).getJson()); - - benchmarkResult.put("benchmarks", benchmarks); + /* + * BenchmarkBase licensedBenchmark = new BenchmarkBase("Licensed", + * "Tools with a license", + * "Number of tools which have a license specified.", "license", null); + * benchmarks.put(BioToolsBenchmark.countLicencedEntries(biotoolsAnnotations, + * licensedBenchmark).getJson()); + * + * BenchmarkBase bioToolBenchmark = new BenchmarkBase("In bio.tools", + * "Available in bio.tools", + * "Number of tools annotated in bio.tools.", null, null); + * benchmarks.put(BioToolsBenchmark.countEntries(biotoolsAnnotations, + * bioToolBenchmark).getJson()); + * + * BenchmarkBase openEBenchmark = new BenchmarkBase("In OpenEBench", + * "Available in OpenEBench", + * "Number of tools tracked in OpenEBench.", null, null); + * benchmarks.put(BioToolsBenchmark.countEntries(biotoolsAnnotations, + * bioToolBenchmark).getJson()); + */ - return benchmarkResult; + return benchmarks; } @@ -160,9 +166,7 @@ private static JSONObject computeBiotoolsBenchmark(SolutionWorkflow workflow) { * @param workflow * @return */ - static JSONObject computeOpenEBenchmarks(SolutionWorkflow workflow) { - JSONObject benchmarkResult = new JSONObject(); - + static JSONArray computeOpenEBenchmarks(SolutionWorkflow workflow) { /* * For each tool in the workflow, get the OpenEBench annotations from OpenEBench * API @@ -191,9 +195,11 @@ static JSONObject computeOpenEBenchmarks(SolutionWorkflow workflow) { "Number of tools which have a license specified.", "license", null); benchmarks.put(OpenEBenchmark.countLicenceOpenness(openEBenchBiotoolsMetrics, licenseBenchmark).getJson()); - benchmarkResult.put("benchmarks", benchmarks); + BenchmarkBase citationsBenchmark = new BenchmarkBase("Citations", "Citations annotated per tool", + "Number of citations per tool.", "citation", null); + benchmarks.put(OpenEBenchmark.countCitationsBenchmark(openEBenchBiotoolsMetrics, citationsBenchmark).getJson()); - return benchmarkResult; + return benchmarks; } diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 9970a1e..c8cec29 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -5,4 +5,4 @@ spring.datasource.password=restape spring.jpa.database-platform=org.hibernate.dialect.H2Dialect #enabling the H2 console spring.h2.console.enabled=true -server.port=8080 \ No newline at end of file +server.port=4444 \ No newline at end of file diff --git a/src/test/java/nl/esciencecenter/restape/RestApeControllerTest.java b/src/test/java/nl/esciencecenter/restape/RestApeControllerTest.java index 946b399..8679cf1 100644 --- a/src/test/java/nl/esciencecenter/restape/RestApeControllerTest.java +++ b/src/test/java/nl/esciencecenter/restape/RestApeControllerTest.java @@ -162,6 +162,7 @@ void runSynthesisAndBenchmarkPass() throws IOException, OWLOntologyCreationExcep String content = FileUtils.readFileToString(APEFiles.readPathToFile(configPath), StandardCharsets.UTF_8); JSONObject jsonObject = new JSONObject(content); + jsonObject.put("solutions", "1"); JSONArray result = ApeAPI.runSynthesis(jsonObject, true); assertFalse(result.isEmpty(), "The encoding should be SAT."); } From c78ed92cd4b4e2f22b339c25a80aaee857d292ee Mon Sep 17 00:00:00 2001 From: Vedran Kasalica Date: Thu, 25 Jan 2024 10:26:47 +0100 Subject: [PATCH 7/8] Update naming and small code improvements --- .../restape/ToolBenchmarkingAPIs.java | 48 +++++-------------- .../restape/ToolBenchmarkingAPIsTest.java | 6 +-- 2 files changed, 15 insertions(+), 39 deletions(-) diff --git a/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java b/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java index e34c7c5..b388ffe 100644 --- a/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java +++ b/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java @@ -6,14 +6,12 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.List; -import java.util.Optional; import okhttp3.OkHttpClient; import okhttp3.Request; import okhttp3.Response; import org.apache.commons.io.FileUtils; -import org.checkerframework.checker.units.qual.t; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; @@ -108,16 +106,14 @@ private static JSONArray computeBiotoolsBenchmark(SolutionWorkflow workflow) { workflow.getModuleNodes().forEach(toolNode -> { String toolID = toolNode.getUsedModule().getPredicateLabel(); + JSONObject biotoolsEntry = new JSONObject(); try { - - JSONObject biotoolsEntry = ToolBenchmarkingAPIs.fetchToolFromBioTools(toolID); - biotoolsEntry.put(ToolBenchmarkingAPIs.restAPEtoolID, toolNode.getUsedModule().getPredicateLabel()); - biotoolsAnnotations.add(biotoolsEntry); + biotoolsEntry = ToolBenchmarkingAPIs.fetchToolFromBioTools(toolID); } catch (JSONException | IOException e) { - JSONObject biotoolsEntry = new JSONObject(); + e.printStackTrace(); + } finally { biotoolsEntry.put(ToolBenchmarkingAPIs.restAPEtoolID, toolNode.getUsedModule().getPredicateLabel()); biotoolsAnnotations.add(biotoolsEntry); - e.printStackTrace(); } }); @@ -135,26 +131,6 @@ private static JSONArray computeBiotoolsBenchmark(SolutionWorkflow workflow) { "Number of tools which support Windows OS.", "operatingSystem", "Windows"); benchmarks.put(BioToolsBenchmark.countOSEntries(biotoolsAnnotations, windowsBenchmark).getJson()); - /* - * BenchmarkBase licensedBenchmark = new BenchmarkBase("Licensed", - * "Tools with a license", - * "Number of tools which have a license specified.", "license", null); - * benchmarks.put(BioToolsBenchmark.countLicencedEntries(biotoolsAnnotations, - * licensedBenchmark).getJson()); - * - * BenchmarkBase bioToolBenchmark = new BenchmarkBase("In bio.tools", - * "Available in bio.tools", - * "Number of tools annotated in bio.tools.", null, null); - * benchmarks.put(BioToolsBenchmark.countEntries(biotoolsAnnotations, - * bioToolBenchmark).getJson()); - * - * BenchmarkBase openEBenchmark = new BenchmarkBase("In OpenEBench", - * "Available in OpenEBench", - * "Number of tools tracked in OpenEBench.", null, null); - * benchmarks.put(BioToolsBenchmark.countEntries(biotoolsAnnotations, - * bioToolBenchmark).getJson()); - */ - return benchmarks; } @@ -260,7 +236,7 @@ public static List fetchToolMetricsPerVersionFromOEB(String toolID) * information. The OpenEBench API does not provide a more direct way to * retrieve the metrics. */ - toolOEBVersionsURLs = swapOEBCallTool2Metric(toolOEBVersionsURLs); + toolOEBVersionsURLs = replaceTool2MetricInOEBCall(toolOEBVersionsURLs); List openEBenchToolVersions = new ArrayList<>(); @@ -313,7 +289,7 @@ public static JSONObject fetchOEBMetricsForBiotoolsVersion(String toolID) * information. The OpenEBench API does not provide a more direct way to * retrieve the metrics. */ - biotoolsVersionURL = swapOEBCallTool2Metric(biotoolsVersionURL); + biotoolsVersionURL = replaceTool2MetricInOEBCall(biotoolsVersionURL); // retrieve the JSON metrics for each tool version String metricsJson = ""; @@ -405,8 +381,8 @@ static boolean filterOutNonBioTools(List openEBenchToolVersionURLs) { * @param openEBenchToolVersionURLs * @return List of URLs that reference tool metrics. */ - static List swapOEBCallTool2Metric(List openEBenchToolVersionURLs) { - return openEBenchToolVersionURLs.stream().map(url -> swapOEBCallTool2Metric(url)).toList(); + static List replaceTool2MetricInOEBCall(List openEBenchToolVersionURLs) { + return openEBenchToolVersionURLs.stream().map(url -> replaceTool2MetricInOEBCall(url)).toList(); } /** @@ -417,7 +393,7 @@ static List swapOEBCallTool2Metric(List openEBenchToolVersionURL * @param openEBenchToolVersionURL * @return URL that references tool metrics. */ - static String swapOEBCallTool2Metric(String openEBenchToolVersionURL) { + static String replaceTool2MetricInOEBCall(String openEBenchToolVersionURL) { return openEBenchToolVersionURL.replaceFirst("/tool/", "/metrics/"); } @@ -455,14 +431,14 @@ static JSONArray fetchToolAggregateFromOEB(String toolID) throws JSONException, * Parse the JSON object returned by OpenEBench API describing the tool metrics * and return whether the tool has an OSI approved license. * - * @param tootMetrics - JSON object returned by OpenEBench API describing the + * @param toolMetrics - JSON object returned by OpenEBench API describing the * tool metrics. * @return true if the tool has an OSI approved license, false otherwise. */ - public static LicenseType isOSIFromOEBMetrics(JSONObject tootMetrics) throws JSONException { + public static LicenseType isOSIFromOEBMetrics(JSONObject toolMetrics) throws JSONException { JSONObject licenseJson; try { - licenseJson = tootMetrics.getJSONObject("project").getJSONObject("license"); + licenseJson = toolMetrics.getJSONObject("project").getJSONObject("license"); } catch (JSONException e) { return LicenseType.Unknown; } diff --git a/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java b/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java index e113ffe..c68c370 100644 --- a/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java +++ b/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java @@ -34,7 +34,7 @@ void testSwapOEBCallTool2Metric() { "http://example.com/api/metrics/version1", "http://example.com/api/test/version2"); - List result = ToolBenchmarkingAPIs.swapOEBCallTool2Metric(input); + List result = ToolBenchmarkingAPIs.replaceTool2MetricInOEBCall(input); assertEquals(expected, result, "The URLs should be correctly transformed"); } @@ -100,9 +100,9 @@ void testFetchToolMetricsPerVersionFromOEB() { try { List allToolMetrics = ToolBenchmarkingAPIs.fetchToolMetricsPerVersionFromOEB(TOOL_ID); assertTrue(allToolMetrics != null && allToolMetrics.size() > 0); - allToolMetrics.forEach(tootMetrics -> { + allToolMetrics.forEach(toolMetrics -> { try { - tootMetrics.get("project"); + toolMetrics.get("project"); } catch (JSONException e) { fail("An exception occurred while checking for 'osi license' in 'project'. The JSON object could not be parsed correctly."); } From 745afebf0c6a2741ab86d6ce807fdbac3114fb20 Mon Sep 17 00:00:00 2001 From: Vedran Kasalica Date: Thu, 25 Jan 2024 14:55:26 +0100 Subject: [PATCH 8/8] Improve benchmarking code (using OEB API) and simplify tests --- .../models/benchmarks/OpenEBenchmark.java | 30 +++++- .../restape/ToolBenchmarkingAPIs.java | 96 +++++++++---------- .../restape/RestApeControllerTest.java | 2 + .../restape/ToolBenchmarkingAPIsTest.java | 6 +- 4 files changed, 77 insertions(+), 57 deletions(-) diff --git a/src/main/java/nl/esciencecenter/models/benchmarks/OpenEBenchmark.java b/src/main/java/nl/esciencecenter/models/benchmarks/OpenEBenchmark.java index 1f31309..46e18df 100644 --- a/src/main/java/nl/esciencecenter/models/benchmarks/OpenEBenchmark.java +++ b/src/main/java/nl/esciencecenter/models/benchmarks/OpenEBenchmark.java @@ -26,7 +26,7 @@ public class OpenEBenchmark { private double desirabilityValue; private List workflow; - public static OpenEBenchmark countLicenceOpenness(List openEBenchBiotoolsMetrics, + public static OpenEBenchmark countLicenseOpenness(List openEBenchBiotoolsMetrics, BenchmarkBase benchmarkTitle) { OpenEBenchmark benchmark = new OpenEBenchmark(benchmarkTitle); int workflowLength = openEBenchBiotoolsMetrics.size(); @@ -53,7 +53,7 @@ private static List evaluateLicenseBenchmark(List biotoolsAnnotations.stream().forEach(toolAnnot -> { WorkflowStepBench biotoolsEntryBenchmark = new WorkflowStepBench(); - LicenseType license = ToolBenchmarkingAPIs.isOSIFromOEBMetrics(toolAnnot); + LicenseType license = isOSIFromOEBMetrics(toolAnnot); // set case for each license type switch (license) { case Unknown: @@ -186,4 +186,30 @@ public JSONObject getJson() { return benchmarkJson; } + /** + * Parse the JSON object returned by OpenEBench API describing the tool metrics + * and return whether the tool has an OSI approved license. + * + * @param toolMetrics - JSON object returned by OpenEBench API describing the + * tool metrics. + * @return true if the tool has an OSI approved license, false otherwise. + */ + public static LicenseType isOSIFromOEBMetrics(JSONObject toolMetrics) throws JSONException { + JSONObject licenseJson; + try { + licenseJson = toolMetrics.getJSONObject("project").getJSONObject("license"); + } catch (JSONException e) { + return LicenseType.Unknown; + } + + boolean isOSI = licenseJson.getBoolean("osi"); + if (isOSI) { + return LicenseType.OSI_Approved; + } else if (licenseJson.getBoolean("open_source")) { + return LicenseType.Open; + } else { + return LicenseType.Closed; + } + } + } diff --git a/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java b/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java index b388ffe..3ba88f8 100644 --- a/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java +++ b/src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java @@ -49,7 +49,7 @@ public class ToolBenchmarkingAPIs { public static final String restAPEtoolID = "restAPEtoolID"; private static final Logger log = LoggerFactory.getLogger(ToolBenchmarkingAPIs.class); - public static final OkHttpClient client = new OkHttpClient(); + private static final OkHttpClient client = new OkHttpClient(); /** * Compute the benchmarks for the workflows. @@ -61,12 +61,12 @@ public class ToolBenchmarkingAPIs { */ static boolean computeBenchmarks(SolutionsList candidateSolutions, String runID) { candidateSolutions.getParallelStream().forEach(workflow -> { - JSONArray biotoolsbenchmark = computeBiotoolsBenchmark(workflow); - JSONArray openEBenchmarks = computeOpenEBenchmarks(workflow); - openEBenchmarks.forEach(biotoolsbenchmark::put); - + /* + * Compute the benchmarks for the workflow and save them in a JSON file. + */ + JSONArray combinedBenchmarks = computeWorkflowBenchmarks(workflow); JSONObject workflowBenchmarks = computeWorkflowSpecificFields(workflow, runID); - workflowBenchmarks.put("benchmarks", biotoolsbenchmark); + workflowBenchmarks.put("benchmarks", combinedBenchmarks); String titleBenchmark = workflow.getFileName() + ".json"; Path solFolder = candidateSolutions.getRunConfiguration().getSolutionDirPath2CWL(); @@ -82,6 +82,23 @@ static boolean computeBenchmarks(SolutionsList candidateSolutions, String runID) return true; } + /** + * Compute the benchmarks (based on bio.tools and OpenEBench APIs) for the workflows and return it in JSON format. + * + * @param workflow - workflow for which the benchmarks should be computed. + * @return JSONArray containing the benchmarks for the workflow. + */ + private static JSONArray computeWorkflowBenchmarks(SolutionWorkflow workflow) { + JSONArray biotoolsBenchmark = computeBiotoolsBenchmark(workflow); + JSONArray openEBenchmarks = computeOpenEBenchmarks(workflow); + + for (int i = 0; i < openEBenchmarks.length(); i++) { + biotoolsBenchmark.put(openEBenchmarks.get(i)); + } + + return biotoolsBenchmark; + } + private static JSONObject computeWorkflowSpecificFields(SolutionWorkflow workflow, String runID) { JSONObject benchmarkResult = new JSONObject(); // Set workflow specific fields @@ -93,11 +110,11 @@ private static JSONObject computeWorkflowSpecificFields(SolutionWorkflow workflo } /** - * Compute the bio.tools benchmarks for the workflows and return it in JSON + * Compute the bio.tools benchmarks for the workflows by using bio.tools API to get information about each tool in the workflow. The result (the benchmarks) is returned in JSON * format. * - * @param workflow - * @return + * @param workflow - workflow for which the benchmarks should be computed. + * @return JSONArray containing the benchmarks for the workflow. */ private static JSONArray computeBiotoolsBenchmark(SolutionWorkflow workflow) { @@ -136,13 +153,13 @@ private static JSONArray computeBiotoolsBenchmark(SolutionWorkflow workflow) { } /** - * Compute the OpenEBench benchmarks for the workflows and return it in JSON + * Compute the OpenEBench benchmarks for the workflows by using OpenEBench API to get information about each tool in the workflow. The result (the benchmarks) is returned in JSON * format. * - * @param workflow - * @return + * @param workflow - workflow for which the benchmarks should be computed. + * @return */ - static JSONArray computeOpenEBenchmarks(SolutionWorkflow workflow) { + private static JSONArray computeOpenEBenchmarks(SolutionWorkflow workflow) { /* * For each tool in the workflow, get the OpenEBench annotations from OpenEBench * API @@ -151,17 +168,16 @@ static JSONArray computeOpenEBenchmarks(SolutionWorkflow workflow) { workflow.getModuleNodes().forEach(toolNode -> { String toolID = toolNode.getUsedModule().getPredicateLabel(); + JSONObject openEBenchEntry = new JSONObject(); try { - JSONObject openEBenchEntry = ToolBenchmarkingAPIs.fetchOEBMetricsForBiotoolsVersion(toolID); - openEBenchEntry.put(ToolBenchmarkingAPIs.restAPEtoolID, - toolNode.getUsedModule().getPredicateLabel()); - openEBenchBiotoolsMetrics.add(openEBenchEntry); - } catch (JSONException | IOException e) { - JSONObject openEBenchEntry = new JSONObject(); - openEBenchEntry.put(ToolBenchmarkingAPIs.restAPEtoolID, - toolNode.getUsedModule().getPredicateLabel()); - openEBenchBiotoolsMetrics.add(openEBenchEntry); + openEBenchEntry = ToolBenchmarkingAPIs.fetchOEBMetricsForBiotoolsVersion(toolID); + } catch (JSONException e) { e.printStackTrace(); + } catch (IOException e) { + log.error("Tool {} not found in OpenEBench. It will not be benchmarked.", toolID); + } finally { + openEBenchEntry.put(ToolBenchmarkingAPIs.restAPEtoolID, toolNode.getUsedModule().getPredicateLabel()); + openEBenchBiotoolsMetrics.add(openEBenchEntry); } }); @@ -169,7 +185,7 @@ static JSONArray computeOpenEBenchmarks(SolutionWorkflow workflow) { BenchmarkBase licenseBenchmark = new BenchmarkBase("License", "License information available", "Number of tools which have a license specified.", "license", null); - benchmarks.put(OpenEBenchmark.countLicenceOpenness(openEBenchBiotoolsMetrics, licenseBenchmark).getJson()); + benchmarks.put(OpenEBenchmark.countLicenseOpenness(openEBenchBiotoolsMetrics, licenseBenchmark).getJson()); BenchmarkBase citationsBenchmark = new BenchmarkBase("Citations", "Citations annotated per tool", "Number of citations per tool.", "citation", null); @@ -192,7 +208,7 @@ static JSONArray computeOpenEBenchmarks(SolutionWorkflow workflow) { * @throws JSONException In case the JSON object returned by bio.tools cannot be * parsed. */ - public static JSONObject fetchToolFromBioTools(String toolID) throws JSONException, IOException { + static JSONObject fetchToolFromBioTools(String toolID) throws JSONException, IOException { JSONObject bioToolAnnotation; toolID = toolID.toLowerCase(); String urlToBioTools = "https://bio.tools/api/" + toolID + @@ -221,11 +237,11 @@ public static JSONObject fetchToolFromBioTools(String toolID) throws JSONExcepti * bio.tools), e.g., "comet", "blast", etc. * @return List of JSONObjects, each containing the metrics for a tool * version. - * @throws IOException - In case the tool is not found in bio.tools. + * @throws IOException - In case the tool is not found in OpenEBench. * @throws JSONException - In case the JSON object returned by bio.tools or * OpenEBench API cannot be parsed. */ - public static List fetchToolMetricsPerVersionFromOEB(String toolID) + static List fetchToolMetricsPerVersionFromOEB(String toolID) throws JSONException, IOException { toolID = toolID.toLowerCase(); JSONArray openEBenchAggregateAnnotation = fetchToolAggregateFromOEB(toolID); @@ -271,7 +287,7 @@ public static List fetchToolMetricsPerVersionFromOEB(String toolID) * @throws JSONException - In case the JSON object returned by bio.tools or * OpenEBench API cannot be parsed. */ - public static JSONObject fetchOEBMetricsForBiotoolsVersion(String toolID) + static JSONObject fetchOEBMetricsForBiotoolsVersion(String toolID) throws JSONException, IOException { toolID = toolID.toLowerCase(); JSONArray openEBenchAggregateAnnotation = fetchToolAggregateFromOEB(toolID); @@ -427,30 +443,4 @@ static JSONArray fetchToolAggregateFromOEB(String toolID) throws JSONException, return openEBenchAnnotation; } - /** - * Parse the JSON object returned by OpenEBench API describing the tool metrics - * and return whether the tool has an OSI approved license. - * - * @param toolMetrics - JSON object returned by OpenEBench API describing the - * tool metrics. - * @return true if the tool has an OSI approved license, false otherwise. - */ - public static LicenseType isOSIFromOEBMetrics(JSONObject toolMetrics) throws JSONException { - JSONObject licenseJson; - try { - licenseJson = toolMetrics.getJSONObject("project").getJSONObject("license"); - } catch (JSONException e) { - return LicenseType.Unknown; - } - - boolean isOSI = licenseJson.getBoolean("osi"); - if (isOSI) { - return LicenseType.OSI_Approved; - } else if (licenseJson.getBoolean("open_source")) { - return LicenseType.Open; - } else { - return LicenseType.Closed; - } - } - } diff --git a/src/test/java/nl/esciencecenter/restape/RestApeControllerTest.java b/src/test/java/nl/esciencecenter/restape/RestApeControllerTest.java index 8679cf1..ea9be24 100644 --- a/src/test/java/nl/esciencecenter/restape/RestApeControllerTest.java +++ b/src/test/java/nl/esciencecenter/restape/RestApeControllerTest.java @@ -128,6 +128,7 @@ void runSynthesisFail() throws IOException, OWLOntologyCreationException { String content = FileUtils.readFileToString(APEFiles.readPathToFile(configPath), StandardCharsets.UTF_8); JSONObject jsonObject = new JSONObject(content); + jsonObject.put("solutions", "1"); JSONArray result = ApeAPI.runSynthesis(jsonObject, false); assertTrue(result.isEmpty(), "The encoding should be UNSAT."); } @@ -145,6 +146,7 @@ void runSynthesisPass() throws IOException, OWLOntologyCreationException { String content = FileUtils.readFileToString(APEFiles.readPathToFile(configPath), StandardCharsets.UTF_8); JSONObject jsonObject = new JSONObject(content); + jsonObject.put("solutions", "1"); JSONArray result = ApeAPI.runSynthesis(jsonObject, false); assertFalse(result.isEmpty(), "The encoding should be SAT."); } diff --git a/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java b/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java index c68c370..e967e13 100644 --- a/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java +++ b/src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java @@ -15,6 +15,8 @@ import org.junit.jupiter.api.Test; import org.springframework.boot.test.context.SpringBootTest; +import nl.esciencecenter.models.benchmarks.OpenEBenchmark; + /** * {@link ToolBenchmarkingAPIsTest} tests the methods in * {@link ToolBenchmarkingAPIs}. @@ -91,7 +93,7 @@ void testIsOSIFromOEBMetrics() { // Testing the isOSIFromOEBMetrics method and asserting that the OSI status is // true - assertTrue(ToolBenchmarkingAPIs.isOSIFromOEBMetrics(mockToolMetrics) == LicenseType.OSI_Approved, + assertTrue(OpenEBenchmark.isOSIFromOEBMetrics(mockToolMetrics) == LicenseType.OSI_Approved, "The method should return true for OSI license"); } @@ -117,7 +119,7 @@ void testFetchToolMetricsPerVersionFromOEB() { void testFetchOEBMetricsForBiotoolsVersion() { try { JSONObject toolMetrics = ToolBenchmarkingAPIs.fetchOEBMetricsForBiotoolsVersion("shelx"); - ToolBenchmarkingAPIs.isOSIFromOEBMetrics(toolMetrics); + OpenEBenchmark.isOSIFromOEBMetrics(toolMetrics); } catch (JSONException e) { fail("An exception occurred while checking for 'osi license' in 'project'. The JSON object could not be parsed correctly."); } catch (ClassCastException e) {