Skip to content

Commit

Permalink
Improve benchmarking code (using OEB API) and simplify tests
Browse files Browse the repository at this point in the history
  • Loading branch information
@vedran-kasalica
vedran-kasalica committed Jan 25, 2024
1 parent c78ed92 commit 745afeb
Show file tree
Hide file tree
Showing 4 changed files with 77 additions and 57 deletions.
30 changes: 28 additions & 2 deletions src/main/java/nl/esciencecenter/models/benchmarks/OpenEBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public class OpenEBenchmark {
private double desirabilityValue;
private List<WorkflowStepBench> workflow;

public static OpenEBenchmark countLicenceOpenness(List<JSONObject> openEBenchBiotoolsMetrics,
public static OpenEBenchmark countLicenseOpenness(List<JSONObject> openEBenchBiotoolsMetrics,
BenchmarkBase benchmarkTitle) {
OpenEBenchmark benchmark = new OpenEBenchmark(benchmarkTitle);
int workflowLength = openEBenchBiotoolsMetrics.size();
Expand All @@ -53,7 +53,7 @@ private static List<WorkflowStepBench> evaluateLicenseBenchmark(List<JSONObject>

biotoolsAnnotations.stream().forEach(toolAnnot -> {
WorkflowStepBench biotoolsEntryBenchmark = new WorkflowStepBench();
LicenseType license = ToolBenchmarkingAPIs.isOSIFromOEBMetrics(toolAnnot);
LicenseType license = isOSIFromOEBMetrics(toolAnnot);
// set case for each license type
switch (license) {
case Unknown:
Expand Down Expand Up @@ -186,4 +186,30 @@ public JSONObject getJson() {
return benchmarkJson;
}

/**
* Parse the JSON object returned by OpenEBench API describing the tool metrics
* and return whether the tool has an OSI approved license.
*
* @param toolMetrics - JSON object returned by OpenEBench API describing the
* tool metrics.
* @return true if the tool has an OSI approved license, false otherwise.
*/
public static LicenseType isOSIFromOEBMetrics(JSONObject toolMetrics) throws JSONException {
JSONObject licenseJson;
try {
licenseJson = toolMetrics.getJSONObject("project").getJSONObject("license");
} catch (JSONException e) {
return LicenseType.Unknown;
}

boolean isOSI = licenseJson.getBoolean("osi");
if (isOSI) {
return LicenseType.OSI_Approved;
} else if (licenseJson.getBoolean("open_source")) {
return LicenseType.Open;
} else {
return LicenseType.Closed;
}
}

}
96 changes: 43 additions & 53 deletions src/main/java/nl/esciencecenter/restape/ToolBenchmarkingAPIs.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public class ToolBenchmarkingAPIs {

public static final String restAPEtoolID = "restAPEtoolID";
private static final Logger log = LoggerFactory.getLogger(ToolBenchmarkingAPIs.class);
public static final OkHttpClient client = new OkHttpClient();
private static final OkHttpClient client = new OkHttpClient();

/**
* Compute the benchmarks for the workflows.
Expand All @@ -61,12 +61,12 @@ public class ToolBenchmarkingAPIs {
*/
static boolean computeBenchmarks(SolutionsList candidateSolutions, String runID) {
candidateSolutions.getParallelStream().forEach(workflow -> {
JSONArray biotoolsbenchmark = computeBiotoolsBenchmark(workflow);
JSONArray openEBenchmarks = computeOpenEBenchmarks(workflow);
openEBenchmarks.forEach(biotoolsbenchmark::put);

/*
* Compute the benchmarks for the workflow and save them in a JSON file.
*/
JSONArray combinedBenchmarks = computeWorkflowBenchmarks(workflow);
JSONObject workflowBenchmarks = computeWorkflowSpecificFields(workflow, runID);
workflowBenchmarks.put("benchmarks", biotoolsbenchmark);
workflowBenchmarks.put("benchmarks", combinedBenchmarks);

String titleBenchmark = workflow.getFileName() + ".json";
Path solFolder = candidateSolutions.getRunConfiguration().getSolutionDirPath2CWL();
Expand All @@ -82,6 +82,23 @@ static boolean computeBenchmarks(SolutionsList candidateSolutions, String runID)
return true;
}

/**
* Compute the benchmarks (based on bio.tools and OpenEBench APIs) for the workflows and return it in JSON format.
*
* @param workflow - workflow for which the benchmarks should be computed.
* @return JSONArray containing the benchmarks for the workflow.
*/
private static JSONArray computeWorkflowBenchmarks(SolutionWorkflow workflow) {
JSONArray biotoolsBenchmark = computeBiotoolsBenchmark(workflow);
JSONArray openEBenchmarks = computeOpenEBenchmarks(workflow);

for (int i = 0; i < openEBenchmarks.length(); i++) {
biotoolsBenchmark.put(openEBenchmarks.get(i));
}

return biotoolsBenchmark;
}

private static JSONObject computeWorkflowSpecificFields(SolutionWorkflow workflow, String runID) {
JSONObject benchmarkResult = new JSONObject();
// Set workflow specific fields
Expand All @@ -93,11 +110,11 @@ private static JSONObject computeWorkflowSpecificFields(SolutionWorkflow workflo
}

/**
* Compute the bio.tools benchmarks for the workflows and return it in JSON
* Compute the bio.tools benchmarks for the workflows by using bio.tools API to get information about each tool in the workflow. The result (the benchmarks) is returned in JSON
* format.
*
* @param workflow
* @return
* @param workflow - workflow for which the benchmarks should be computed.
* @return JSONArray containing the benchmarks for the workflow.
*/
private static JSONArray computeBiotoolsBenchmark(SolutionWorkflow workflow) {

Expand Down Expand Up @@ -136,13 +153,13 @@ private static JSONArray computeBiotoolsBenchmark(SolutionWorkflow workflow) {
}

/**
* Compute the OpenEBench benchmarks for the workflows and return it in JSON
* Compute the OpenEBench benchmarks for the workflows by using OpenEBench API to get information about each tool in the workflow. The result (the benchmarks) is returned in JSON
* format.
*
* @param workflow
* @return
* @param workflow - workflow for which the benchmarks should be computed.
* @return
*/
static JSONArray computeOpenEBenchmarks(SolutionWorkflow workflow) {
private static JSONArray computeOpenEBenchmarks(SolutionWorkflow workflow) {
/*
* For each tool in the workflow, get the OpenEBench annotations from OpenEBench
* API
Expand All @@ -151,25 +168,24 @@ static JSONArray computeOpenEBenchmarks(SolutionWorkflow workflow) {

workflow.getModuleNodes().forEach(toolNode -> {
String toolID = toolNode.getUsedModule().getPredicateLabel();
JSONObject openEBenchEntry = new JSONObject();
try {
JSONObject openEBenchEntry = ToolBenchmarkingAPIs.fetchOEBMetricsForBiotoolsVersion(toolID);
openEBenchEntry.put(ToolBenchmarkingAPIs.restAPEtoolID,
toolNode.getUsedModule().getPredicateLabel());
openEBenchBiotoolsMetrics.add(openEBenchEntry);
} catch (JSONException | IOException e) {
JSONObject openEBenchEntry = new JSONObject();
openEBenchEntry.put(ToolBenchmarkingAPIs.restAPEtoolID,
toolNode.getUsedModule().getPredicateLabel());
openEBenchBiotoolsMetrics.add(openEBenchEntry);
openEBenchEntry = ToolBenchmarkingAPIs.fetchOEBMetricsForBiotoolsVersion(toolID);
} catch (JSONException e) {
e.printStackTrace();
} catch (IOException e) {
log.error("Tool {} not found in OpenEBench. It will not be benchmarked.", toolID);
} finally {
openEBenchEntry.put(ToolBenchmarkingAPIs.restAPEtoolID, toolNode.getUsedModule().getPredicateLabel());
openEBenchBiotoolsMetrics.add(openEBenchEntry);
}
});

JSONArray benchmarks = new JSONArray();

BenchmarkBase licenseBenchmark = new BenchmarkBase("License", "License information available",
"Number of tools which have a license specified.", "license", null);
benchmarks.put(OpenEBenchmark.countLicenceOpenness(openEBenchBiotoolsMetrics, licenseBenchmark).getJson());
benchmarks.put(OpenEBenchmark.countLicenseOpenness(openEBenchBiotoolsMetrics, licenseBenchmark).getJson());

BenchmarkBase citationsBenchmark = new BenchmarkBase("Citations", "Citations annotated per tool",
"Number of citations per tool.", "citation", null);
Expand All @@ -192,7 +208,7 @@ static JSONArray computeOpenEBenchmarks(SolutionWorkflow workflow) {
* @throws JSONException In case the JSON object returned by bio.tools cannot be
* parsed.
*/
public static JSONObject fetchToolFromBioTools(String toolID) throws JSONException, IOException {
static JSONObject fetchToolFromBioTools(String toolID) throws JSONException, IOException {
JSONObject bioToolAnnotation;
toolID = toolID.toLowerCase();
String urlToBioTools = "https://bio.tools/api/" + toolID +
Expand Down Expand Up @@ -221,11 +237,11 @@ public static JSONObject fetchToolFromBioTools(String toolID) throws JSONExcepti
* bio.tools), e.g., "comet", "blast", etc.
* @return List of JSONObjects, each containing the metrics for a tool
* version.
* @throws IOException - In case the tool is not found in bio.tools.
* @throws IOException - In case the tool is not found in OpenEBench.
* @throws JSONException - In case the JSON object returned by bio.tools or
* OpenEBench API cannot be parsed.
*/
public static List<JSONObject> fetchToolMetricsPerVersionFromOEB(String toolID)
static List<JSONObject> fetchToolMetricsPerVersionFromOEB(String toolID)
throws JSONException, IOException {
toolID = toolID.toLowerCase();
JSONArray openEBenchAggregateAnnotation = fetchToolAggregateFromOEB(toolID);
Expand Down Expand Up @@ -271,7 +287,7 @@ public static List<JSONObject> fetchToolMetricsPerVersionFromOEB(String toolID)
* @throws JSONException - In case the JSON object returned by bio.tools or
* OpenEBench API cannot be parsed.
*/
public static JSONObject fetchOEBMetricsForBiotoolsVersion(String toolID)
static JSONObject fetchOEBMetricsForBiotoolsVersion(String toolID)
throws JSONException, IOException {
toolID = toolID.toLowerCase();
JSONArray openEBenchAggregateAnnotation = fetchToolAggregateFromOEB(toolID);
Expand Down Expand Up @@ -427,30 +443,4 @@ static JSONArray fetchToolAggregateFromOEB(String toolID) throws JSONException,
return openEBenchAnnotation;
}

/**
* Parse the JSON object returned by OpenEBench API describing the tool metrics
* and return whether the tool has an OSI approved license.
*
* @param toolMetrics - JSON object returned by OpenEBench API describing the
* tool metrics.
* @return true if the tool has an OSI approved license, false otherwise.
*/
public static LicenseType isOSIFromOEBMetrics(JSONObject toolMetrics) throws JSONException {
JSONObject licenseJson;
try {
licenseJson = toolMetrics.getJSONObject("project").getJSONObject("license");
} catch (JSONException e) {
return LicenseType.Unknown;
}

boolean isOSI = licenseJson.getBoolean("osi");
if (isOSI) {
return LicenseType.OSI_Approved;
} else if (licenseJson.getBoolean("open_source")) {
return LicenseType.Open;
} else {
return LicenseType.Closed;
}
}

}
2 changes: 2 additions & 0 deletions src/test/java/nl/esciencecenter/restape/RestApeControllerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ void runSynthesisFail() throws IOException, OWLOntologyCreationException {
String content = FileUtils.readFileToString(APEFiles.readPathToFile(configPath),
StandardCharsets.UTF_8);
JSONObject jsonObject = new JSONObject(content);
jsonObject.put("solutions", "1");
JSONArray result = ApeAPI.runSynthesis(jsonObject, false);
assertTrue(result.isEmpty(), "The encoding should be UNSAT.");
}
Expand All @@ -145,6 +146,7 @@ void runSynthesisPass() throws IOException, OWLOntologyCreationException {
String content = FileUtils.readFileToString(APEFiles.readPathToFile(configPath),
StandardCharsets.UTF_8);
JSONObject jsonObject = new JSONObject(content);
jsonObject.put("solutions", "1");
JSONArray result = ApeAPI.runSynthesis(jsonObject, false);
assertFalse(result.isEmpty(), "The encoding should be SAT.");
}
Expand Down
6 changes: 4 additions & 2 deletions src/test/java/nl/esciencecenter/restape/ToolBenchmarkingAPIsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;

import nl.esciencecenter.models.benchmarks.OpenEBenchmark;

/**
* {@link ToolBenchmarkingAPIsTest} tests the methods in
* {@link ToolBenchmarkingAPIs}.
Expand Down Expand Up @@ -91,7 +93,7 @@ void testIsOSIFromOEBMetrics() {

// Testing the isOSIFromOEBMetrics method and asserting that the OSI status is
// true
assertTrue(ToolBenchmarkingAPIs.isOSIFromOEBMetrics(mockToolMetrics) == LicenseType.OSI_Approved,
assertTrue(OpenEBenchmark.isOSIFromOEBMetrics(mockToolMetrics) == LicenseType.OSI_Approved,
"The method should return true for OSI license");
}

Expand All @@ -117,7 +119,7 @@ void testFetchToolMetricsPerVersionFromOEB() {
void testFetchOEBMetricsForBiotoolsVersion() {
try {
JSONObject toolMetrics = ToolBenchmarkingAPIs.fetchOEBMetricsForBiotoolsVersion("shelx");
ToolBenchmarkingAPIs.isOSIFromOEBMetrics(toolMetrics);
OpenEBenchmark.isOSIFromOEBMetrics(toolMetrics);
} catch (JSONException e) {
fail("An exception occurred while checking for 'osi license' in 'project'. The JSON object could not be parsed correctly.");
} catch (ClassCastException e) {
Expand Down

0 comments on commit 745afeb

Please sign in to comment.