From 318e5e95a22abb39540a357675c6b9dde676bf93 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Tue, 20 Sep 2022 15:30:48 +0200 Subject: [PATCH 1/9] Add JSON schema validator originally developed in OERSI (#443) --- metafacture-json/build.gradle | 1 + .../org/metafacture/json/JsonValidator.java | 176 ++++++++++++++++++ .../main/resources/flux-commands.properties | 1 + .../metafacture/json/JsonValidatorTest.java | 110 +++++++++++ .../src/test/resources/schemas/id.json | 8 + .../src/test/resources/schemas/schema.json | 13 ++ 6 files changed, 309 insertions(+) create mode 100644 metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java create mode 100644 metafacture-json/src/test/java/org/metafacture/json/JsonValidatorTest.java create mode 100644 metafacture-json/src/test/resources/schemas/id.json create mode 100644 metafacture-json/src/test/resources/schemas/schema.json diff --git a/metafacture-json/build.gradle b/metafacture-json/build.gradle index 1ffa26d2b..14f574988 100644 --- a/metafacture-json/build.gradle +++ b/metafacture-json/build.gradle @@ -22,6 +22,7 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-core:2.13.0' implementation 'com.fasterxml.jackson.core:jackson-databind:2.13.0' implementation 'com.jayway.jsonpath:json-path:2.6.0' + implementation 'com.github.erosb:everit-json-schema:1.14.1' testImplementation 'junit:junit:4.12' testImplementation 'org.mockito:mockito-core:2.5.5' testRuntimeOnly 'org.slf4j:slf4j-simple:1.7.21' diff --git a/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java b/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java new file mode 100644 index 000000000..7e8e6e4df --- /dev/null +++ b/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java @@ -0,0 +1,176 @@ +/* + * Copyright 2021, 2022 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.metafacture.json; + +import org.metafacture.framework.FluxCommand; +import org.metafacture.framework.MetafactureException; +import org.metafacture.framework.ObjectReceiver; +import org.metafacture.framework.annotations.Description; +import org.metafacture.framework.annotations.In; +import org.metafacture.framework.annotations.Out; +import org.metafacture.framework.helpers.DefaultObjectPipe; + +import org.everit.json.schema.Schema; +import org.everit.json.schema.ValidationException; +import org.everit.json.schema.loader.SchemaClient; +import org.everit.json.schema.loader.SchemaLoader; +import org.json.JSONException; +import org.json.JSONObject; +import org.json.JSONTokener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; + +/** + * Validate JSON against a given schema, pass only valid input to the receiver. + * + * @author Fabian Steeg (fsteeg) + */ +@Description("Validate JSON against a given schema, send only valid input to the receiver. Pass the schema location to validate against. " + + "Set 'schemaRoot' for resolving sub-schemas referenced in '$id' or '$ref' (defaults to the classpath root: '/'). " + + "Write valid and/or invalid output to locations specified with 'writeValid' and 'writeInvalid'.") +@In(String.class) +@Out(String.class) +@FluxCommand("validate-json") +public final class JsonValidator extends DefaultObjectPipe> { + + private static final Logger LOG = LoggerFactory.getLogger(JsonValidator.class); + private static final String DEFAULT_SCHEMA_ROOT = "/"; + private String schemaUrl; + private Schema schema; + private long fail; + private long success; + private FileWriter writeInvalid; + private FileWriter writeValid; + private String schemaRoot = DEFAULT_SCHEMA_ROOT; + + /** + * @param url The URL of the schema to validate against. + */ + public JsonValidator(final String url) { + this.schemaUrl = url; + } + + /** + * @param schemaRoot The root location for resolving sub-schemas referenced in '$id' or '$ref'. + */ + public void setSchemaRoot(final String schemaRoot) { + this.schemaRoot = schemaRoot; + } + + /** + * @param writeValid The location to write valid data to. + */ + public void setWriteValid(final String writeValid) { + this.writeValid = fileWriter(writeValid); + } + + /** + * @param writeInvalid The location to write invalid data to. + */ + public void setWriteInvalid(final String writeInvalid) { + this.writeInvalid = fileWriter(writeInvalid); + } + + @Override + public void process(final String json) { + JSONObject object = null; + try { + object = new JSONObject(json); // throws JSONException on syntax error + } + catch (final JSONException e) { + handleInvalid(json, null, e.getMessage()); + } + try { + initSchema(); + schema.validate(object); // throws ValidationException if invalid + getReceiver().process(json); + ++success; + write(json, writeValid); + } + catch (final ValidationException e) { + handleInvalid(json, object, e.getAllMessages().toString()); + } + } + + @Override + protected void onCloseStream() { + close(writeInvalid); + close(writeValid); + LOG.debug("Success: {}, Fail: {}", success, fail); + super.onCloseStream(); + } + + private void initSchema() { + if (schema != null) { + return; + } + try (InputStream inputStream = getClass().getResourceAsStream(schemaUrl)) { + schema = SchemaLoader.builder() + .schemaJson(new JSONObject(new JSONTokener(inputStream))) + .schemaClient(SchemaClient.classPathAwareClient()) + .resolutionScope("classpath://" + schemaRoot) + .build().load().build(); + } + catch (final IOException | JSONException e) { + throw new MetafactureException(e.getMessage(), e); + } + } + + private FileWriter fileWriter(final String fileLocation) { + try { + return new FileWriter(fileLocation); + } + catch (final IOException e) { + throw new MetafactureException(e.getMessage(), e); + } + } + + private void handleInvalid(final String json, final JSONObject object, + final String errorMessage) { + LOG.info("Invalid JSON: {} in {}", errorMessage, object != null ? object.opt("id") : json); + ++fail; + write(json, writeInvalid); + } + + private void write(final String json, final FileWriter fileWriter) { + if (fileWriter != null) { + try { + fileWriter.append(json); + fileWriter.append("\n"); + } + catch (final IOException e) { + throw new MetafactureException(e.getMessage(), e); + } + } + } + + private void close(final FileWriter fileWriter) { + if (fileWriter != null) { + try { + fileWriter.close(); + } + catch (final IOException e) { + throw new MetafactureException(e.getMessage(), e); + } + } + } + +} diff --git a/metafacture-json/src/main/resources/flux-commands.properties b/metafacture-json/src/main/resources/flux-commands.properties index 2d9cedee1..27bc9a13b 100644 --- a/metafacture-json/src/main/resources/flux-commands.properties +++ b/metafacture-json/src/main/resources/flux-commands.properties @@ -15,3 +15,4 @@ # encode-json org.metafacture.json.JsonEncoder decode-json org.metafacture.json.JsonDecoder +validate-json org.metafacture.json.JsonValidator diff --git a/metafacture-json/src/test/java/org/metafacture/json/JsonValidatorTest.java b/metafacture-json/src/test/java/org/metafacture/json/JsonValidatorTest.java new file mode 100644 index 000000000..e60bb7083 --- /dev/null +++ b/metafacture-json/src/test/java/org/metafacture/json/JsonValidatorTest.java @@ -0,0 +1,110 @@ +/* + * Copyright 2021, 2022 Fabian Steeg, hbz + * + * Licensed under the Apache License, Version 2.0 the "License"; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.metafacture.json; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.metafacture.framework.MetafactureException; +import org.metafacture.framework.ObjectReceiver; +import org.mockito.InOrder; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockitoAnnotations; + +/** + * Tests for {@link JsonValidator}. + * + * @author Fabian Steeg + * + */ +public final class JsonValidatorTest { + + private static final String SCHEMA = "/schemas/schema.json"; + private static final String JSON_VALID = "{\"id\":\"http://example.org/\"}"; + private static final String JSON_INVALID_MISSING_REQUIRED = "{}"; + private static final String JSON_INVALID_URI_FORMAT= "{\"id\":\"example.org/\"}"; + private static final String JSON_INVALID_DUPLICATE_KEY = "{\"id\":\"val\",\"id\":\"val\"}"; + private static final String JSON_INVALID_SYNTAX_ERROR = "{\"id1\":\"val\",\"id2\":\"val\""; + + private JsonValidator validator; + + @Mock + private ObjectReceiver receiver; + private InOrder inOrder; + + @Before + public void setup() { + MockitoAnnotations.initMocks(this); + validator = new JsonValidator(SCHEMA); + validator.setSchemaRoot("/schemas/"); + validator.setReceiver(receiver); + inOrder = Mockito.inOrder(receiver); + } + + @Test + public void testShouldValidate() { + validator.process(JSON_VALID); + inOrder.verify(receiver, Mockito.calls(1)).process(JSON_VALID); + } + + @Test + public void testShouldInvalidateMissingRequired() { + validator.process(JSON_INVALID_MISSING_REQUIRED); + inOrder.verifyNoMoreInteractions(); + } + + @Test + public void testShouldInvalidateUriFormat() { + validator.process(JSON_INVALID_URI_FORMAT); + inOrder.verifyNoMoreInteractions(); + } + + @Test + public void testShouldInvalidateDuplicateKey() { + validator.process(JSON_INVALID_DUPLICATE_KEY); + inOrder.verifyNoMoreInteractions(); + } + + @Test + public void testShouldInvalidateSyntaxError() { + validator.process(JSON_INVALID_SYNTAX_ERROR); + inOrder.verifyNoMoreInteractions(); + } + + @Test(expected = MetafactureException.class) + public void testShouldCatchMissingSchemaFile() { + new JsonValidator("").process(""); + } + + @Test(expected = MetafactureException.class) + public void testShouldCatchMissingValidOutputFile() { + validator.setWriteValid(""); + validator.process(JSON_INVALID_MISSING_REQUIRED); + } + + @Test(expected = MetafactureException.class) + public void testShouldCatchMissingInvalidOutputFile() { + validator.setWriteInvalid(""); + validator.process(JSON_INVALID_MISSING_REQUIRED); + } + + @After + public void cleanup() { + validator.closeStream(); + } + +} diff --git a/metafacture-json/src/test/resources/schemas/id.json b/metafacture-json/src/test/resources/schemas/id.json new file mode 100644 index 000000000..255cefcfd --- /dev/null +++ b/metafacture-json/src/test/resources/schemas/id.json @@ -0,0 +1,8 @@ +{ + "$id": "id.json", + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "URL", + "description": "The URL/URI of the resource", + "type": "string", + "format": "uri" +} diff --git a/metafacture-json/src/test/resources/schemas/schema.json b/metafacture-json/src/test/resources/schemas/schema.json new file mode 100644 index 000000000..fac9e0a53 --- /dev/null +++ b/metafacture-json/src/test/resources/schemas/schema.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "schema.json", + "type": "object", + "properties": { + "id": { + "$ref": "id.json" + } + }, + "required": [ + "id" + ] +} From f182204d62c46e378e806ea352e4b065d1aefae8 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Thu, 22 Sep 2022 10:15:27 +0200 Subject: [PATCH 2/9] Avoid duplicate failure handling on syntax errors (#443) --- .../src/main/java/org/metafacture/json/JsonValidator.java | 6 +++++- .../test/java/org/metafacture/json/JsonValidatorTest.java | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java b/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java index 7e8e6e4df..a8a9c71e5 100644 --- a/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java +++ b/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java @@ -91,13 +91,17 @@ public void setWriteInvalid(final String writeInvalid) { @Override public void process(final String json) { - JSONObject object = null; + final JSONObject object; try { object = new JSONObject(json); // throws JSONException on syntax error + validate(json, object); } catch (final JSONException e) { handleInvalid(json, null, e.getMessage()); } + } + + private void validate(final String json, final JSONObject object) { try { initSchema(); schema.validate(object); // throws ValidationException if invalid diff --git a/metafacture-json/src/test/java/org/metafacture/json/JsonValidatorTest.java b/metafacture-json/src/test/java/org/metafacture/json/JsonValidatorTest.java index e60bb7083..aae7dffaf 100644 --- a/metafacture-json/src/test/java/org/metafacture/json/JsonValidatorTest.java +++ b/metafacture-json/src/test/java/org/metafacture/json/JsonValidatorTest.java @@ -87,7 +87,7 @@ public void testShouldInvalidateSyntaxError() { @Test(expected = MetafactureException.class) public void testShouldCatchMissingSchemaFile() { - new JsonValidator("").process(""); + new JsonValidator("").process("{}"); } @Test(expected = MetafactureException.class) From 8a1ceb59b8487221e7c4fbb3270ce217f6dc6ebc Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Thu, 22 Sep 2022 10:34:14 +0200 Subject: [PATCH 3/9] Make JSON key for the record ID value configurable (#443) --- .../java/org/metafacture/json/JsonValidator.java | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java b/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java index a8a9c71e5..77ec4cfd1 100644 --- a/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java +++ b/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java @@ -45,7 +45,8 @@ */ @Description("Validate JSON against a given schema, send only valid input to the receiver. Pass the schema location to validate against. " + "Set 'schemaRoot' for resolving sub-schemas referenced in '$id' or '$ref' (defaults to the classpath root: '/'). " + - "Write valid and/or invalid output to locations specified with 'writeValid' and 'writeInvalid'.") + "Write valid and/or invalid output to locations specified with 'writeValid' and 'writeInvalid'." + + "Set the JSON key for the record ID value with 'idKey' (for logging output, defaults to 'id').") @In(String.class) @Out(String.class) @FluxCommand("validate-json") @@ -53,6 +54,7 @@ public final class JsonValidator extends DefaultObjectPipe Date: Thu, 22 Sep 2022 10:37:09 +0200 Subject: [PATCH 4/9] Use backticks for Markdown code formatting in `@Description` (#443) --- .../src/main/java/org/metafacture/json/JsonValidator.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java b/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java index 77ec4cfd1..ed99265f9 100644 --- a/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java +++ b/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java @@ -44,9 +44,9 @@ * @author Fabian Steeg (fsteeg) */ @Description("Validate JSON against a given schema, send only valid input to the receiver. Pass the schema location to validate against. " + - "Set 'schemaRoot' for resolving sub-schemas referenced in '$id' or '$ref' (defaults to the classpath root: '/'). " + - "Write valid and/or invalid output to locations specified with 'writeValid' and 'writeInvalid'." + - "Set the JSON key for the record ID value with 'idKey' (for logging output, defaults to 'id').") + "Set `schemaRoot` for resolving sub-schemas referenced in `$id` or `$ref` (defaults to the classpath root: `/`). " + + "Write valid and/or invalid output to locations specified with `writeValid` and `writeInvalid`." + + "Set the JSON key for the record ID value with `idKey` (for logging output, defaults to `id`).") @In(String.class) @Out(String.class) @FluxCommand("validate-json") From 43989affc1f9e62e6e5751f2f2b322b9262d71a8 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Thu, 22 Sep 2022 11:05:06 +0200 Subject: [PATCH 5/9] Add missing space in `@Description`, inline JSONObject (#443) --- .../src/main/java/org/metafacture/json/JsonValidator.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java b/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java index ed99265f9..3b603aff2 100644 --- a/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java +++ b/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java @@ -45,7 +45,7 @@ */ @Description("Validate JSON against a given schema, send only valid input to the receiver. Pass the schema location to validate against. " + "Set `schemaRoot` for resolving sub-schemas referenced in `$id` or `$ref` (defaults to the classpath root: `/`). " + - "Write valid and/or invalid output to locations specified with `writeValid` and `writeInvalid`." + + "Write valid and/or invalid output to locations specified with `writeValid` and `writeInvalid`. " + "Set the JSON key for the record ID value with `idKey` (for logging output, defaults to `id`).") @In(String.class) @Out(String.class) @@ -101,10 +101,8 @@ public void setIdKey(final String idKey) { @Override public void process(final String json) { - final JSONObject object; try { - object = new JSONObject(json); // throws JSONException on syntax error - validate(json, object); + validate(json, new JSONObject(json) /* throws JSONException on syntax error */); } catch (final JSONException e) { handleInvalid(json, null, e.getMessage()); From 2a9cb327a09ea46cacc17a639d4fdc3626630820 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Wed, 1 Feb 2023 12:56:24 +0100 Subject: [PATCH 6/9] Mock serving the local test schema via HTTP (#443) --- metafacture-json/build.gradle | 1 + .../metafacture/json/JsonValidatorTest.java | 37 ++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/metafacture-json/build.gradle b/metafacture-json/build.gradle index 14f574988..01a5b23a7 100644 --- a/metafacture-json/build.gradle +++ b/metafacture-json/build.gradle @@ -25,5 +25,6 @@ dependencies { implementation 'com.github.erosb:everit-json-schema:1.14.1' testImplementation 'junit:junit:4.12' testImplementation 'org.mockito:mockito-core:2.5.5' + testImplementation 'com.github.tomakehurst:wiremock-jre8:2.33.2' testRuntimeOnly 'org.slf4j:slf4j-simple:1.7.21' } diff --git a/metafacture-json/src/test/java/org/metafacture/json/JsonValidatorTest.java b/metafacture-json/src/test/java/org/metafacture/json/JsonValidatorTest.java index aae7dffaf..df5f37dba 100644 --- a/metafacture-json/src/test/java/org/metafacture/json/JsonValidatorTest.java +++ b/metafacture-json/src/test/java/org/metafacture/json/JsonValidatorTest.java @@ -15,8 +15,21 @@ */ package org.metafacture.json; +import static org.hamcrest.CoreMatchers.both; +import static org.hamcrest.CoreMatchers.containsString; +import static org.junit.Assert.assertThat; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.MalformedURLException; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.util.stream.Collectors; + import org.junit.After; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; import org.metafacture.framework.MetafactureException; import org.metafacture.framework.ObjectReceiver; @@ -25,6 +38,11 @@ import org.mockito.Mockito; import org.mockito.MockitoAnnotations; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.core.WireMockConfiguration; +import com.github.tomakehurst.wiremock.junit.WireMockRule; + + /** * Tests for {@link JsonValidator}. * @@ -46,15 +64,32 @@ public final class JsonValidatorTest { private ObjectReceiver receiver; private InOrder inOrder; + @Rule + public WireMockRule wireMockRule = new WireMockRule(WireMockConfiguration.wireMockConfig() + .jettyAcceptors(Runtime.getRuntime().availableProcessors()).dynamicPort()); + @Before - public void setup() { + public void setup() throws IOException { MockitoAnnotations.initMocks(this); + WireMock.stubFor(WireMock.request("GET", WireMock.urlEqualTo("/schema")) + .willReturn(WireMock.ok().withBody(readToString(getClass().getResource(SCHEMA))))); validator = new JsonValidator(SCHEMA); validator.setSchemaRoot("/schemas/"); validator.setReceiver(receiver); inOrder = Mockito.inOrder(receiver); } + private String readToString(final URL url) throws IOException { + return new BufferedReader(new InputStreamReader(url.openStream(), StandardCharsets.UTF_8)) + .lines().collect(Collectors.joining("\n")); + } + + @Test + public void callWireMockSchema() throws MalformedURLException, IOException { + final String schemaContent = readToString(new URL(wireMockRule.baseUrl() + "/schema")); + assertThat(schemaContent, both(containsString("$schema")).and(containsString("$ref"))); + } + @Test public void testShouldValidate() { validator.process(JSON_VALID); From 9efce49e04b20a9504d7128713021bae77bebbe7 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Wed, 1 Feb 2023 17:04:28 +0100 Subject: [PATCH 7/9] Support loading main schema and relative refs from URLs (#443) Remove schemaRoot config, resolutionScope is now set to base path --- .../org/metafacture/json/JsonValidator.java | 44 +++++++++++-------- .../metafacture/json/JsonValidatorTest.java | 44 ++++++++++++++++--- 2 files changed, 63 insertions(+), 25 deletions(-) diff --git a/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java b/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java index 3b603aff2..4188efa51 100644 --- a/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java +++ b/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java @@ -1,5 +1,5 @@ /* - * Copyright 2021, 2022 Fabian Steeg, hbz + * Copyright 2021, 2023 Fabian Steeg, hbz * * Licensed under the Apache License, Version 2.0 the "License"; * you may not use this file except in compliance with the License. @@ -28,6 +28,7 @@ import org.everit.json.schema.ValidationException; import org.everit.json.schema.loader.SchemaClient; import org.everit.json.schema.loader.SchemaLoader; +import org.everit.json.schema.loader.SchemaLoader.SchemaLoaderBuilder; import org.json.JSONException; import org.json.JSONObject; import org.json.JSONTokener; @@ -37,6 +38,7 @@ import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; +import java.net.URL; /** * Validate JSON against a given schema, pass only valid input to the receiver. @@ -44,7 +46,6 @@ * @author Fabian Steeg (fsteeg) */ @Description("Validate JSON against a given schema, send only valid input to the receiver. Pass the schema location to validate against. " + - "Set `schemaRoot` for resolving sub-schemas referenced in `$id` or `$ref` (defaults to the classpath root: `/`). " + "Write valid and/or invalid output to locations specified with `writeValid` and `writeInvalid`. " + "Set the JSON key for the record ID value with `idKey` (for logging output, defaults to `id`).") @In(String.class) @@ -53,7 +54,6 @@ public final class JsonValidator extends DefaultObjectPipe> { private static final Logger LOG = LoggerFactory.getLogger(JsonValidator.class); - private static final String DEFAULT_SCHEMA_ROOT = "/"; private static final String DEFAULT_ID_KEY = "id"; private String schemaUrl; private Schema schema; @@ -61,7 +61,6 @@ public final class JsonValidator extends DefaultObjectPipe receiver; private InOrder inOrder; + private Function schemaLocationGetter; @Rule public WireMockRule wireMockRule = new WireMockRule(WireMockConfiguration.wireMockConfig() .jettyAcceptors(Runtime.getRuntime().availableProcessors()).dynamicPort()); + @Parameterized.Parameters(name = "{index}") + public static Collection siteMaps() { + return Arrays.asList((Object[][]) (new Function[][] { // + // Pass the schema to each test as path on classpath, file url, and http url: + { (Object rule) -> MAIN_SCHEMA }, + { (Object rule) -> JsonValidatorTest.class.getResource(MAIN_SCHEMA).toString() }, + { (Object rule) -> ((WireMockRule) rule).baseUrl() + MAIN_SCHEMA } })); + } + + public JsonValidatorTest(Function schemaLocationGetter) { + this.schemaLocationGetter = schemaLocationGetter; + } + @Before public void setup() throws IOException { MockitoAnnotations.initMocks(this); - WireMock.stubFor(WireMock.request("GET", WireMock.urlEqualTo("/schema")) - .willReturn(WireMock.ok().withBody(readToString(getClass().getResource(SCHEMA))))); - validator = new JsonValidator(SCHEMA); - validator.setSchemaRoot("/schemas/"); + wireMock(MAIN_SCHEMA, ID_SCHEMA); + String schemaLocation = schemaLocationGetter.apply(wireMockRule); + validator = new JsonValidator(schemaLocation); validator.setReceiver(receiver); inOrder = Mockito.inOrder(receiver); } + private void wireMock(final String... schemaLocations) throws IOException { + for (String schemaLocation : schemaLocations) { + stubFor(request("GET", WireMock.urlEqualTo(schemaLocation)).willReturn( + WireMock.ok().withBody(readToString(getClass().getResource(schemaLocation))) + .withHeader("Content-type", "application/json"))); + } + } + private String readToString(final URL url) throws IOException { return new BufferedReader(new InputStreamReader(url.openStream(), StandardCharsets.UTF_8)) .lines().collect(Collectors.joining("\n")); @@ -86,7 +116,7 @@ private String readToString(final URL url) throws IOException { @Test public void callWireMockSchema() throws MalformedURLException, IOException { - final String schemaContent = readToString(new URL(wireMockRule.baseUrl() + "/schema")); + final String schemaContent = readToString(new URL(wireMockRule.baseUrl() + MAIN_SCHEMA)); assertThat(schemaContent, both(containsString("$schema")).and(containsString("$ref"))); } From 8b7ad59ef7adf52ce1cdfd3da19d2316f441c150 Mon Sep 17 00:00:00 2001 From: Fabian Steeg Date: Thu, 2 Feb 2023 09:53:24 +0100 Subject: [PATCH 8/9] Load schema in constructor to avoid reload on each validation (#443) --- .../src/main/java/org/metafacture/json/JsonValidator.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java b/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java index 4188efa51..664078511 100644 --- a/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java +++ b/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java @@ -55,7 +55,6 @@ public final class JsonValidator extends DefaultObjectPipe Date: Fri, 3 Feb 2023 10:34:01 +0100 Subject: [PATCH 9/9] Switch logging output about schema loading details to DEBUG (#443) --- .../src/main/java/org/metafacture/json/JsonValidator.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java b/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java index 664078511..8db6d0bbe 100644 --- a/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java +++ b/metafacture-json/src/main/java/org/metafacture/json/JsonValidator.java @@ -131,7 +131,7 @@ private void initSchema(final String schemaUrl) { .resolutionScope(baseFor(url.toString())); } catch (final IOException e) { - LOG.info("Could not read as URL: {}, trying to load from class path", schemaUrl); + LOG.debug("Could not read as URL: {}, trying to load from class path", schemaUrl); schemaLoader = schemaLoader.schemaClient(SchemaClient.classPathAwareClient()) .schemaJson(jsonFrom(getClass().getResourceAsStream(schemaUrl))) .resolutionScope("classpath://" + baseFor(schemaUrl));