Skip to content

Commit

Permalink
MODINV-1012: Implementation for Validation for NatureContentTerms add…
Browse files Browse the repository at this point in the history
…ed for Instance. Tests added. (#715)

* MODINV-1012: Implementation for Validation for NatureContentTerms added for Instance. Tests added.

* MODINV-1012: Documentation added.

* MODINV-1012: Documentation added.

* MODINV-1012: Tests added.

(cherry picked from commit 2019a17)
  • Loading branch information
VRohach authored and KaterynaSenchenko committed Apr 22, 2024
1 parent cdec7ba commit a8d77b0
Show file tree
Hide file tree
Showing 6 changed files with 281 additions and 3 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.folio.inventory.dataimport.services.OrderHelperService;
import org.folio.inventory.dataimport.util.AdditionalFieldsUtil;
import org.folio.inventory.dataimport.util.ParsedRecordUtil;
import org.folio.inventory.dataimport.util.ValidationUtil;
import org.folio.inventory.domain.instances.Instance;
import org.folio.inventory.domain.instances.InstanceCollection;
import org.folio.inventory.domain.relationship.RecordToEntity;
Expand Down Expand Up @@ -118,15 +119,24 @@ public CompletableFuture<DataImportEventPayload> handle(DataImportEventPayload d
.compose(v -> {
InstanceCollection instanceCollection = storage.getInstanceCollection(context);
JsonObject instanceAsJson = prepareInstance(dataImportEventPayload, instanceId, jobExecutionId);
List<String> errors = EventHandlingUtil.validateJsonByRequiredFields(instanceAsJson, requiredFields);
if (!errors.isEmpty()) {
String msg = format("Mapped Instance is invalid: %s, by jobExecutionId: '%s' and recordId: '%s' and chunkId: '%s' ", errors,
List<String> requiredFieldsErrors = EventHandlingUtil.validateJsonByRequiredFields(instanceAsJson, requiredFields);
if (!requiredFieldsErrors.isEmpty()) {
String msg = format("Mapped Instance is invalid: %s, by jobExecutionId: '%s' and recordId: '%s' and chunkId: '%s' ", requiredFieldsErrors,
jobExecutionId, recordId, chunkId);
LOGGER.warn(msg);
return Future.failedFuture(msg);
}

Instance mappedInstance = Instance.fromJson(instanceAsJson);

List<String> invalidUUIDsErrors = ValidationUtil.validateUUIDs(mappedInstance);
if (!invalidUUIDsErrors.isEmpty()) {
String msg = format("Mapped Instance is invalid: %s, by jobExecutionId: '%s' and recordId: '%s' and chunkId: '%s' ", invalidUUIDsErrors,
jobExecutionId, recordId, chunkId);
LOGGER.warn(msg);
return Future.failedFuture(msg);
}

return addInstance(mappedInstance, instanceCollection)
.compose(createdInstance -> getPrecedingSucceedingTitlesHelper().createPrecedingSucceedingTitles(mappedInstance, context).map(createdInstance))
.compose(createdInstance -> executeFieldsManipulation(createdInstance, targetRecord))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import org.folio.inventory.dataimport.cache.MappingMetadataCache;
import org.folio.inventory.dataimport.handlers.matching.util.EventHandlingUtil;
import org.folio.inventory.dataimport.util.AdditionalFieldsUtil;
import org.folio.inventory.dataimport.util.ValidationUtil;
import org.folio.inventory.domain.instances.Instance;
import org.folio.inventory.domain.instances.InstanceCollection;
import org.folio.inventory.exceptions.NotFoundException;
Expand Down Expand Up @@ -194,6 +195,14 @@ private void processInstanceUpdate(DataImportEventPayload dataImportEventPayload
org.folio.rest.jaxrs.model.Record targetRecord = Json.decodeValue(marcBibAsJson, org.folio.rest.jaxrs.model.Record.class);

Instance mappedInstance = Instance.fromJson(instanceAsJson);
List<String> invalidUUIDsErrors = ValidationUtil.validateUUIDs(mappedInstance);
if (!invalidUUIDsErrors.isEmpty()) {
String msg = format("Mapped Instance is invalid: %s, by jobExecutionId: '%s' and recordId: '%s' and chunkId: '%s' ", invalidUUIDsErrors,
jobExecutionId, recordId, chunkId);
LOGGER.warn(msg);
return Future.failedFuture(msg);
}

return updateInstanceAndRetryIfOlExists(mappedInstance, instanceCollection, dataImportEventPayload)
.compose(updatedInstance -> getPrecedingSucceedingTitlesHelper().getExistingPrecedingSucceedingTitles(mappedInstance, context))
.map(precedingSucceedingTitles -> precedingSucceedingTitles.stream()
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package org.folio.inventory.dataimport.util;

import org.folio.inventory.domain.instances.Instance;

import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

/**
* Util for detailed validation different entities.
*/
public class ValidationUtil {

private ValidationUtil() {
}

/**
* Validate fields inside the Instance entity. Validation based on checking if specific fields were mapped as UUIDs.
* If not - then the list with errors will be returned.
* Example: "Value 'invalid not UUID value' is not a UUID for someFieldName field"
* @param instance target Instance for validation
* @return ArrayList with errors when the needed fields are NOT as UUID.
*/
public static List<String> validateUUIDs(Instance instance) {
ArrayList<String> errorMessages = new ArrayList<>();

//TODO: This will be extended for different fields and entities.That's why there are so many methods just for 1 field.
// Branch for it extending validation: MODINV-1012-extended
validateField(errorMessages, instance.getNatureOfContentTermIds(), "natureOfContentTermIds");

return errorMessages;
}

private static void validateField(List<String> errorMessages, List<String> values, String fieldName) {
values.stream()
.filter(value -> !isUUID(value))
.forEach(value -> errorMessages.add(String.format("Value '%s' is not a UUID for %s field", value, fieldName)));
}

private static boolean isUUID(String value) {
try {
UUID.fromString(value);
return true;
} catch (Exception ex) {
return false;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.folio.processing.mapping.defaultmapper.processor.parameters.MappingParameters;
import org.folio.processing.mapping.mapper.reader.Reader;
import org.folio.processing.mapping.mapper.reader.record.marc.MarcBibReaderFactory;
import org.folio.processing.value.ListValue;
import org.folio.processing.value.MissingValue;
import org.folio.processing.value.StringValue;
import org.folio.rest.client.SourceStorageRecordsClient;
Expand Down Expand Up @@ -172,6 +173,44 @@ public class CreateInstanceEventHandlerTest {
.withContentType(MAPPING_PROFILE)
.withContent(JsonObject.mapFrom(mappingProfile).getMap())))));

private JobProfile jobProfileWithNatureOfContentTerm = new JobProfile()
.withId(UUID.randomUUID().toString())
.withName("Create MARC Bibs with NatureOfContentTerm")
.withDataType(JobProfile.DataType.MARC);

private ActionProfile actionProfileWithNatureOfContentTerm = new ActionProfile()
.withId(UUID.randomUUID().toString())
.withName("Create preliminary Item with NatureOfContentTerm")
.withAction(ActionProfile.Action.CREATE)
.withFolioRecord(INSTANCE);

private MappingProfile mappingProfileWithNatureOfContentTerm = new MappingProfile()
.withId(UUID.randomUUID().toString())
.withName("Prelim item from MARC with NatureOfContentTerm")
.withIncomingRecordType(EntityType.MARC_BIBLIOGRAPHIC)
.withExistingRecordType(EntityType.INSTANCE)
.withMappingDetails(new MappingDetail()
.withMappingFields(Lists.newArrayList(
new MappingRule().withPath("instance.instanceTypeId").withValue("\"instanceTypeIdExpression\"").withEnabled("true"),
new MappingRule().withPath("instance.title").withValue("\"titleExpression\"").withEnabled("true"),
new MappingRule().withPath("instance.natureOfContentTermIds[]").withValue("\"not uuid\"").withEnabled("true").withRepeatableFieldAction(MappingRule.RepeatableFieldAction.EXTEND_EXISTING))));

private ProfileSnapshotWrapper profileSnapshotWrapperWithNatureOfContentTerm = new ProfileSnapshotWrapper()
.withId(UUID.randomUUID().toString())
.withProfileId(jobProfileWithNatureOfContentTerm.getId())
.withContentType(JOB_PROFILE)
.withContent(jobProfileWithNatureOfContentTerm)
.withChildSnapshotWrappers(Collections.singletonList(
new ProfileSnapshotWrapper()
.withProfileId(actionProfileWithNatureOfContentTerm.getId())
.withContentType(ACTION_PROFILE)
.withContent(actionProfileWithNatureOfContentTerm)
.withChildSnapshotWrappers(Collections.singletonList(
new ProfileSnapshotWrapper()
.withProfileId(mappingProfileWithNatureOfContentTerm.getId())
.withContentType(MAPPING_PROFILE)
.withContent(JsonObject.mapFrom(mappingProfileWithNatureOfContentTerm).getMap())))));

private CreateInstanceEventHandler createInstanceEventHandler;

@Before
Expand Down Expand Up @@ -586,6 +625,55 @@ public void shouldNotProcessEventIfRequiredFieldIsEmpty() throws InterruptedExce
future.get(5, TimeUnit.MILLISECONDS);
}

@Test(expected = Exception.class)
public void shouldNotProcessEventIfNatureContentFieldIsNotUUID() throws InterruptedException, ExecutionException, TimeoutException {
Reader fakeReader = Mockito.mock(Reader.class);

String instanceTypeId = "fe19bae4-da28-472b-be90-d442e2428ead";
String recordId = "567859ad-505a-400d-a699-0028a1fdbf84";
String instanceId = "4d4545df-b5ba-4031-a031-70b1c1b2fc5d";
String title = "titleValue";
RecordToEntity recordToInstance = RecordToEntity.builder().recordId(recordId).entityId(instanceId)
.build();

when(fakeReader.read(any(MappingRule.class))).thenReturn(StringValue.of(instanceTypeId), StringValue.of(title), ListValue.of(Lists.newArrayList("not uuid")));

when(fakeReaderFactory.createReader()).thenReturn(fakeReader);

when(storage.getInstanceCollection(any())).thenReturn(instanceRecordCollection);

when(instanceIdStorageService.store(any(), any(), any())).thenReturn(Future.succeededFuture(recordToInstance));

MappingManager.registerReaderFactory(fakeReaderFactory);
MappingManager.registerWriterFactory(new InstanceWriterFactory());

HashMap<String, String> context = new HashMap<>();
Record record = new Record().withParsedRecord(new ParsedRecord().withContent(PARSED_CONTENT));
record.setId(recordId);

context.put(MARC_BIBLIOGRAPHIC.value(), Json.encode(record));

Buffer buffer = BufferImpl.buffer("{\"parsedRecord\":{" +
"\"id\":\"990fad8b-64ec-4de4-978c-9f8bbed4c6d3\"," +
"\"content\":\"{\\\"leader\\\":\\\"00574nam 22001211a 4500\\\",\\\"fields\\\":[{\\\"035\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"(in001)ybp7406411\\\"}],\\\"ind1\\\":\\\" \\\",\\\"ind2\\\":\\\" \\\"}},{\\\"245\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"titleValue\\\"}],\\\"ind1\\\":\\\"1\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"336\\\":{\\\"subfields\\\":[{\\\"b\\\":\\\"b6698d38-149f-11ec-82a8-0242ac130003\\\"}],\\\"ind1\\\":\\\"1\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"780\\\":{\\\"subfields\\\":[{\\\"t\\\":\\\"Houston oil directory\\\"}],\\\"ind1\\\":\\\"0\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"785\\\":{\\\"subfields\\\":[{\\\"t\\\":\\\"SAIS review of international affairs\\\"},{\\\"x\\\":\\\"1945-4724\\\"}],\\\"ind1\\\":\\\"0\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"500\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"Adaptation of Xi xiang ji by Wang Shifu.\\\"}],\\\"ind1\\\":\\\" \\\",\\\"ind2\\\":\\\" \\\"}},{\\\"520\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"Ben shu miao shu le cui ying ying he zhang sheng wei zheng qu hun yin zi you li jin qu zhe jian xin zhi hou, zhong cheng juan shu de ai qing gu shi. jie lu le bao ban hun yin he feng jian li jiao de zui e.\\\"}],\\\"ind1\\\":\\\" \\\",\\\"ind2\\\":\\\" \\\"}},{\\\"999\\\":{\\\"subfields\\\":[{\\\"i\\\":\\\"4d4545df-b5ba-4031-a031-70b1c1b2fc5d\\\"}],\\\"ind1\\\":\\\"f\\\",\\\"ind2\\\":\\\"f\\\"}}]}\"" +
"}}");
HttpResponse<Buffer> resp = buildHttpResponseWithBuffer(buffer);
when(sourceStorageClient.postSourceStorageRecords(any())).thenReturn(Future.succeededFuture(resp));

DataImportEventPayload dataImportEventPayload = new DataImportEventPayload()
.withEventType(DI_INVENTORY_INSTANCE_CREATED.value())
.withContext(context)
.withCurrentNode(profileSnapshotWrapperWithNatureOfContentTerm.getChildSnapshotWrappers().get(0))
.withTenant(TENANT_ID)
.withOkapiUrl(mockServer.baseUrl())
.withToken(TOKEN)
.withJobExecutionId(UUID.randomUUID().toString())
.withOkapiUrl(mockServer.baseUrl());

CompletableFuture<DataImportEventPayload> future = createInstanceEventHandler.handle(dataImportEventPayload);
future.get(10, TimeUnit.SECONDS);
}

@Test(expected = Exception.class)
public void shouldNotProcessEventIfRecordContains999field() throws InterruptedException, ExecutionException, TimeoutException {
var recordId = UUID.randomUUID().toString();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import org.folio.processing.mapping.defaultmapper.processor.parameters.MappingParameters;
import org.folio.processing.mapping.mapper.reader.Reader;
import org.folio.processing.mapping.mapper.reader.record.marc.MarcBibReaderFactory;
import org.folio.processing.value.ListValue;
import org.folio.processing.value.MissingValue;
import org.folio.processing.value.StringValue;
import org.folio.rest.client.SourceStorageRecordsClient;
Expand Down Expand Up @@ -198,6 +199,44 @@ public class ReplaceInstanceEventHandlerTest {
.withContentType(MAPPING_PROFILE)
.withContent(JsonObject.mapFrom(mappingProfile).getMap())))));

private JobProfile jobProfileWithNatureOfContentTerm = new JobProfile()
.withId(UUID.randomUUID().toString())
.withName("Create MARC Bibs with NatureOfContentTerm")
.withDataType(JobProfile.DataType.MARC);

private ActionProfile actionProfileWithNatureOfContentTerm = new ActionProfile()
.withId(UUID.randomUUID().toString())
.withName("Replace preliminary Item with NatureOfContentTerm")
.withAction(ActionProfile.Action.UPDATE)
.withFolioRecord(INSTANCE);

private MappingProfile mappingProfileWithNatureOfContentTerm = new MappingProfile()
.withId(UUID.randomUUID().toString())
.withName("Prelim item from MARC with NatureOfContentTerm")
.withIncomingRecordType(EntityType.MARC_BIBLIOGRAPHIC)
.withExistingRecordType(EntityType.INSTANCE)
.withMappingDetails(new MappingDetail()
.withMappingFields(Lists.newArrayList(
new MappingRule().withPath("instance.instanceTypeId").withValue("\"instanceTypeIdExpression\"").withEnabled("true"),
new MappingRule().withPath("instance.title").withValue("\"titleExpression\"").withEnabled("true"),
new MappingRule().withPath("instance.natureOfContentTermIds[]").withValue("\"not uuid\"").withEnabled("true").withRepeatableFieldAction(MappingRule.RepeatableFieldAction.EXTEND_EXISTING))));

private ProfileSnapshotWrapper profileSnapshotWrapperWithNatureOfContentTerm = new ProfileSnapshotWrapper()
.withId(UUID.randomUUID().toString())
.withProfileId(jobProfileWithNatureOfContentTerm.getId())
.withContentType(JOB_PROFILE)
.withContent(jobProfileWithNatureOfContentTerm)
.withChildSnapshotWrappers(Collections.singletonList(
new ProfileSnapshotWrapper()
.withProfileId(actionProfileWithNatureOfContentTerm.getId())
.withContentType(ACTION_PROFILE)
.withContent(actionProfileWithNatureOfContentTerm)
.withChildSnapshotWrappers(Collections.singletonList(
new ProfileSnapshotWrapper()
.withProfileId(mappingProfileWithNatureOfContentTerm.getId())
.withContentType(MAPPING_PROFILE)
.withContent(JsonObject.mapFrom(mappingProfileWithNatureOfContentTerm).getMap())))));

private ReplaceInstanceEventHandler replaceInstanceEventHandler;
private PrecedingSucceedingTitlesHelper precedingSucceedingTitlesHelper;

Expand Down Expand Up @@ -824,6 +863,56 @@ public void shouldNotProcessEventIfRequiredFieldIsEmpty() throws InterruptedExce
future.get(5, TimeUnit.MILLISECONDS);
}

@Test(expected = ExecutionException.class)
public void shouldNotProcessEventIfNatureContentFieldIsNotUUID() throws InterruptedException, ExecutionException, TimeoutException {
Reader fakeReader = Mockito.mock(Reader.class);

String instanceTypeId = UUID.randomUUID().toString();
String title = "titleValue";

when(fakeReader.read(any(MappingRule.class))).thenReturn(StringValue.of(instanceTypeId), StringValue.of(title), ListValue.of(Lists.newArrayList("not uuid")));

when(fakeReaderFactory.createReader()).thenReturn(fakeReader);

when(storage.getInstanceCollection(any())).thenReturn(instanceRecordCollection);

MappingManager.registerReaderFactory(fakeReaderFactory);
MappingManager.registerWriterFactory(new InstanceWriterFactory());

HashMap<String, String> context = new HashMap<>();
Record record = new Record().withParsedRecord(new ParsedRecord().withContent(PARSED_CONTENT));
context.put(MARC_BIBLIOGRAPHIC.value(), Json.encode(record));
context.put(INSTANCE.value(), new JsonObject()
.put("id", instanceId)
.put("hrid", UUID.randomUUID().toString())
.put("source", MARC_INSTANCE_SOURCE)
.put("_version", INSTANCE_VERSION)
.put("discoverySuppress", false)
.encode());

mockInstance(MARC_INSTANCE_SOURCE);

Buffer buffer = BufferImpl.buffer("{\"parsedRecord\":{" +
"\"id\":\"990fad8b-64ec-4de4-978c-9f8bbed4c6d3\"," +
"\"content\":\"{\\\"leader\\\":\\\"00574nam 22001211a 4500\\\",\\\"fields\\\":[{\\\"035\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"(in001)ybp7406411\\\"}],\\\"ind1\\\":\\\" \\\",\\\"ind2\\\":\\\" \\\"}},{\\\"245\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"titleValue\\\"}],\\\"ind1\\\":\\\"1\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"336\\\":{\\\"subfields\\\":[{\\\"b\\\":\\\"b6698d38-149f-11ec-82a8-0242ac130003\\\"}],\\\"ind1\\\":\\\"1\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"780\\\":{\\\"subfields\\\":[{\\\"t\\\":\\\"Houston oil directory\\\"}],\\\"ind1\\\":\\\"0\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"785\\\":{\\\"subfields\\\":[{\\\"t\\\":\\\"SAIS review of international affairs\\\"},{\\\"x\\\":\\\"1945-4724\\\"}],\\\"ind1\\\":\\\"0\\\",\\\"ind2\\\":\\\"0\\\"}},{\\\"500\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"Adaptation of Xi xiang ji by Wang Shifu.\\\"}],\\\"ind1\\\":\\\" \\\",\\\"ind2\\\":\\\" \\\"}},{\\\"520\\\":{\\\"subfields\\\":[{\\\"a\\\":\\\"Ben shu miao shu le cui ying ying he zhang sheng wei zheng qu hun yin zi you li jin qu zhe jian xin zhi hou, zhong cheng juan shu de ai qing gu shi. jie lu le bao ban hun yin he feng jian li jiao de zui e.\\\"}],\\\"ind1\\\":\\\" \\\",\\\"ind2\\\":\\\" \\\"}},{\\\"999\\\":{\\\"subfields\\\":[{\\\"i\\\":\\\"4d4545df-b5ba-4031-a031-70b1c1b2fc5d\\\"}],\\\"ind1\\\":\\\"f\\\",\\\"ind2\\\":\\\"f\\\"}}]}\"" +
"}}");
HttpResponse<Buffer> respForPass = buildHttpResponseWithBuffer(buffer, HttpStatus.SC_OK);
when(sourceStorageClient.putSourceStorageRecordsGenerationById(any(), any())).thenReturn(Future.succeededFuture(respForPass));

DataImportEventPayload dataImportEventPayload = new DataImportEventPayload()
.withEventType(DI_INVENTORY_INSTANCE_CREATED.value())
.withContext(context)
.withCurrentNode(profileSnapshotWrapperWithNatureOfContentTerm.getChildSnapshotWrappers().get(0))
.withTenant(TENANT_ID)
.withOkapiUrl(mockServer.baseUrl())
.withToken(TOKEN)
.withJobExecutionId(UUID.randomUUID().toString());

CompletableFuture<DataImportEventPayload> future = replaceInstanceEventHandler.handle(dataImportEventPayload);
future.get(10, TimeUnit.SECONDS);
}


@Test
public void shouldReturnFailedFutureIfCurrentActionProfileHasNoMappingProfile() {
HashMap<String, String> context = new HashMap<>();
Expand Down
Loading

0 comments on commit a8d77b0

Please sign in to comment.