Skip to content

Commit

Permalink
Merged in DSC-776 (pull request DSpace#920)
Browse files Browse the repository at this point in the history 
[DSC-776] Restored mutliple entries import for bibtex files

Approved-by: Andrea Bollini
  • Loading branch information
@vins01-4science @abollini
vins01-4science authored and abollini committed Nov 1, 2023
2 parents 944ee2f + 61f5039 commit a81fbf6
Show file tree
Hide file tree
Showing 6 changed files with 210 additions and 55 deletions.
5 changes: 5 additions & 0 deletions ...va/org/dspace/importer/external/bibtex/service/BibtexImportMetadataSourceServiceImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -117,4 +117,9 @@ public void setMetadataFieldMap(@SuppressWarnings("rawtypes") Map metadataFieldM
super.setMetadataFieldMap(metadataFieldMap);
}

@Override
public boolean canImportMultipleRecords() {
return true;
}

}
89 changes: 72 additions & 17 deletions dspace-api/src/main/java/org/dspace/importer/external/service/ImportService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -311,36 +311,91 @@ public boolean canImportFromFile(String originalName) {
/*
* Get a collection of record from File,
* The first match will be return.
*
*
* @param file The file from which will read records
* @param originalName The original file name or full path
* @return a single record contains the metadatum
* @throws FileMultipleOccurencesException if more than one entry is found
*/
public ImportRecord getRecord(File file, String originalName)
throws FileMultipleOccurencesException, FileSourceException {
ImportRecord importRecords = null;
for (MetadataSource metadataSource : importSources.values()) {
try (InputStream fileInputStream = new FileInputStream(file)) {
if (metadataSource instanceof FileSource) {
FileSource fileSource = (FileSource)metadataSource;
if (fileSource.isValidSourceForFile(originalName)) {
importRecords = fileSource.getRecord(fileInputStream);
break;
try (InputStream fileInputStream = new FileInputStream(file)) {
FileSource fileSource = this.getFileSource(fileInputStream, originalName);
try {
if (fileSource.isValidSourceForFile(originalName)) {
return fileSource.getRecord(fileInputStream);
}
} catch (FileSourceException e) {
log.debug(fileSource.getImportSource() + " isn't a valid parser for file", e);
}
//catch statements is required because we could have supported format (i.e. XML)
//which fail on schema validation
} catch (FileMultipleOccurencesException e) {
log.debug("File contains multiple metadata, return with error");
throw e;
} catch (IOException e1) {
throw new FileSourceException("File cannot be read, may be null");
}
return null;
}

/**
* Get a collection of record from File,
*
* @param file The file from which will read records
* @param originalName The original file name or full path
* @return records containing metdatum
* @throws FileMultipleOccurencesException if the import configured for the {@code file}
* doesn't allow multiple records import.
* @throws FileSourceException if the file cannot be read.
*/
public List<ImportRecord> getRecords(File file, String originalName)
throws FileMultipleOccurencesException, FileSourceException {
try (InputStream fileInputStream = new FileInputStream(file)) {
FileSource fileSource = this.getFileSource(fileInputStream, originalName);
try {
if (fileSource.isValidSourceForFile(originalName)) {
List<ImportRecord> records = fileSource.getRecords(fileInputStream);
if (!fileSource.canImportMultipleRecords() && records.size() > 1) {
throw new FileMultipleOccurencesException(
"Found " + records.size() + " entries in file ( " +
originalName +
" ) but import source ( " +
fileSource.getImportSource() +
" ) not allowed to import multiple records"
);
}
return records;
}
} catch (FileSourceException e) {
log.debug(fileSource.getImportSource() + " isn't a valid parser for file", e);
}
//catch statements is required because we could have supported format (i.e. XML)
//which fail on schema validation
} catch (FileSourceException e) {
log.debug(metadataSource.getImportSource() + " isn't a valid parser for file");
} catch (FileMultipleOccurencesException e) {
log.debug("File contains multiple metadata, return with error");
throw e;
} catch (IOException e1) {
throw new FileSourceException("File cannot be read, may be null");
} catch (IOException e1) {
throw new FileSourceException("File cannot be read, may be null");
}
return null;
}

protected FileSource getFileSource(File file, String originalName) throws FileSourceException {
try (InputStream fileInputStream = new FileInputStream(file)) {
return getFileSource(file, originalName);
} catch (IOException e1) {
throw new FileSourceException("File cannot be read, may be null");
}
}

protected FileSource getFileSource(InputStream fileInputStream, String originalName) {
for (MetadataSource metadataSource : importSources.values()) {
if (metadataSource instanceof FileSource) {
FileSource fileSource = (FileSource)metadataSource;
if (fileSource.isValidSourceForFile(originalName)) {
return fileSource;
}
}
}
return importRecords;
return null;
}

/**
Expand Down
5 changes: 4 additions & 1 deletion ...ain/java/org/dspace/importer/external/service/components/AbstractPlainMetadataSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public abstract class AbstractPlainMetadataSource

/**
* Set the file extensions supported by this metadata service
*
*
* @param supportedExtensions the file extensions (xml,txt,...) supported by this service
*/
public void setSupportedExtensions(List<String> supportedExtensions) {
Expand All @@ -64,6 +64,9 @@ public List<String> getSupportedExtensions() {
@Override
public List<ImportRecord> getRecords(InputStream is) throws FileSourceException {
List<PlainMetadataSourceDto> datas = readData(is);
if (datas == null) {
return List.of();
}
List<ImportRecord> records = new ArrayList<>();
for (PlainMetadataSourceDto item : datas) {
records.add(toRecord(item));
Expand Down
11 changes: 10 additions & 1 deletion dspace-api/src/main/java/org/dspace/importer/external/service/components/FileSource.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public ImportRecord getRecord(InputStream inputStream)

/**
* This method is used to decide if the FileSource manage the file format
*
*
* @param originalName the file file original name
* @return true if the FileSource can parse the file, false otherwise
*/
Expand All @@ -67,4 +67,13 @@ public default boolean isValidSourceForFile(String originalName) {
return false;
}

/**
* This method is used to determine if we can import multiple records at once placed in the same source file.
*
* @return true if allowed to import multiple records in the same file, false otherwise
*/
public default boolean canImportMultipleRecords() {
return false;
}

}
45 changes: 23 additions & 22 deletions ...rver-webapp/src/main/java/org/dspace/app/rest/repository/WorkspaceItemRestRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -317,9 +317,9 @@ public Iterable<WorkspaceItemRest> upload(Context context, HttpServletRequest re
for (MultipartFile mpFile : uploadfiles) {
File file = Utils.getFile(mpFile, "upload-loader", "filedataloader");
try {
ImportRecord record = importService.getRecord(file, mpFile.getOriginalFilename());
if (record != null) {
records.add(record);
List<ImportRecord> recordsFound = importService.getRecords(file, mpFile.getOriginalFilename());
if (recordsFound != null && !recordsFound.isEmpty()) {
records.addAll(recordsFound);
break;
}
} catch (Exception e) {
Expand All @@ -334,11 +334,15 @@ public Iterable<WorkspaceItemRest> upload(Context context, HttpServletRequest re
} catch (Exception e) {
log.error("Error importing metadata", e);
}
WorkspaceItem source = submissionService.
createWorkspaceItem(context, getRequestService().getCurrentRequest());
merge(context, records, source);
result = new ArrayList<>();
result.add(source);
result = new ArrayList<>(records.size());
for (ImportRecord importRecord : records) {
WorkspaceItem source = submissionService.
createWorkspaceItem(context, getRequestService().getCurrentRequest());

merge(context, importRecord, source);

result.add(source);
}

//perform upload of bitstream if there is exact one result and convert workspaceitem to entity rest
if (!result.isEmpty()) {
Expand All @@ -348,18 +352,17 @@ public Iterable<WorkspaceItemRest> upload(Context context, HttpServletRequest re
//load bitstream into bundle ORIGINAL only if there is one result (approximately this is the
// right behaviour for pdf file but not for other bibliographic format e.g. bibtex)
if (result.size() == 1) {
ClassLoader loader = this.getClass().getClassLoader();
for (int i = 0; i < submissionConfig.getNumberOfSteps(); i++) {
SubmissionStepConfig stepConfig = submissionConfig.getStep(i);
ClassLoader loader = this.getClass().getClassLoader();
Class stepClass;
try {
stepClass = loader.loadClass(stepConfig.getProcessingClassName());
Object stepInstance = stepClass.newInstance();
Class<?> stepClass = loader.loadClass(stepConfig.getProcessingClassName());
Object stepInstance = stepClass.getConstructor().newInstance();
if (UploadableStep.class.isAssignableFrom(stepClass)) {
UploadableStep uploadableStep = (UploadableStep) stepInstance;
for (MultipartFile mpFile : uploadfiles) {
ErrorRest err = uploadableStep.upload(context,
submissionService, stepConfig, wi, mpFile);
ErrorRest err =
uploadableStep.upload(context, submissionService, stepConfig, wi, mpFile);
if (err != null) {
errors.add(err);
}
Expand Down Expand Up @@ -449,7 +452,7 @@ private BaseObjectRest<?> findItemRestById(Context context, String itemId) throw
return authorizationRestUtil.getObject(context, objectId);
}

private void merge(Context context, List<ImportRecord> records, WorkspaceItem item) throws SQLException {
private void merge(Context context, ImportRecord record, WorkspaceItem item) throws SQLException {
for (MetadataValue metadataValue : itemService.getMetadata(
item.getItem(), Item.ANY, Item.ANY, Item.ANY, Item.ANY)) {
itemService.clearMetadata(context, item.getItem(),
Expand All @@ -458,13 +461,11 @@ private void merge(Context context, List<ImportRecord> records, WorkspaceItem it
metadataValue.getMetadataField().getQualifier(),
metadataValue.getLanguage());
}
for (ImportRecord record : records) {
if (record != null && record.getValueList() != null) {
for (MetadatumDTO metadataValue : record.getValueList()) {
itemService.addMetadata(context, item.getItem(), metadataValue.getSchema(),
metadataValue.getElement(), metadataValue.getQualifier(), null,
metadataValue.getValue());
}
if (record != null && record.getValueList() != null) {
for (MetadatumDTO metadataValue : record.getValueList()) {
itemService.addMetadata(context, item.getItem(), metadataValue.getSchema(),
metadataValue.getElement(), metadataValue.getQualifier(), null,
metadataValue.getValue());
}
}
}
Expand Down
110 changes: 96 additions & 14 deletions dspace-server-webapp/src/test/java/org/dspace/app/rest/WorkspaceItemRestRepositoryIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,9 +60,9 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.jayway.jsonpath.matchers.JsonPathMatchers;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringEscapeUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.apache.commons.text.StringEscapeUtils;
import org.dspace.app.rest.matcher.CollectionMatcher;
import org.dspace.app.rest.matcher.ItemMatcher;
import org.dspace.app.rest.matcher.MetadataMatcher;
Expand Down Expand Up @@ -2027,27 +2027,109 @@ public void createSingleWorkspaceItemsFromSingleFileWithMultipleEntriesTest() th
Collection col1 = CollectionBuilder.createCollection(context, child1)
.withName("Collection 1")
.withSubmitterGroup(eperson)
.withEntityType("Publication")
.withSubmissionDefinition("traditional")
.build();
Collection col2 = CollectionBuilder.createCollection(context, child1)
.withName("Collection 2")
.withSubmitterGroup(eperson)
.withEntityType("Publication")
.withSubmissionDefinition("traditional")
.build();

InputStream bibtex = getClass().getResourceAsStream("bibtex-test-3-entries.bib");
final MockMultipartFile bibtexFile = new MockMultipartFile("file", "bibtex-test-3-entries.bib",
"application/x-bibtex",
bibtex);
try (InputStream bibtex = getClass().getResourceAsStream("bibtex-test-3-entries.bib")) {
final MockMultipartFile bibtexFile =
new MockMultipartFile(
"file", "bibtex-test-3-entries.bib",
"application/x-bibtex", bibtex
);

context.restoreAuthSystemState();
context.restoreAuthSystemState();

String authToken = getAuthToken(eperson.getEmail(), password);
// create a workspaceitem from a single bibliographic entry file explicitly in the default collection (col1)
getClient(authToken).perform(multipart("/api/submission/workspaceitems")
.file(bibtexFile))
// create should return return a 422 because we don't allow/support bibliographic files
// that have multiple metadata records
.andExpect(status().is(422));
bibtex.close();
String authToken = getAuthToken(eperson.getEmail(), password);
// create a workspaceitem from a single bibliographic entry file explicitly in the default collection (col1)
getClient(authToken)
.perform(
multipart("/api/submission/workspaceitems").file(bibtexFile)
)
// bulk create should return 200, 201 (created) is better for single resource
.andExpect(status().isOk())
.andExpect(
jsonPath(
"$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value",
is("My Article")
)
)
.andExpect(
jsonPath(
"$._embedded.workspaceitems[0]._embedded.collection.id",
is(col1.getID().toString())
)
)
.andExpect(
jsonPath(
"$._embedded.workspaceitems[1].sections.traditionalpageone['dc.title'][0].value",
is("My Article 2")
)
)
.andExpect(
jsonPath(
"$._embedded.workspaceitems[1]._embedded.collection.id",
is(col1.getID().toString())
)
)
.andExpect(
jsonPath(
"$._embedded.workspaceitems[2].sections.traditionalpageone['dc.title'][0].value",
is("My Article 3")
)
)
.andExpect(
jsonPath(
"$._embedded.workspaceitems[2]._embedded.collection.id",
is(col1.getID().toString())
)
)
.andExpect(
jsonPath("$._embedded.workspaceitems[*]._embedded.upload").doesNotExist());
getClient(authToken)
.perform(
multipart("/api/submission/workspaceitems")
.file(bibtexFile)
.param("owningCollection", col2.getID().toString())
)
.andExpect(status().isOk())
.andExpect(
jsonPath(
"$._embedded.workspaceitems[0].sections.traditionalpageone['dc.title'][0].value",
is("My Article")
)
)
.andExpect(
jsonPath(
"$._embedded.workspaceitems[0]._embedded.collection.id",
is(col2.getID().toString())
)
)
.andExpect(
jsonPath(
"$._embedded.workspaceitems[1].sections.traditionalpageone['dc.title'][0].value",
is("My Article 2")
)
)
.andExpect(
jsonPath(
"$._embedded.workspaceitems[1]._embedded.collection.id",
is(col2.getID().toString())
)
)
.andExpect(
jsonPath(
"$._embedded.workspaceitems[2].sections.traditionalpageone['dc.title'][0].value",
is("My Article 3")
)
);
}
}

@Test
Expand Down

0 comments on commit a81fbf6

Please sign in to comment.