errors = passwordValidatorService.validate(password, new Date(), false);
- final JsonArrayBuilder errorArray = Json.createArrayBuilder();
- errors.forEach(errorArray::add);
- return ok(Json.createObjectBuilder().add("password", password).add("errors", errorArray));
- }
-
- @GET
- @Path("/isOrcid")
- public Response isOrcidEnabled() {
- return authSvc.isOrcidEnabled() ? ok("Orcid is enabled") : ok("no orcid for you.");
- }
+ /**
+ * This method is used in API tests, called from UtilIt.java.
+ */
+ @GET
+ @Path("datasets/thumbnailMetadata/{id}")
+ public Response getDatasetThumbnailMetadata(@PathParam("id") Long idSupplied) {
+ Dataset dataset = datasetSvc.find(idSupplied);
+ if (dataset == null) {
+ return error(Response.Status.NOT_FOUND, "Could not find dataset based on id supplied: " + idSupplied + ".");
+ }
+ JsonObjectBuilder data = Json.createObjectBuilder();
+ DatasetThumbnail datasetThumbnail = dataset.getDatasetThumbnail(ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE);
+ data.add("isUseGenericThumbnail", dataset.isUseGenericThumbnail());
+ data.add("datasetLogoPresent", DatasetUtil.isDatasetLogoPresent(dataset, ImageThumbConverter.DEFAULT_CARDIMAGE_SIZE));
+ if (datasetThumbnail != null) {
+ data.add("datasetThumbnailBase64image", datasetThumbnail.getBase64image());
+ DataFile dataFile = datasetThumbnail.getDataFile();
+ if (dataFile != null) {
+ /**
+ * @todo Change this from a String to a long.
+ */
+ data.add("dataFileId", dataFile.getId().toString());
+ }
+ }
+ return ok(data);
+ }
+
+ /**
+ * validatePassword
+ *
+ * Validate a password with an API call
+ *
+ * @param password
+ * The password
+ * @return A response with the validation result.
+ */
+ @Path("validatePassword")
+ @POST
+ public Response validatePassword(String password) {
+
+ final List errors = passwordValidatorService.validate(password, new Date(), false);
+ final JsonArrayBuilder errorArray = Json.createArrayBuilder();
+ errors.forEach(errorArray::add);
+ return ok(Json.createObjectBuilder().add("password", password).add("errors", errorArray));
+ }
+
+ @GET
+ @Path("/isOrcid")
+ public Response isOrcidEnabled() {
+ return authSvc.isOrcidEnabled() ? ok("Orcid is enabled") : ok("no orcid for you.");
+ }
@POST
- @AuthRequired
+ @AuthRequired
@Path("{id}/reregisterHDLToPID")
public Response reregisterHdlToPID(@Context ContainerRequestContext crc, @PathParam("id") String id) {
logger.info("Starting to reregister " + id + " Dataset Id. (from hdl to doi)" + new Date());
@@ -1825,7 +1805,7 @@ public Response updateHashValues(@Context ContainerRequestContext crc, @PathPara
}
@POST
- @AuthRequired
+ @AuthRequired
@Path("/computeDataFileHashValue/{fileId}/algorithm/{alg}")
public Response computeDataFileHashValue(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId, @PathParam("alg") String alg) {
@@ -1887,7 +1867,7 @@ public Response computeDataFileHashValue(@Context ContainerRequestContext crc, @
}
@POST
- @AuthRequired
+ @AuthRequired
@Path("/validateDataFileHashValue/{fileId}")
public Response validateDataFileHashValue(@Context ContainerRequestContext crc, @PathParam("fileId") String fileId) {
@@ -1954,7 +1934,7 @@ public Response validateDataFileHashValue(@Context ContainerRequestContext crc,
}
@POST
- @AuthRequired
+ @AuthRequired
@Path("/submitDatasetVersionToArchive/{id}/{version}")
public Response submitDatasetVersionToArchive(@Context ContainerRequestContext crc, @PathParam("id") String dsid,
@PathParam("version") String versionNumber) {
@@ -2027,7 +2007,7 @@ public void run() {
* @return
*/
@POST
- @AuthRequired
+ @AuthRequired
@Path("/archiveAllUnarchivedDatasetVersions")
public Response archiveAllUnarchivedDatasetVersions(@Context ContainerRequestContext crc, @QueryParam("listonly") boolean listonly, @QueryParam("limit") Integer limit, @QueryParam("latestonly") boolean latestonly) {
@@ -2126,7 +2106,7 @@ public Response clearMetricsCacheByName(@PathParam("name") String name) {
}
@GET
- @AuthRequired
+ @AuthRequired
@Path("/dataverse/{alias}/addRoleAssignmentsToChildren")
public Response addRoleAssignementsToChildren(@Context ContainerRequestContext crc, @PathParam("alias") String alias) throws WrappedResponse {
Dataverse owner = dataverseSvc.findByAlias(alias);
@@ -2157,90 +2137,90 @@ public Response addRoleAssignementsToChildren(@Context ContainerRequestContext c
}
@GET
- @AuthRequired
+ @AuthRequired
@Path("/dataverse/{alias}/storageDriver")
public Response getStorageDriver(@Context ContainerRequestContext crc, @PathParam("alias") String alias) throws WrappedResponse {
- Dataverse dataverse = dataverseSvc.findByAlias(alias);
- if (dataverse == null) {
- return error(Response.Status.NOT_FOUND, "Could not find dataverse based on alias supplied: " + alias + ".");
- }
- try {
- AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
- if (!user.isSuperuser()) {
- return error(Response.Status.FORBIDDEN, "Superusers only.");
- }
- } catch (WrappedResponse wr) {
- return wr.getResponse();
- }
- //Note that this returns what's set directly on this dataverse. If null/DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER, the user would have to recurse the chain of parents to find the effective storageDriver
- return ok(dataverse.getStorageDriverId());
+ Dataverse dataverse = dataverseSvc.findByAlias(alias);
+ if (dataverse == null) {
+ return error(Response.Status.NOT_FOUND, "Could not find dataverse based on alias supplied: " + alias + ".");
+ }
+ try {
+ AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
+ if (!user.isSuperuser()) {
+ return error(Response.Status.FORBIDDEN, "Superusers only.");
+ }
+ } catch (WrappedResponse wr) {
+ return wr.getResponse();
+ }
+ //Note that this returns what's set directly on this dataverse. If null/DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER, the user would have to recurse the chain of parents to find the effective storageDriver
+ return ok(dataverse.getStorageDriverId());
}
@PUT
- @AuthRequired
+ @AuthRequired
@Path("/dataverse/{alias}/storageDriver")
public Response setStorageDriver(@Context ContainerRequestContext crc, @PathParam("alias") String alias, String label) throws WrappedResponse {
- Dataverse dataverse = dataverseSvc.findByAlias(alias);
- if (dataverse == null) {
- return error(Response.Status.NOT_FOUND, "Could not find dataverse based on alias supplied: " + alias + ".");
- }
- try {
- AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
- if (!user.isSuperuser()) {
- return error(Response.Status.FORBIDDEN, "Superusers only.");
- }
- } catch (WrappedResponse wr) {
- return wr.getResponse();
- }
- for (Entry store: DataAccess.getStorageDriverLabels().entrySet()) {
- if(store.getKey().equals(label)) {
- dataverse.setStorageDriverId(store.getValue());
- return ok("Storage set to: " + store.getKey() + "/" + store.getValue());
- }
- }
- return error(Response.Status.BAD_REQUEST,
- "No Storage Driver found for : " + label);
+ Dataverse dataverse = dataverseSvc.findByAlias(alias);
+ if (dataverse == null) {
+ return error(Response.Status.NOT_FOUND, "Could not find dataverse based on alias supplied: " + alias + ".");
+ }
+ try {
+ AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
+ if (!user.isSuperuser()) {
+ return error(Response.Status.FORBIDDEN, "Superusers only.");
+ }
+ } catch (WrappedResponse wr) {
+ return wr.getResponse();
+ }
+ for (Entry store: DataAccess.getStorageDriverLabels().entrySet()) {
+ if(store.getKey().equals(label)) {
+ dataverse.setStorageDriverId(store.getValue());
+ return ok("Storage set to: " + store.getKey() + "/" + store.getValue());
+ }
+ }
+ return error(Response.Status.BAD_REQUEST,
+ "No Storage Driver found for : " + label);
}
@DELETE
- @AuthRequired
+ @AuthRequired
@Path("/dataverse/{alias}/storageDriver")
public Response resetStorageDriver(@Context ContainerRequestContext crc, @PathParam("alias") String alias) throws WrappedResponse {
- Dataverse dataverse = dataverseSvc.findByAlias(alias);
- if (dataverse == null) {
- return error(Response.Status.NOT_FOUND, "Could not find dataverse based on alias supplied: " + alias + ".");
- }
- try {
- AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
- if (!user.isSuperuser()) {
- return error(Response.Status.FORBIDDEN, "Superusers only.");
- }
- } catch (WrappedResponse wr) {
- return wr.getResponse();
- }
- dataverse.setStorageDriverId("");
- return ok("Storage reset to default: " + DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
+ Dataverse dataverse = dataverseSvc.findByAlias(alias);
+ if (dataverse == null) {
+ return error(Response.Status.NOT_FOUND, "Could not find dataverse based on alias supplied: " + alias + ".");
+ }
+ try {
+ AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
+ if (!user.isSuperuser()) {
+ return error(Response.Status.FORBIDDEN, "Superusers only.");
+ }
+ } catch (WrappedResponse wr) {
+ return wr.getResponse();
+ }
+ dataverse.setStorageDriverId("");
+ return ok("Storage reset to default: " + DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER);
}
@GET
- @AuthRequired
+ @AuthRequired
@Path("/dataverse/storageDrivers")
public Response listStorageDrivers(@Context ContainerRequestContext crc) throws WrappedResponse {
- try {
- AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
- if (!user.isSuperuser()) {
- return error(Response.Status.FORBIDDEN, "Superusers only.");
- }
- } catch (WrappedResponse wr) {
- return wr.getResponse();
- }
- JsonObjectBuilder bld = jsonObjectBuilder();
- DataAccess.getStorageDriverLabels().entrySet().forEach(s -> bld.add(s.getKey(), s.getValue()));
- return ok(bld);
+ try {
+ AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
+ if (!user.isSuperuser()) {
+ return error(Response.Status.FORBIDDEN, "Superusers only.");
+ }
+ } catch (WrappedResponse wr) {
+ return wr.getResponse();
+ }
+ JsonObjectBuilder bld = jsonObjectBuilder();
+ DataAccess.getStorageDriverLabels().entrySet().forEach(s -> bld.add(s.getKey(), s.getValue()));
+ return ok(bld);
}
@GET
- @AuthRequired
+ @AuthRequired
@Path("/dataverse/{alias}/curationLabelSet")
public Response getCurationLabelSet(@Context ContainerRequestContext crc, @PathParam("alias") String alias) throws WrappedResponse {
Dataverse dataverse = dataverseSvc.findByAlias(alias);
@@ -2262,7 +2242,7 @@ public Response getCurationLabelSet(@Context ContainerRequestContext crc, @PathP
}
@PUT
- @AuthRequired
+ @AuthRequired
@Path("/dataverse/{alias}/curationLabelSet")
public Response setCurationLabelSet(@Context ContainerRequestContext crc, @PathParam("alias") String alias, @QueryParam("name") String name) throws WrappedResponse {
Dataverse dataverse = dataverseSvc.findByAlias(alias);
@@ -2293,7 +2273,7 @@ public Response setCurationLabelSet(@Context ContainerRequestContext crc, @PathP
}
@DELETE
- @AuthRequired
+ @AuthRequired
@Path("/dataverse/{alias}/curationLabelSet")
public Response resetCurationLabelSet(@Context ContainerRequestContext crc, @PathParam("alias") String alias) throws WrappedResponse {
Dataverse dataverse = dataverseSvc.findByAlias(alias);
@@ -2313,7 +2293,7 @@ public Response resetCurationLabelSet(@Context ContainerRequestContext crc, @Pat
}
@GET
- @AuthRequired
+ @AuthRequired
@Path("/dataverse/curationLabelSets")
public Response listCurationLabelSets(@Context ContainerRequestContext crc) throws WrappedResponse {
try {
@@ -2423,7 +2403,7 @@ public Response getBannerMessages(@PathParam("id") Long id) throws WrappedRespon
}
@POST
- @AuthRequired
+ @AuthRequired
@Consumes("application/json")
@Path("/requestSignedUrl")
public Response getSignedUrl(@Context ContainerRequestContext crc, JsonObject urlInfo) {
@@ -2541,4 +2521,160 @@ public Response getFeatureFlag(@PathParam("flag") String flagIn) {
}
}
+ @GET
+ @AuthRequired
+ @Path("/datafiles/auditFiles")
+ public Response getAuditFiles(@Context ContainerRequestContext crc,
+ @QueryParam("firstId") Long firstId, @QueryParam("lastId") Long lastId,
+ @QueryParam("datasetIdentifierList") String datasetIdentifierList) throws WrappedResponse {
+ try {
+ AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
+ if (!user.isSuperuser()) {
+ return error(Response.Status.FORBIDDEN, "Superusers only.");
+ }
+ } catch (WrappedResponse wr) {
+ return wr.getResponse();
+ }
+
+ int datasetsChecked = 0;
+ long startId = (firstId == null ? 0 : firstId);
+ long endId = (lastId == null ? Long.MAX_VALUE : lastId);
+
+ List datasetIdentifiers;
+ if (datasetIdentifierList == null || datasetIdentifierList.isEmpty()) {
+ datasetIdentifiers = Collections.emptyList();
+ } else {
+ startId = 0;
+ endId = Long.MAX_VALUE;
+ datasetIdentifiers = List.of(datasetIdentifierList.split(","));
+ }
+ if (endId < startId) {
+ return badRequest("Invalid Parameters: lastId must be equal to or greater than firstId");
+ }
+
+ NullSafeJsonBuilder jsonObjectBuilder = NullSafeJsonBuilder.jsonObjectBuilder();
+ JsonArrayBuilder jsonDatasetsArrayBuilder = Json.createArrayBuilder();
+ JsonArrayBuilder jsonFailuresArrayBuilder = Json.createArrayBuilder();
+
+ if (startId > 0) {
+ jsonObjectBuilder.add("firstId", startId);
+ }
+ if (endId < Long.MAX_VALUE) {
+ jsonObjectBuilder.add("lastId", endId);
+ }
+
+ // compile the list of ids to process
+ List datasetIds;
+ if (datasetIdentifiers.isEmpty()) {
+ datasetIds = datasetService.findAllLocalDatasetIds();
+ } else {
+ datasetIds = new ArrayList<>(datasetIdentifiers.size());
+ JsonArrayBuilder jab = Json.createArrayBuilder();
+ datasetIdentifiers.forEach(id -> {
+ String dId = id.trim();
+ jab.add(dId);
+ Dataset d = datasetService.findByGlobalId(dId);
+ if (d != null) {
+ datasetIds.add(d.getId());
+ } else {
+ NullSafeJsonBuilder job = NullSafeJsonBuilder.jsonObjectBuilder();
+ job.add("datasetIdentifier",dId);
+ job.add("reason","Not Found");
+ jsonFailuresArrayBuilder.add(job);
+ }
+ });
+ jsonObjectBuilder.add("datasetIdentifierList", jab);
+ }
+
+ for (Long datasetId : datasetIds) {
+ if (datasetId < startId) {
+ continue;
+ } else if (datasetId > endId) {
+ break;
+ }
+ Dataset dataset;
+ try {
+ dataset = findDatasetOrDie(String.valueOf(datasetId));
+ datasetsChecked++;
+ } catch (WrappedResponse e) {
+ NullSafeJsonBuilder job = NullSafeJsonBuilder.jsonObjectBuilder();
+ job.add("datasetId", datasetId);
+ job.add("reason", e.getMessage());
+ jsonFailuresArrayBuilder.add(job);
+ continue;
+ }
+
+ List missingFiles = new ArrayList<>();
+ List missingFileMetadata = new ArrayList<>();
+ try {
+ Predicate filter = s -> true;
+ StorageIO datasetIO = DataAccess.getStorageIO(dataset);
+ final List result = datasetIO.cleanUp(filter, true);
+ // add files that are in dataset files but not in cleanup result or DataFiles with missing FileMetadata
+ dataset.getFiles().forEach(df -> {
+ try {
+ StorageIO datafileIO = df.getStorageIO();
+ String storageId = df.getStorageIdentifier();
+ FileMetadata fm = df.getFileMetadata();
+ if (!datafileIO.exists()) {
+ missingFiles.add(storageId + "," + (fm != null ?
+ (fm.getDirectoryLabel() != null || !fm.getDirectoryLabel().isEmpty() ? "directoryLabel,"+fm.getDirectoryLabel()+"," : "")
+ +"label,"+fm.getLabel() : "type,"+df.getContentType()));
+ }
+ if (fm == null) {
+ missingFileMetadata.add(storageId + ",dataFileId," + df.getId());
+ }
+ } catch (IOException e) {
+ NullSafeJsonBuilder job = NullSafeJsonBuilder.jsonObjectBuilder();
+ job.add("dataFileId", df.getId());
+ job.add("reason", e.getMessage());
+ jsonFailuresArrayBuilder.add(job);
+ }
+ });
+ } catch (IOException e) {
+ NullSafeJsonBuilder job = NullSafeJsonBuilder.jsonObjectBuilder();
+ job.add("datasetId", datasetId);
+ job.add("reason", e.getMessage());
+ jsonFailuresArrayBuilder.add(job);
+ }
+
+ JsonObjectBuilder job = Json.createObjectBuilder();
+ if (!missingFiles.isEmpty() || !missingFileMetadata.isEmpty()) {
+ job.add("id", dataset.getId());
+ job.add("pid", dataset.getProtocol() + ":" + dataset.getAuthority() + "/" + dataset.getIdentifier());
+ job.add("persistentURL", dataset.getPersistentURL());
+ if (!missingFileMetadata.isEmpty()) {
+ JsonArrayBuilder jabMissingFileMetadata = Json.createArrayBuilder();
+ missingFileMetadata.forEach(mm -> {
+ String[] missingMetadata = mm.split(",");
+ NullSafeJsonBuilder jobj = NullSafeJsonBuilder.jsonObjectBuilder()
+ .add("storageIdentifier", missingMetadata[0])
+ .add(missingMetadata[1], missingMetadata[2]);
+ jabMissingFileMetadata.add(jobj);
+ });
+ job.add("missingFileMetadata", jabMissingFileMetadata);
+ }
+ if (!missingFiles.isEmpty()) {
+ JsonArrayBuilder jabMissingFiles = Json.createArrayBuilder();
+ missingFiles.forEach(mf -> {
+ String[] missingFile = mf.split(",");
+ NullSafeJsonBuilder jobj = NullSafeJsonBuilder.jsonObjectBuilder()
+ .add("storageIdentifier", missingFile[0]);
+ for (int i = 2; i < missingFile.length; i+=2) {
+ jobj.add(missingFile[i-1], missingFile[i]);
+ }
+ jabMissingFiles.add(jobj);
+ });
+ job.add("missingFiles", jabMissingFiles);
+ }
+ jsonDatasetsArrayBuilder.add(job);
+ }
+ }
+
+ jsonObjectBuilder.add("datasetsChecked", datasetsChecked);
+ jsonObjectBuilder.add("datasets", jsonDatasetsArrayBuilder);
+ jsonObjectBuilder.add("failures", jsonFailuresArrayBuilder);
+
+ return ok(jsonObjectBuilder);
+ }
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java
index 01c51dc2b4c..907295ad848 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/DatasetFieldServiceApi.java
@@ -126,7 +126,7 @@ public Response getByName(@PathParam("name") String name) {
String solrFieldSearchable = dsf.getSolrField().getNameSearchable();
String solrFieldFacetable = dsf.getSolrField().getNameFacetable();
String metadataBlock = dsf.getMetadataBlock().getName();
- String uri=dsf.getUri();
+ String uri = dsf.getUri();
boolean hasParent = dsf.isHasParent();
boolean allowsMultiples = dsf.isAllowMultiples();
boolean isRequired = dsf.isRequired();
@@ -243,7 +243,9 @@ public Response loadDatasetFields(File file) {
br = new BufferedReader(new FileReader("/" + file));
while ((line = br.readLine()) != null) {
lineNumber++;
- values = line.split(splitBy);
+ values = Arrays.stream(line.split(splitBy))
+ .map(String::trim)
+ .toArray(String[]::new);
if (values[0].startsWith("#")) { // Header row
switch (values[0]) {
case "#metadataBlock":
@@ -326,7 +328,7 @@ public Response loadDatasetFields(File file) {
*/
public String getGeneralErrorMessage(HeaderType header, int lineNumber, String message) {
List arguments = new ArrayList<>();
- arguments.add(header.name());
+ arguments.add(header != null ? header.name() : "unknown");
arguments.add(String.valueOf(lineNumber));
arguments.add(message);
return BundleUtil.getStringFromBundle("api.admin.datasetfield.load.GeneralErrorMessage", arguments);
@@ -334,9 +336,9 @@ public String getGeneralErrorMessage(HeaderType header, int lineNumber, String m
/**
* Turn ArrayIndexOutOfBoundsException into an informative error message
- * @param lineNumber
* @param header
- * @param e
+ * @param lineNumber
+ * @param wrongIndex
* @return
*/
public String getArrayIndexOutOfBoundMessage(HeaderType header,
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
index 369a22fe8d7..2ec10816acc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java
@@ -2171,8 +2171,32 @@ public Response getAssignments(@Context ContainerRequestContext crc, @PathParam(
@GET
@AuthRequired
+ @Deprecated(forRemoval = true, since = "2024-10-17")
@Path("{id}/privateUrl")
public Response getPrivateUrlData(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied) {
+ return getPreviewUrlData(crc, idSupplied);
+ }
+
+ @POST
+ @AuthRequired
+ @Deprecated(forRemoval = true, since = "2024-10-17")
+ @Path("{id}/privateUrl")
+ public Response createPrivateUrl(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied, @DefaultValue("false") @QueryParam("anonymizedAccess") boolean anonymizedAccess) {
+ return createPreviewUrl(crc, idSupplied, anonymizedAccess);
+ }
+
+ @DELETE
+ @AuthRequired
+ @Deprecated(forRemoval = true, since = "2024-10-17")
+ @Path("{id}/privateUrl")
+ public Response deletePrivateUrl(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied) {
+ return deletePreviewUrl(crc, idSupplied);
+ }
+
+ @GET
+ @AuthRequired
+ @Path("{id}/previewUrl")
+ public Response getPreviewUrlData(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied) {
return response( req -> {
PrivateUrl privateUrl = execCommand(new GetPrivateUrlCommand(req, findDatasetOrDie(idSupplied)));
return (privateUrl != null) ? ok(json(privateUrl))
@@ -2182,8 +2206,8 @@ public Response getPrivateUrlData(@Context ContainerRequestContext crc, @PathPar
@POST
@AuthRequired
- @Path("{id}/privateUrl")
- public Response createPrivateUrl(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied,@DefaultValue("false") @QueryParam ("anonymizedAccess") boolean anonymizedAccess) {
+ @Path("{id}/previewUrl")
+ public Response createPreviewUrl(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied,@DefaultValue("false") @QueryParam ("anonymizedAccess") boolean anonymizedAccess) {
if(anonymizedAccess && settingsSvc.getValueForKey(SettingsServiceBean.Key.AnonymizedFieldTypeNames)==null) {
throw new NotAcceptableException("Anonymized Access not enabled");
}
@@ -2194,8 +2218,8 @@ public Response createPrivateUrl(@Context ContainerRequestContext crc, @PathPara
@DELETE
@AuthRequired
- @Path("{id}/privateUrl")
- public Response deletePrivateUrl(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied) {
+ @Path("{id}/previewUrl")
+ public Response deletePreviewUrl(@Context ContainerRequestContext crc, @PathParam("id") String idSupplied) {
return response( req -> {
Dataset dataset = findDatasetOrDie(idSupplied);
PrivateUrl privateUrl = execCommand(new GetPrivateUrlCommand(req, dataset));
@@ -2208,6 +2232,7 @@ public Response deletePrivateUrl(@Context ContainerRequestContext crc, @PathPara
}, getRequestUser(crc));
}
+
@GET
@AuthRequired
@Path("{id}/thumbnail/candidates")
@@ -2992,6 +3017,26 @@ public Response cleanStorage(@Context ContainerRequestContext crc, @PathParam("i
}
+ @GET
+ @AuthRequired
+ @Path("{id}/versions/{versionId1}/compare/{versionId2}")
+ public Response getCompareVersions(@Context ContainerRequestContext crc, @PathParam("id") String id,
+ @PathParam("versionId1") String versionId1,
+ @PathParam("versionId2") String versionId2,
+ @Context UriInfo uriInfo, @Context HttpHeaders headers) {
+ try {
+ DataverseRequest req = createDataverseRequest(getRequestUser(crc));
+ DatasetVersion dsv1 = getDatasetVersionOrDie(req, versionId1, findDatasetOrDie(id), uriInfo, headers);
+ DatasetVersion dsv2 = getDatasetVersionOrDie(req, versionId2, findDatasetOrDie(id), uriInfo, headers);
+ if (dsv1.getCreateTime().getTime() > dsv2.getCreateTime().getTime()) {
+ return error(BAD_REQUEST, BundleUtil.getStringFromBundle("dataset.version.compare.incorrect.order"));
+ }
+ return ok(DatasetVersion.compareVersions(dsv1, dsv2));
+ } catch (WrappedResponse wr) {
+ return wr.getResponse();
+ }
+ }
+
private static Set getDatasetFilenames(Dataset dataset) {
Set files = new HashSet<>();
for (DataFile dataFile: dataset.getFiles()) {
@@ -4833,6 +4878,33 @@ public Response getPrivateUrlDatasetVersion(@PathParam("privateUrlToken") String
}
return ok(responseJson);
}
+
+ @GET
+ @Path("previewUrlDatasetVersion/{previewUrlToken}")
+ public Response getPreviewUrlDatasetVersion(@PathParam("previewUrlToken") String previewUrlToken, @QueryParam("returnOwners") boolean returnOwners) {
+ PrivateUrlUser privateUrlUser = privateUrlService.getPrivateUrlUserFromToken(previewUrlToken);
+ if (privateUrlUser == null) {
+ return notFound("Private URL user not found");
+ }
+ boolean isAnonymizedAccess = privateUrlUser.hasAnonymizedAccess();
+ String anonymizedFieldTypeNames = settingsSvc.getValueForKey(SettingsServiceBean.Key.AnonymizedFieldTypeNames);
+ if(isAnonymizedAccess && anonymizedFieldTypeNames == null) {
+ throw new NotAcceptableException("Anonymized Access not enabled");
+ }
+ DatasetVersion dsv = privateUrlService.getDraftDatasetVersionFromToken(previewUrlToken);
+ if (dsv == null || dsv.getId() == null) {
+ return notFound("Dataset version not found");
+ }
+ JsonObjectBuilder responseJson;
+ if (isAnonymizedAccess) {
+ List anonymizedFieldTypeNamesList = new ArrayList<>(Arrays.asList(anonymizedFieldTypeNames.split(",\\s")));
+ responseJson = json(dsv, anonymizedFieldTypeNamesList, true, returnOwners);
+ } else {
+ responseJson = json(dsv, null, true, returnOwners);
+ }
+ return ok(responseJson);
+ }
+
@GET
@Path("privateUrlDatasetVersion/{privateUrlToken}/citation")
@@ -4845,6 +4917,18 @@ public Response getPrivateUrlDatasetVersionCitation(@PathParam("privateUrlToken"
return (dsv == null || dsv.getId() == null) ? notFound("Dataset version not found")
: ok(dsv.getCitation(true, privateUrlUser.hasAnonymizedAccess()));
}
+
+ @GET
+ @Path("previewUrlDatasetVersion/{previewUrlToken}/citation")
+ public Response getPreviewUrlDatasetVersionCitation(@PathParam("previewUrlToken") String previewUrlToken) {
+ PrivateUrlUser privateUrlUser = privateUrlService.getPrivateUrlUserFromToken(previewUrlToken);
+ if (privateUrlUser == null) {
+ return notFound("Private URL user not found");
+ }
+ DatasetVersion dsv = privateUrlService.getDraftDatasetVersionFromToken(previewUrlToken);
+ return (dsv == null || dsv.getId() == null) ? notFound("Dataset version not found")
+ : ok(dsv.getCitation(true, privateUrlUser.hasAnonymizedAccess()));
+ }
@GET
@AuthRequired
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
index 0ee146ed99b..f864a5a9d1c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Dataverses.java
@@ -3,12 +3,9 @@
import edu.harvard.iq.dataverse.*;
import edu.harvard.iq.dataverse.api.auth.AuthRequired;
import edu.harvard.iq.dataverse.api.datadeposit.SwordServiceBean;
-import edu.harvard.iq.dataverse.api.dto.DataverseMetadataBlockFacetDTO;
+import edu.harvard.iq.dataverse.api.dto.*;
import edu.harvard.iq.dataverse.authorization.DataverseRole;
-import edu.harvard.iq.dataverse.api.dto.ExplicitGroupDTO;
-import edu.harvard.iq.dataverse.api.dto.RoleAssignmentDTO;
-import edu.harvard.iq.dataverse.api.dto.RoleDTO;
import edu.harvard.iq.dataverse.api.imports.ImportException;
import edu.harvard.iq.dataverse.api.imports.ImportServiceBean;
import edu.harvard.iq.dataverse.authorization.Permission;
@@ -127,75 +124,158 @@ public Response addRoot(@Context ContainerRequestContext crc, String body) {
@Path("{identifier}")
public Response addDataverse(@Context ContainerRequestContext crc, String body, @PathParam("identifier") String parentIdtf) {
Dataverse newDataverse;
- JsonObject newDataverseJson;
try {
- newDataverseJson = JsonUtil.getJsonObject(body);
- newDataverse = jsonParser().parseDataverse(newDataverseJson);
+ newDataverse = parseAndValidateAddDataverseRequestBody(body);
} catch (JsonParsingException jpe) {
- logger.log(Level.SEVERE, "Json: {0}", body);
return error(Status.BAD_REQUEST, MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.create.error.jsonparse"), jpe.getMessage()));
} catch (JsonParseException ex) {
- logger.log(Level.SEVERE, "Error parsing dataverse from json: " + ex.getMessage(), ex);
return error(Status.BAD_REQUEST, MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.create.error.jsonparsetodataverse"), ex.getMessage()));
}
try {
- JsonObject metadataBlocksJson = newDataverseJson.getJsonObject("metadataBlocks");
- List inputLevels = null;
- List metadataBlocks = null;
- List facetList = null;
- if (metadataBlocksJson != null) {
- JsonArray inputLevelsArray = metadataBlocksJson.getJsonArray("inputLevels");
- inputLevels = inputLevelsArray != null ? parseInputLevels(inputLevelsArray, newDataverse) : null;
-
- JsonArray metadataBlockNamesArray = metadataBlocksJson.getJsonArray("metadataBlockNames");
- metadataBlocks = metadataBlockNamesArray != null ? parseNewDataverseMetadataBlocks(metadataBlockNamesArray) : null;
-
- JsonArray facetIdsArray = metadataBlocksJson.getJsonArray("facetIds");
- facetList = facetIdsArray != null ? parseFacets(facetIdsArray) : null;
- }
+ List inputLevels = parseInputLevels(body, newDataverse);
+ List metadataBlocks = parseMetadataBlocks(body);
+ List facets = parseFacets(body);
if (!parentIdtf.isEmpty()) {
Dataverse owner = findDataverseOrDie(parentIdtf);
newDataverse.setOwner(owner);
}
- // set the dataverse - contact relationship in the contacts
- for (DataverseContact dc : newDataverse.getDataverseContacts()) {
- dc.setDataverse(newDataverse);
- }
-
AuthenticatedUser u = getRequestAuthenticatedUserOrDie(crc);
- newDataverse = execCommand(new CreateDataverseCommand(newDataverse, createDataverseRequest(u), facetList, inputLevels, metadataBlocks));
+ newDataverse = execCommand(new CreateDataverseCommand(newDataverse, createDataverseRequest(u), facets, inputLevels, metadataBlocks));
return created("/dataverses/" + newDataverse.getAlias(), json(newDataverse));
- } catch (WrappedResponse ww) {
-
- String error = ConstraintViolationUtil.getErrorStringForConstraintViolations(ww.getCause());
- if (!error.isEmpty()) {
- logger.log(Level.INFO, error);
- return ww.refineResponse(error);
- }
- return ww.getResponse();
+ } catch (WrappedResponse ww) {
+ return handleWrappedResponse(ww);
} catch (EJBException ex) {
- Throwable cause = ex;
- StringBuilder sb = new StringBuilder();
- sb.append("Error creating dataverse.");
- while (cause.getCause() != null) {
- cause = cause.getCause();
- if (cause instanceof ConstraintViolationException) {
- sb.append(ConstraintViolationUtil.getErrorStringForConstraintViolations(cause));
- }
- }
- logger.log(Level.SEVERE, sb.toString());
- return error(Response.Status.INTERNAL_SERVER_ERROR, "Error creating dataverse: " + sb.toString());
+ return handleEJBException(ex, "Error creating dataverse.");
} catch (Exception ex) {
logger.log(Level.SEVERE, "Error creating dataverse", ex);
return error(Response.Status.INTERNAL_SERVER_ERROR, "Error creating dataverse: " + ex.getMessage());
+ }
+ }
+
+ private Dataverse parseAndValidateAddDataverseRequestBody(String body) throws JsonParsingException, JsonParseException {
+ try {
+ JsonObject addDataverseJson = JsonUtil.getJsonObject(body);
+ return jsonParser().parseDataverse(addDataverseJson);
+ } catch (JsonParsingException jpe) {
+ logger.log(Level.SEVERE, "Json: {0}", body);
+ throw jpe;
+ } catch (JsonParseException ex) {
+ logger.log(Level.SEVERE, "Error parsing dataverse from json: " + ex.getMessage(), ex);
+ throw ex;
+ }
+ }
+
+ @PUT
+ @AuthRequired
+ @Path("{identifier}")
+ public Response updateDataverse(@Context ContainerRequestContext crc, String body, @PathParam("identifier") String identifier) {
+ Dataverse dataverse;
+ try {
+ dataverse = findDataverseOrDie(identifier);
+ } catch (WrappedResponse e) {
+ return e.getResponse();
+ }
+
+ DataverseDTO updatedDataverseDTO;
+ try {
+ updatedDataverseDTO = parseAndValidateUpdateDataverseRequestBody(body);
+ } catch (JsonParsingException jpe) {
+ return error(Status.BAD_REQUEST, MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.create.error.jsonparse"), jpe.getMessage()));
+ } catch (JsonParseException ex) {
+ return error(Status.BAD_REQUEST, MessageFormat.format(BundleUtil.getStringFromBundle("dataverse.create.error.jsonparsetodataverse"), ex.getMessage()));
+ }
+
+ try {
+ List inputLevels = parseInputLevels(body, dataverse);
+ List metadataBlocks = parseMetadataBlocks(body);
+ List facets = parseFacets(body);
+
+ AuthenticatedUser u = getRequestAuthenticatedUserOrDie(crc);
+ dataverse = execCommand(new UpdateDataverseCommand(dataverse, facets, null, createDataverseRequest(u), inputLevels, metadataBlocks, updatedDataverseDTO, true));
+ return ok(json(dataverse));
+ } catch (WrappedResponse ww) {
+ return handleWrappedResponse(ww);
+ } catch (EJBException ex) {
+ return handleEJBException(ex, "Error updating dataverse.");
+ } catch (Exception ex) {
+ logger.log(Level.SEVERE, "Error updating dataverse", ex);
+ return error(Response.Status.INTERNAL_SERVER_ERROR, "Error updating dataverse: " + ex.getMessage());
}
}
+ private DataverseDTO parseAndValidateUpdateDataverseRequestBody(String body) throws JsonParsingException, JsonParseException {
+ try {
+ JsonObject updateDataverseJson = JsonUtil.getJsonObject(body);
+ return jsonParser().parseDataverseDTO(updateDataverseJson);
+ } catch (JsonParsingException jpe) {
+ logger.log(Level.SEVERE, "Json: {0}", body);
+ throw jpe;
+ } catch (JsonParseException ex) {
+ logger.log(Level.SEVERE, "Error parsing DataverseDTO from json: " + ex.getMessage(), ex);
+ throw ex;
+ }
+ }
+
+ private List parseInputLevels(String body, Dataverse dataverse) throws WrappedResponse {
+ JsonObject metadataBlocksJson = getMetadataBlocksJson(body);
+ if (metadataBlocksJson == null) {
+ return null;
+ }
+ JsonArray inputLevelsArray = metadataBlocksJson.getJsonArray("inputLevels");
+ return inputLevelsArray != null ? parseInputLevels(inputLevelsArray, dataverse) : null;
+ }
+
+ private List parseMetadataBlocks(String body) throws WrappedResponse {
+ JsonObject metadataBlocksJson = getMetadataBlocksJson(body);
+ if (metadataBlocksJson == null) {
+ return null;
+ }
+ JsonArray metadataBlocksArray = metadataBlocksJson.getJsonArray("metadataBlockNames");
+ return metadataBlocksArray != null ? parseNewDataverseMetadataBlocks(metadataBlocksArray) : null;
+ }
+
+ private List parseFacets(String body) throws WrappedResponse {
+ JsonObject metadataBlocksJson = getMetadataBlocksJson(body);
+ if (metadataBlocksJson == null) {
+ return null;
+ }
+ JsonArray facetsArray = metadataBlocksJson.getJsonArray("facetIds");
+ return facetsArray != null ? parseFacets(facetsArray) : null;
+ }
+
+ private JsonObject getMetadataBlocksJson(String body) {
+ JsonObject dataverseJson = JsonUtil.getJsonObject(body);
+ return dataverseJson.getJsonObject("metadataBlocks");
+ }
+
+ private Response handleWrappedResponse(WrappedResponse ww) {
+ String error = ConstraintViolationUtil.getErrorStringForConstraintViolations(ww.getCause());
+ if (!error.isEmpty()) {
+ logger.log(Level.INFO, error);
+ return ww.refineResponse(error);
+ }
+ return ww.getResponse();
+ }
+
+ private Response handleEJBException(EJBException ex, String action) {
+ Throwable cause = ex;
+ StringBuilder sb = new StringBuilder();
+ sb.append(action);
+ while (cause.getCause() != null) {
+ cause = cause.getCause();
+ if (cause instanceof ConstraintViolationException) {
+ sb.append(ConstraintViolationUtil.getErrorStringForConstraintViolations(cause));
+ }
+ }
+ logger.log(Level.SEVERE, sb.toString());
+ return error(Response.Status.INTERNAL_SERVER_ERROR, sb.toString());
+ }
+
private List parseNewDataverseMetadataBlocks(JsonArray metadataBlockNamesArray) throws WrappedResponse {
List selectedMetadataBlocks = new ArrayList<>();
for (JsonString metadataBlockName : metadataBlockNamesArray.getValuesAs(JsonString.class)) {
@@ -621,62 +701,22 @@ public Response deleteDataverse(@Context ContainerRequestContext crc, @PathParam
public Response updateAttribute(@Context ContainerRequestContext crc, @PathParam("identifier") String identifier,
@PathParam("attribute") String attribute, @QueryParam("value") String value) {
try {
- Dataverse collection = findDataverseOrDie(identifier);
- User user = getRequestUser(crc);
- DataverseRequest dvRequest = createDataverseRequest(user);
-
- // TODO: The cases below use hard coded strings, because we have no place for definitions of those!
- // They are taken from util.json.JsonParser / util.json.JsonPrinter. This shall be changed.
- // This also should be extended to more attributes, like the type, theme, contacts, some booleans, etc.
- switch (attribute) {
- case "alias":
- collection.setAlias(value);
- break;
- case "name":
- collection.setName(value);
- break;
- case "description":
- collection.setDescription(value);
- break;
- case "affiliation":
- collection.setAffiliation(value);
- break;
- /* commenting out the code from the draft pr #9462:
- case "versionPidsConduct":
- CollectionConduct conduct = CollectionConduct.findBy(value);
- if (conduct == null) {
- return badRequest("'" + value + "' is not one of [" +
- String.join(",", CollectionConduct.asList()) + "]");
- }
- collection.setDatasetVersionPidConduct(conduct);
- break;
- */
- case "filePIDsEnabled":
- if(!user.isSuperuser()) {
- return forbidden("You must be a superuser to change this setting");
- }
- if(!settingsService.isTrueForKey(SettingsServiceBean.Key.AllowEnablingFilePIDsPerCollection, false)) {
- return forbidden("Changing File PID policy per collection is not enabled on this server");
- }
- collection.setFilePIDsEnabled(parseBooleanOrDie(value));
- break;
- default:
- return badRequest("'" + attribute + "' is not a supported attribute");
- }
-
- // Off to persistence layer
- execCommand(new UpdateDataverseCommand(collection, null, null, dvRequest, null));
-
- // Also return modified collection to user
- return ok("Update successful", JsonPrinter.json(collection));
-
- // TODO: This is an anti-pattern, necessary due to this bean being an EJB, causing very noisy and unnecessary
- // logging by the EJB container for bubbling exceptions. (It would be handled by the error handlers.)
+ Dataverse dataverse = findDataverseOrDie(identifier);
+ Object formattedValue = formatAttributeValue(attribute, value);
+ dataverse = execCommand(new UpdateDataverseAttributeCommand(createDataverseRequest(getRequestUser(crc)), dataverse, attribute, formattedValue));
+ return ok("Update successful", JsonPrinter.json(dataverse));
} catch (WrappedResponse e) {
return e.getResponse();
}
}
+ private Object formatAttributeValue(String attribute, String value) throws WrappedResponse {
+ if (attribute.equals("filePIDsEnabled")) {
+ return parseBooleanOrDie(value);
+ }
+ return value;
+ }
+
@GET
@AuthRequired
@Path("{identifier}/inputLevels")
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Info.java b/src/main/java/edu/harvard/iq/dataverse/api/Info.java
index 257519677d3..2439c996816 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Info.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Info.java
@@ -12,12 +12,17 @@
import jakarta.ws.rs.Produces;
import org.apache.commons.io.IOUtils;
+import edu.harvard.iq.dataverse.export.ExportService;
import edu.harvard.iq.dataverse.settings.JvmSettings;
import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
import edu.harvard.iq.dataverse.util.BundleUtil;
import edu.harvard.iq.dataverse.util.SystemConfig;
+import io.gdcc.spi.export.Exporter;
+import io.gdcc.spi.export.ExportException;
+import io.gdcc.spi.export.XMLExporter;
import jakarta.ejb.EJB;
import jakarta.json.Json;
+import jakarta.json.JsonObjectBuilder;
import jakarta.json.JsonValue;
import jakarta.ws.rs.GET;
import jakarta.ws.rs.Path;
@@ -92,6 +97,32 @@ public Response getZipDownloadLimit() {
return ok(zipDownloadLimit);
}
+ @GET
+ @Path("exportFormats")
+ public Response getExportFormats() {
+ JsonObjectBuilder responseModel = Json.createObjectBuilder();
+ ExportService instance = ExportService.getInstance();
+ for (String[] labels : instance.getExportersLabels()) {
+ try {
+ Exporter exporter = instance.getExporter(labels[1]);
+ JsonObjectBuilder exporterObject = Json.createObjectBuilder().add("displayName", labels[0])
+ .add("mediaType", exporter.getMediaType()).add("isHarvestable", exporter.isHarvestable())
+ .add("isVisibleInUserInterface", exporter.isAvailableToUsers());
+ if (exporter instanceof XMLExporter xmlExporter) {
+ exporterObject.add("XMLNameSpace", xmlExporter.getXMLNameSpace())
+ .add("XMLSchemaLocation", xmlExporter.getXMLSchemaLocation())
+ .add("XMLSchemaVersion", xmlExporter.getXMLSchemaVersion());
+ }
+ responseModel.add(labels[1], exporterObject);
+ }
+ catch (ExportException ex){
+ logger.warning("Failed to get: " + labels[1]);
+ logger.warning(ex.getLocalizedMessage());
+ }
+ }
+ return ok(responseModel);
+ }
+
private Response getSettingResponseByKey(SettingsServiceBean.Key key) {
String setting = settingsService.getValueForKey(key);
if (setting != null) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java
index 1f2f1039327..306b863c9e4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/MakeDataCountApi.java
@@ -19,6 +19,9 @@
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.util.Iterator;
import java.util.List;
@@ -152,10 +155,17 @@ public Response updateCitationsForDataset(@PathParam("id") String id) throws IOE
// DataCite wants "doi=", not "doi:".
String authorityPlusIdentifier = persistentId.replaceFirst("doi:", "");
// Request max page size and then loop to handle multiple pages
- URL url = new URL(JvmSettings.DATACITE_REST_API_URL.lookup() +
+ URL url = null;
+ try {
+ url = new URI(JvmSettings.DATACITE_REST_API_URL.lookup(pidProvider.getId()) +
"/events?doi=" +
authorityPlusIdentifier +
- "&source=crossref&page[size]=1000");
+ "&source=crossref&page[size]=1000").toURL();
+ } catch (URISyntaxException e) {
+ //Nominally this means a config error/ bad DATACITE_REST_API_URL for this provider
+ logger.warning("Unable to create URL for " + persistentId + ", pidProvider " + pidProvider.getId());
+ return error(Status.INTERNAL_SERVER_ERROR, "Unable to create DataCite URL to retrieve citations.");
+ }
logger.fine("Retrieving Citations from " + url.toString());
boolean nextPage = true;
JsonArrayBuilder dataBuilder = Json.createArrayBuilder();
@@ -178,7 +188,12 @@ public Response updateCitationsForDataset(@PathParam("id") String id) throws IOE
dataBuilder.add(iter.next());
}
if (links.containsKey("next")) {
- url = new URL(links.getString("next"));
+ try {
+ url = new URI(links.getString("next")).toURL();
+ } catch (URISyntaxException e) {
+ logger.warning("Unable to create URL from DataCite response: " + links.getString("next"));
+ return error(Status.INTERNAL_SERVER_ERROR, "Unable to retrieve all results from DataCite");
+ }
} else {
nextPage = false;
}
@@ -187,7 +202,7 @@ public Response updateCitationsForDataset(@PathParam("id") String id) throws IOE
JsonArray allData = dataBuilder.build();
List datasetExternalCitations = datasetExternalCitationsService.parseCitations(allData);
/*
- * ToDo: If this is the only source of citations, we should remove all the existing ones for the dataset and repopuate them.
+ * ToDo: If this is the only source of citations, we should remove all the existing ones for the dataset and repopulate them.
* As is, this call doesn't remove old citations if there are now none (legacy issue if we decide to stop counting certain types of citation
* as we've done for 'hasPart').
* If there are some, this call individually checks each one and if a matching item exists, it removes it and adds it back. Faster and better to delete all and
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/SavedSearches.java b/src/main/java/edu/harvard/iq/dataverse/api/SavedSearches.java
index 33a11a2df23..e6519c9ff36 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/SavedSearches.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/SavedSearches.java
@@ -181,7 +181,7 @@ public Response delete(@PathParam("id") long doomedId, @QueryParam("unlink") boo
try {
wasDeleted = savedSearchSvc.delete(doomedId, unlink);
} catch (Exception e) {
- return error(INTERNAL_SERVER_ERROR, "Problem while trying to unlink links of saved search id " + doomedId);
+ return error(INTERNAL_SERVER_ERROR, "Problem while trying to unlink links of saved search id " + doomedId + ". Exception: " + e.getLocalizedMessage());
}
if (wasDeleted) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Search.java b/src/main/java/edu/harvard/iq/dataverse/api/Search.java
index 6b9fcb38305..f86f9f446fa 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Search.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Search.java
@@ -175,7 +175,7 @@ public Response search(
JsonArrayBuilder itemsArrayBuilder = Json.createArrayBuilder();
List solrSearchResults = solrQueryResponse.getSolrSearchResults();
for (SolrSearchResult solrSearchResult : solrSearchResults) {
- itemsArrayBuilder.add(solrSearchResult.json(showRelevance, showEntityIds, showApiUrls, metadataFields, getDatasetFileCount(solrSearchResult)));
+ itemsArrayBuilder.add(solrSearchResult.json(showRelevance, showEntityIds, showApiUrls, metadataFields));
}
JsonObjectBuilder spelling_alternatives = Json.createObjectBuilder();
@@ -229,15 +229,6 @@ public Response search(
}
}
- private Long getDatasetFileCount(SolrSearchResult solrSearchResult) {
- DvObject dvObject = solrSearchResult.getEntity();
- if (dvObject.isInstanceofDataset()) {
- DatasetVersion datasetVersion = ((Dataset) dvObject).getVersionFromId(solrSearchResult.getDatasetVersionId());
- return datasetVersionFilesServiceBean.getFileMetadataCount(datasetVersion);
- }
- return null;
- }
-
private User getUser(ContainerRequestContext crc) throws WrappedResponse {
User userToExecuteSearchAs = GuestUser.get();
try {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Users.java b/src/main/java/edu/harvard/iq/dataverse/api/Users.java
index c1a7c95dbff..ecf7839e616 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/Users.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/Users.java
@@ -137,15 +137,20 @@ public Response deleteToken(@Context ContainerRequestContext crc) {
@Path("token")
@AuthRequired
@GET
- public Response getTokenExpirationDate() {
- ApiToken token = authSvc.findApiToken(getRequestApiKey());
-
- if (token == null) {
- return notFound("Token " + getRequestApiKey() + " not found.");
+ public Response getTokenExpirationDate(@Context ContainerRequestContext crc) {
+ try {
+ AuthenticatedUser user = getRequestAuthenticatedUserOrDie(crc);
+ ApiToken token = authSvc.findApiTokenByUser(user);
+
+ if (token == null) {
+ return notFound("Token not found.");
+ }
+
+ return ok(String.format("Token %s expires on %s", token.getTokenString(), token.getExpireTime()));
+
+ } catch (WrappedResponse wr) {
+ return wr.getResponse();
}
-
- return ok("Token " + getRequestApiKey() + " expires on " + token.getExpireTime());
-
}
@Path("token/recreate")
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/dto/DataverseDTO.java b/src/main/java/edu/harvard/iq/dataverse/api/dto/DataverseDTO.java
new file mode 100644
index 00000000000..4f2f1032c07
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/api/dto/DataverseDTO.java
@@ -0,0 +1,63 @@
+package edu.harvard.iq.dataverse.api.dto;
+
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.DataverseContact;
+
+import java.util.List;
+
+public class DataverseDTO {
+ private String alias;
+ private String name;
+ private String description;
+ private String affiliation;
+ private List dataverseContacts;
+ private Dataverse.DataverseType dataverseType;
+
+ public String getAlias() {
+ return alias;
+ }
+
+ public void setAlias(String alias) {
+ this.alias = alias;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ public void setDescription(String description) {
+ this.description = description;
+ }
+
+ public String getAffiliation() {
+ return affiliation;
+ }
+
+ public void setAffiliation(String affiliation) {
+ this.affiliation = affiliation;
+ }
+
+ public List getDataverseContacts() {
+ return dataverseContacts;
+ }
+
+ public void setDataverseContacts(List dataverseContacts) {
+ this.dataverseContacts = dataverseContacts;
+ }
+
+ public Dataverse.DataverseType getDataverseType() {
+ return dataverseType;
+ }
+
+ public void setDataverseType(Dataverse.DataverseType dataverseType) {
+ this.dataverseType = dataverseType;
+ }
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
index 85d4868605d..35d35316f73 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java
@@ -210,7 +210,7 @@ private void processDDI(ImportType importType, XMLStreamReader xmlr, DatasetDTO
// study description section. we'll use the one we found in
// the codeBook entry:
FieldDTO otherIdValue = FieldDTO.createPrimitiveFieldDTO("otherIdValue", codeBookLevelId);
- FieldDTO otherId = FieldDTO.createCompoundFieldDTO("otherId", otherIdValue);
+ FieldDTO otherId = FieldDTO.createMultipleCompoundFieldDTO("otherId", otherIdValue);
citationBlock.getFields().add(otherId);
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java
index d32a548c8bf..aa5b25e3967 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java
@@ -150,12 +150,16 @@ public DatasetDTO processXML( XMLStreamReader xmlr, ForeignMetadataFormatMapping
}
- // Helper method for importing harvested Dublin Core xml.
+ // Helper methods for importing harvested Dublin Core xml.
// Dublin Core is considered a mandatory, built in metadata format mapping.
// It is distributed as required content, in reference_data.sql.
// Note that arbitrary formatting tags are supported for the outer xml
// wrapper. -- L.A. 4.5
public DatasetDTO processOAIDCxml(String DcXmlToParse) throws XMLStreamException {
+ return processOAIDCxml(DcXmlToParse, null, false);
+ }
+
+ public DatasetDTO processOAIDCxml(String DcXmlToParse, String oaiIdentifier, boolean preferSuppliedIdentifier) throws XMLStreamException {
// look up DC metadata mapping:
ForeignMetadataFormatMapping dublinCoreMapping = findFormatMappingByName(DCTERMS);
@@ -185,18 +189,37 @@ public DatasetDTO processOAIDCxml(String DcXmlToParse) throws XMLStreamException
datasetDTO.getDatasetVersion().setVersionState(DatasetVersion.VersionState.RELEASED);
- // Our DC import handles the contents of the dc:identifier field
- // as an "other id". In the context of OAI harvesting, we expect
- // the identifier to be a global id, so we need to rearrange that:
+ // In some cases, the identifier that we want to use for the dataset is
+ // already supplied to the method explicitly. For example, in some
+ // harvesting cases we'll want to use the OAI identifier (the identifier
+ // from the section of the OAI record) for that purpose, without
+ // expecting to find a valid persistent id in the body of the DC record:
- String identifier = getOtherIdFromDTO(datasetDTO.getDatasetVersion());
- logger.fine("Imported identifier: "+identifier);
+ String globalIdentifier;
- String globalIdentifier = reassignIdentifierAsGlobalId(identifier, datasetDTO);
- logger.fine("Detected global identifier: "+globalIdentifier);
+ if (oaiIdentifier != null) {
+ logger.fine("Attempting to use " + oaiIdentifier + " as the persistentId of the imported dataset");
+
+ globalIdentifier = reassignIdentifierAsGlobalId(oaiIdentifier, datasetDTO);
+ } else {
+ // Our DC import handles the contents of the dc:identifier field
+ // as an "other id". Unless we are using an externally supplied
+ // global id, we will be using the first such "other id" that we
+ // can parse and recognize as the global id for the imported dataset
+ // (note that this is the default behavior during harvesting),
+ // so we need to reaassign it accordingly:
+ String identifier = selectIdentifier(datasetDTO.getDatasetVersion(), oaiIdentifier, preferSuppliedIdentifier);
+ logger.fine("Imported identifier: " + identifier);
+
+ globalIdentifier = reassignIdentifierAsGlobalId(identifier, datasetDTO);
+ logger.fine("Detected global identifier: " + globalIdentifier);
+ }
if (globalIdentifier == null) {
- throw new EJBException("Failed to find a global identifier in the OAI_DC XML record.");
+ String exceptionMsg = oaiIdentifier == null ?
+ "Failed to find a global identifier in the OAI_DC XML record." :
+ "Failed to parse the supplied identifier as a valid Persistent Id";
+ throw new EJBException(exceptionMsg);
}
return datasetDTO;
@@ -205,8 +228,17 @@ public DatasetDTO processOAIDCxml(String DcXmlToParse) throws XMLStreamException
private void processXMLElement(XMLStreamReader xmlr, String currentPath, String openingTag, ForeignMetadataFormatMapping foreignFormatMapping, DatasetDTO datasetDTO) throws XMLStreamException {
logger.fine("entering processXMLElement; ("+currentPath+")");
-
- for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) {
+
+ while (xmlr.hasNext()) {
+
+ int event;
+ try {
+ event = xmlr.next();
+ } catch (XMLStreamException ex) {
+ logger.warning("Error occurred in the XML parsing : " + ex.getMessage());
+ continue; // Skip Undeclared namespace prefix and Unexpected close tag related to com.ctc.wstx.exc.WstxParsingException
+ }
+
if (event == XMLStreamConstants.START_ELEMENT) {
String currentElement = xmlr.getLocalName();
@@ -335,8 +367,20 @@ private FieldDTO makeDTO(DatasetFieldType dataverseFieldType, FieldDTO value, St
return value;
}
- private String getOtherIdFromDTO(DatasetVersionDTO datasetVersionDTO) {
+ public String selectIdentifier(DatasetVersionDTO datasetVersionDTO, String suppliedIdentifier) {
+ return selectIdentifier(datasetVersionDTO, suppliedIdentifier, false);
+ }
+
+ private String selectIdentifier(DatasetVersionDTO datasetVersionDTO, String suppliedIdentifier, boolean preferSuppliedIdentifier) {
List otherIds = new ArrayList<>();
+
+ if (suppliedIdentifier != null && preferSuppliedIdentifier) {
+ // This supplied identifier (in practice, his is likely the OAI-PMH
+ // identifier from the section) will be our first
+ // choice candidate for the pid of the imported dataset:
+ otherIds.add(suppliedIdentifier);
+ }
+
for (Map.Entry entry : datasetVersionDTO.getMetadataBlocks().entrySet()) {
String key = entry.getKey();
MetadataBlockDTO value = entry.getValue();
@@ -354,6 +398,16 @@ private String getOtherIdFromDTO(DatasetVersionDTO datasetVersionDTO) {
}
}
}
+
+ if (suppliedIdentifier != null && !preferSuppliedIdentifier) {
+ // Unless specifically instructed to prefer this extra identifier
+ // (in practice, this is likely the OAI-PMH identifier from the
+ // section), we will try to use it as the *last*
+ // possible candidate for the pid, so, adding it to the end of the
+ // list:
+ otherIds.add(suppliedIdentifier);
+ }
+
if (!otherIds.isEmpty()) {
// We prefer doi or hdl identifiers like "doi:10.7910/DVN/1HE30F"
for (String otherId : otherIds) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
index d2bba56f884..7dc2aed799e 100644
--- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportServiceBean.java
@@ -7,7 +7,6 @@
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
-import edu.harvard.iq.dataverse.DataFile;
import edu.harvard.iq.dataverse.Dataset;
import edu.harvard.iq.dataverse.DatasetField;
import edu.harvard.iq.dataverse.DatasetFieldConstant;
@@ -20,6 +19,7 @@
import edu.harvard.iq.dataverse.DataverseContact;
import edu.harvard.iq.dataverse.DataverseServiceBean;
import edu.harvard.iq.dataverse.EjbDataverseEngine;
+import edu.harvard.iq.dataverse.GlobalId;
import edu.harvard.iq.dataverse.MetadataBlockServiceBean;
import edu.harvard.iq.dataverse.api.dto.DatasetDTO;
import edu.harvard.iq.dataverse.api.imports.ImportUtil.ImportType;
@@ -31,6 +31,7 @@
import edu.harvard.iq.dataverse.engine.command.impl.CreateHarvestedDatasetCommand;
import edu.harvard.iq.dataverse.engine.command.impl.CreateNewDatasetCommand;
import edu.harvard.iq.dataverse.engine.command.impl.DestroyDatasetCommand;
+import edu.harvard.iq.dataverse.engine.command.impl.UpdateHarvestedDatasetCommand;
import edu.harvard.iq.dataverse.harvest.client.HarvestingClient;
import edu.harvard.iq.dataverse.search.IndexServiceBean;
import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
@@ -40,6 +41,7 @@
import edu.harvard.iq.dataverse.util.json.JsonUtil;
import edu.harvard.iq.dataverse.license.LicenseServiceBean;
import edu.harvard.iq.dataverse.pidproviders.PidUtil;
+import edu.harvard.iq.dataverse.util.DatasetFieldUtil;
import java.io.File;
import java.io.FileOutputStream;
@@ -206,9 +208,15 @@ public JsonObjectBuilder handleFile(DataverseRequest dataverseRequest, Dataverse
}
@TransactionAttribute(TransactionAttributeType.REQUIRES_NEW)
- public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, HarvestingClient harvestingClient, String harvestIdentifier, String metadataFormat, File metadataFile, Date oaiDateStamp, PrintWriter cleanupLog) throws ImportException, IOException {
+ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest,
+ HarvestingClient harvestingClient,
+ String harvestIdentifier,
+ String metadataFormat,
+ File metadataFile,
+ Date oaiDateStamp,
+ PrintWriter cleanupLog) throws ImportException, IOException {
if (harvestingClient == null || harvestingClient.getDataverse() == null) {
- throw new ImportException("importHarvestedDataset called wiht a null harvestingClient, or an invalid harvestingClient.");
+ throw new ImportException("importHarvestedDataset called with a null harvestingClient, or an invalid harvestingClient.");
}
Dataverse owner = harvestingClient.getDataverse();
Dataset importedDataset = null;
@@ -242,8 +250,8 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
} else if ("dc".equalsIgnoreCase(metadataFormat) || "oai_dc".equals(metadataFormat)) {
logger.fine("importing DC "+metadataFile.getAbsolutePath());
try {
- String xmlToParse = new String(Files.readAllBytes(metadataFile.toPath()));
- dsDTO = importGenericService.processOAIDCxml(xmlToParse);
+ String xmlToParse = new String(Files.readAllBytes(metadataFile.toPath()));
+ dsDTO = importGenericService.processOAIDCxml(xmlToParse, harvestIdentifier, harvestingClient.isUseOaiIdentifiersAsPids());
} catch (IOException | XMLStreamException e) {
throw new ImportException("Failed to process Dublin Core XML record: "+ e.getClass() + " (" + e.getMessage() + ")");
}
@@ -268,116 +276,121 @@ public Dataset doImportHarvestedDataset(DataverseRequest dataverseRequest, Harve
}
JsonObject obj = JsonUtil.getJsonObject(json);
- //and call parse Json to read it into a dataset
+
+ String protocol = obj.getString("protocol", null);
+ String authority = obj.getString("authority", null);
+ String identifier = obj.getString("identifier",null);
+
+ GlobalId globalId;
+
+ // A Global ID is required:
+ // (meaning, we will fail with an exception if the imports above have
+ // not managed to find an acceptable global identifier in the harvested
+ // metadata)
+
+ try {
+ globalId = PidUtil.parseAsGlobalID(protocol, authority, identifier);
+ } catch (IllegalArgumentException iax) {
+ throw new ImportException("The harvested metadata record with the OAI server identifier " + harvestIdentifier + " does not contain a global identifier this Dataverse can parse, skipping.");
+ }
+
+ if (globalId == null) {
+ throw new ImportException("The harvested metadata record with the OAI server identifier " + harvestIdentifier + " does not contain a global identifier this Dataverse recognizes, skipping.");
+ }
+
+ String globalIdString = globalId.asString();
+
+ if (StringUtils.isEmpty(globalIdString)) {
+ // @todo this check may not be necessary, now that there's a null check above
+ throw new ImportException("The harvested metadata record with the OAI server identifier " + harvestIdentifier + " does not contain a global identifier this Dataverse recognizes, skipping.");
+ }
+
+ DatasetVersion harvestedVersion;
+
+ Dataset existingDataset = datasetService.findByGlobalId(globalIdString);
+
try {
+ Dataset harvestedDataset;
+
JsonParser parser = new JsonParser(datasetfieldService, metadataBlockService, settingsService, licenseService, datasetTypeService, harvestingClient);
parser.setLenient(true);
- Dataset ds = parser.parseDataset(obj);
- // For ImportType.NEW, if the metadata contains a global identifier, and it's not a protocol
- // we support, it should be rejected.
- // (TODO: ! - add some way of keeping track of supported protocols!)
- //if (ds.getGlobalId() != null && !ds.getProtocol().equals(settingsService.getValueForKey(SettingsServiceBean.Key.Protocol, ""))) {
- // throw new ImportException("Could not register id " + ds.getGlobalId() + ", protocol not supported");
- //}
- ds.setOwner(owner);
- ds.getLatestVersion().setDatasetFields(ds.getLatestVersion().initDatasetFields());
+ if (existingDataset == null) {
+ // Creating a new dataset from scratch:
+
+ harvestedDataset = parser.parseDataset(obj);
- if (ds.getVersions().get(0).getReleaseTime() == null) {
- ds.getVersions().get(0).setReleaseTime(oaiDateStamp);
- }
-
- // Check data against required contraints
- List> violations = ds.getVersions().get(0).validateRequired();
- if (!violations.isEmpty()) {
- // For migration and harvest, add NA for missing required values
- for (ConstraintViolation v : violations) {
- DatasetField f = v.getRootBean();
- f.setSingleValue(DatasetField.NA_VALUE);
+ harvestedDataset.setHarvestedFrom(harvestingClient);
+ harvestedDataset.setHarvestIdentifier(harvestIdentifier);
+
+ harvestedVersion = harvestedDataset.getVersions().get(0);
+ } else {
+ // We already have a dataset with this id in the database.
+ // Let's check a few things before we go any further with it:
+
+ // If this dataset already exists IN ANOTHER COLLECTION
+ // we are just going to skip it!
+ if (existingDataset.getOwner() != null && !owner.getId().equals(existingDataset.getOwner().getId())) {
+ throw new ImportException("The dataset with the global id " + globalIdString + " already exists, in the dataverse " + existingDataset.getOwner().getAlias() + ", skipping.");
}
- }
-
- // Check data against validation constraints
- // If we are migrating and "scrub migration data" is true we attempt to fix invalid data
- // if the fix fails stop processing of this file by throwing exception
- Set invalidViolations = ds.getVersions().get(0).validate();
- ValidatorFactory factory = Validation.buildDefaultValidatorFactory();
- Validator validator = factory.getValidator();
- if (!invalidViolations.isEmpty()) {
- for (ConstraintViolation v : invalidViolations) {
- DatasetFieldValue f = v.getRootBean();
- boolean fixed = false;
- boolean converted = false;
- // TODO: Is this scrubbing something we want to continue doing?
- if (settingsService.isTrueForKey(SettingsServiceBean.Key.ScrubMigrationData, false)) {
- fixed = processMigrationValidationError(f, cleanupLog, metadataFile.getName());
- converted = true;
- if (fixed) {
- Set> scrubbedViolations = validator.validate(f);
- if (!scrubbedViolations.isEmpty()) {
- fixed = false;
- }
- }
- }
- if (!fixed) {
- String msg = "Data modified - File: " + metadataFile.getName() + "; Field: " + f.getDatasetField().getDatasetFieldType().getDisplayName() + "; "
- + "Invalid value: '" + f.getValue() + "'" + " Converted Value:'" + DatasetField.NA_VALUE + "'";
- cleanupLog.println(msg);
- f.setValue(DatasetField.NA_VALUE);
-
- }
+ // And if we already have a dataset with this same global id at
+ // this Dataverse instance, but it is a LOCAL dataset (can happen!),
+ // we're going to skip it also:
+ if (!existingDataset.isHarvested()) {
+ throw new ImportException("A LOCAL dataset with the global id " + globalIdString + " already exists in this dataverse; skipping.");
}
+ // For harvested datasets, there should always only be one version.
+ if (existingDataset.getVersions().size() != 1) {
+ throw new ImportException("Error importing Harvested Dataset, existing dataset has " + existingDataset.getVersions().size() + " versions");
+ }
+
+ // We will attempt to import the new version, and replace the
+ // current, already existing version with it.
+ harvestedVersion = parser.parseDatasetVersion(obj.getJsonObject("datasetVersion"));
+
+ // For the purposes of validation, the version needs to be attached
+ // to a non-null dataset. We will create a throwaway temporary
+ // dataset for this:
+ harvestedDataset = createTemporaryHarvestedDataset(harvestedVersion);
}
+
+ harvestedDataset.setOwner(owner);
- // A Global ID is required, in order for us to be able to harvest and import
- // this dataset:
- if (StringUtils.isEmpty(ds.getGlobalId().asString())) {
- throw new ImportException("The harvested metadata record with the OAI server identifier "+harvestIdentifier+" does not contain a global unique identifier that we could recognize, skipping.");
- }
-
- ds.setHarvestedFrom(harvestingClient);
- ds.setHarvestIdentifier(harvestIdentifier);
+ // Either a full new import, or an update of an existing harvested
+ // Dataset, perform some cleanup on the new version imported from the
+ // parsed metadata:
- Dataset existingDs = datasetService.findByGlobalId(ds.getGlobalId().asString());
+ harvestedVersion.setDatasetFields(harvestedVersion.initDatasetFields());
- if (existingDs != null) {
- // If this dataset already exists IN ANOTHER DATAVERSE
- // we are just going to skip it!
- if (existingDs.getOwner() != null && !owner.getId().equals(existingDs.getOwner().getId())) {
- throw new ImportException("The dataset with the global id "+ds.getGlobalId().asString()+" already exists, in the dataverse "+existingDs.getOwner().getAlias()+", skipping.");
- }
- // And if we already have a dataset with this same id, in this same
- // dataverse, but it is LOCAL dataset (can happen!), we're going to
- // skip it also:
- if (!existingDs.isHarvested()) {
- throw new ImportException("A LOCAL dataset with the global id "+ds.getGlobalId().asString()+" already exists in this dataverse; skipping.");
- }
- // For harvested datasets, there should always only be one version.
- // We will replace the current version with the imported version.
- if (existingDs.getVersions().size() != 1) {
- throw new ImportException("Error importing Harvested Dataset, existing dataset has " + existingDs.getVersions().size() + " versions");
- }
- // Purge all the SOLR documents associated with this client from the
- // index server:
- indexService.deleteHarvestedDocuments(existingDs);
- // files from harvested datasets are removed unceremoniously,
- // directly in the database. no need to bother calling the
- // DeleteFileCommand on them.
- for (DataFile harvestedFile : existingDs.getFiles()) {
- DataFile merged = em.merge(harvestedFile);
- em.remove(merged);
- harvestedFile = null;
- }
- // TODO:
- // Verify what happens with the indexed files in SOLR?
- // are they going to be overwritten by the reindexing of the dataset?
- existingDs.setFiles(null);
- Dataset merged = em.merge(existingDs);
- // harvested datasets don't have physical files - so no need to worry about that.
- engineSvc.submit(new DestroyDatasetCommand(merged, dataverseRequest));
+ if (harvestedVersion.getReleaseTime() == null) {
+ harvestedVersion.setReleaseTime(oaiDateStamp);
}
+
+ // Check data against validation constraints.
+ // Make an attempt to sanitize any invalid fields encountered -
+ // missing required fields or invalid values, by filling the values
+ // with NAs.
+
+ boolean sanitized = validateAndSanitizeVersionMetadata(harvestedVersion, cleanupLog);
+
+ // Note: this sanitizing approach, of replacing invalid values with
+ // "NA" does not work with certain fields. For example, using it to
+ // populate a GeoBox coordinate value will result in an invalid
+ // field. So we will attempt to re-validate the santized version.
+ // This time around, it will throw an exception if still invalid, so
+ // that we'll stop before proceeding any further:
- importedDataset = engineSvc.submit(new CreateHarvestedDatasetCommand(ds, dataverseRequest));
+ if (sanitized) {
+ validateVersionMetadata(harvestedVersion, cleanupLog);
+ }
+
+ DatasetFieldUtil.tidyUpFields(harvestedVersion.getDatasetFields(), true);
+
+ if (existingDataset != null) {
+ importedDataset = engineSvc.submit(new UpdateHarvestedDatasetCommand(existingDataset, harvestedVersion, dataverseRequest));
+ } else {
+ importedDataset = engineSvc.submit(new CreateHarvestedDatasetCommand(harvestedDataset, dataverseRequest));
+ }
} catch (JsonParseException | ImportException | CommandException ex) {
logger.fine("Failed to import harvested dataset: " + ex.getClass() + ": " + ex.getMessage());
@@ -439,7 +452,7 @@ public JsonObjectBuilder doImport(DataverseRequest dataverseRequest, Dataverse o
ds.setOwner(owner);
ds.getLatestVersion().setDatasetFields(ds.getLatestVersion().initDatasetFields());
- // Check data against required contraints
+ // Check data against required constraints
List> violations = ds.getVersions().get(0).validateRequired();
if (!violations.isEmpty()) {
if ( importType.equals(ImportType.HARVEST) ) {
@@ -696,6 +709,104 @@ private String convertInvalidDateString(String inString){
return null;
}
+ /**
+ * A shortcut method for validating AND attempting to sanitize a DatasetVersion
+ * @param version
+ * @param cleanupLog - any invalid values and their replacements are logged there
+ * @return true if any invalid values were encountered and sanitized
+ * @throws ImportException (although it should never happen in this mode)
+ */
+ private boolean validateAndSanitizeVersionMetadata(DatasetVersion version, PrintWriter cleanupLog) throws ImportException {
+ return validateVersionMetadata(version, true, cleanupLog);
+ }
+
+ /**
+ * A shortcut method for validating a DatasetVersion; will throw an exception
+ * if invalid, without attempting to sanitize the invalid values.
+ * @param version
+ * @param log - will log the invalid fields encountered there
+ * @throws ImportException
+ */
+ private void validateVersionMetadata(DatasetVersion version, PrintWriter log) throws ImportException {
+ validateVersionMetadata(version, false, log);
+ }
+
+ /**
+ * Validate the metadata fields of a newly-created version, and depending on
+ * the "sanitize" flag supplied, may or may not attempt to sanitize the supplied
+ * values by replacing them with "NA"s.
+ * @param version
+ * @param sanitize - boolean indicating whether to attempt to fix invalid values
+ * @param cleanupLog - to log any invalid values encountered will be logged
+ * @return - true if any invalid values have been replaced
+ * @throws ImportException
+ */
+ private boolean validateVersionMetadata(DatasetVersion version, boolean sanitize, PrintWriter cleanupLog) throws ImportException {
+ boolean fixed = false;
+ Set invalidViolations = version.validate();
+ if (!invalidViolations.isEmpty()) {
+ for (ConstraintViolation v : invalidViolations) {
+ Object invalid = v.getRootBean();
+ String msg = "";
+ if (invalid instanceof DatasetField) {
+ DatasetField f = (DatasetField) invalid;
+
+ msg += "Missing required field: " + f.getDatasetFieldType().getDisplayName() + ";";
+ if (sanitize) {
+ msg += " populated with '" + DatasetField.NA_VALUE + "'";
+ f.setSingleValue(DatasetField.NA_VALUE);
+ fixed = true;
+ }
+ } else if (invalid instanceof DatasetFieldValue) {
+ DatasetFieldValue fv = (DatasetFieldValue) invalid;
+
+ msg += "Invalid metadata field: " + fv.getDatasetField().getDatasetFieldType().getDisplayName() + "; "
+ + "Invalid value: '" + fv.getValue() + "'";
+ if (sanitize) {
+ msg += ", replaced with '" + DatasetField.NA_VALUE + "'";
+ fv.setValue(DatasetField.NA_VALUE);
+ fixed = true;
+ }
+ } else {
+ // DatasetVersion.validate() can also produce constraint violations
+ // in TermsOfUse and FileMetadata classes.
+ // We do not make any attempt to sanitize those.
+ if (invalid != null) {
+ msg += "Invalid " + invalid.getClass().getName() + ": " + v.getMessage();
+ }
+ }
+ cleanupLog.println(msg);
+
+ // Note: "NA" does not work with certain fields. For example,
+ // using it to populate a GeoBox coordinate value is going
+ // to result in an invalid field. So we'll need to validate the
+ // version again after the first, sanitizing pass and see if it
+ // helped or not.
+ }
+ if (!sanitize) {
+ throw new ImportException("Version was still failing validation after the first attempt to sanitize the invalid values.");
+ }
+ }
+ return fixed;
+ }
+
+ /**
+ * Helper method that creates a throwaway Harvested Dataset to temporarily
+ * attach the newly-harvested version to. We need this when, instead of
+ * importing a brand-new harvested dataset from scratch, we are planning to
+ * attempt to update an already existing dataset harvested from the same
+ * archival location.
+ * @param harvestedVersion - a newly created Version imported from harvested metadata
+ * @return - a temporary dataset to which the new version has been attached
+ */
+ private Dataset createTemporaryHarvestedDataset(DatasetVersion harvestedVersion) {
+ Dataset tempDataset = new Dataset();
+ harvestedVersion.setDataset(tempDataset);
+ tempDataset.setVersions(new ArrayList<>(1));
+ tempDataset.getVersions().add(harvestedVersion);
+
+ return tempDataset;
+ }
private static class MyCustomFormatter extends Formatter {
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
index 3bf2107e52b..d0da66c38e0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/GlobusOverlayAccessIO.java
@@ -215,7 +215,7 @@ public long retrieveSizeFromMedia() {
JsonArray dataArray = responseJson.getJsonArray("DATA");
if (dataArray != null && dataArray.size() != 0) {
//File found
- return (long) responseJson.getJsonArray("DATA").getJsonObject(0).getJsonNumber("size").longValueExact();
+ return (long) dataArray.getJsonObject(0).getJsonNumber("size").longValueExact();
}
} else {
logger.warning("Response from " + get.getURI().toString() + " was "
diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
index d2fdec7b323..5b9e496281f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
+++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java
@@ -753,6 +753,12 @@ public Path getFileSystemPath() throws UnsupportedDataAccessOperationException {
@Override
public boolean exists() {
+ try {
+ key = getMainFileKey();
+ } catch (IOException e) {
+ logger.warning("Caught an IOException in S3AccessIO.exists(): " + e.getMessage());
+ return false;
+ }
String destinationKey = null;
if (dvObject instanceof DataFile) {
destinationKey = key;
diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
index a470f08f736..6b98848021c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java
@@ -136,8 +136,6 @@ public class AddReplaceFileHelper{
private String newFileName; // step 30
private String newFileContentType; // step 30
private String newStorageIdentifier; // step 30
- private String newCheckSum; // step 30
- private ChecksumType newCheckSumType; //step 30
// -- Optional
private DataFile fileToReplace; // step 25
@@ -146,6 +144,7 @@ public class AddReplaceFileHelper{
private DatasetVersion clone;
List initialFileList;
List finalFileList;
+ private boolean trustSuppliedFileSizes;
// -----------------------------------
// Ingested files
@@ -610,15 +609,9 @@ private boolean runAddReplacePhase1(Dataset owner,
return false;
}
- if(optionalFileParams != null) {
- if(optionalFileParams.hasCheckSum()) {
- newCheckSum = optionalFileParams.getCheckSum();
- newCheckSumType = optionalFileParams.getCheckSumType();
- }
- }
msgt("step_030_createNewFilesViaIngest");
- if (!this.step_030_createNewFilesViaIngest()){
+ if (!this.step_030_createNewFilesViaIngest(optionalFileParams)){
return false;
}
@@ -1191,7 +1184,7 @@ private boolean step_007_auto_isReplacementInLatestVersion(DataFile existingFile
}
- private boolean step_030_createNewFilesViaIngest(){
+ private boolean step_030_createNewFilesViaIngest(OptionalFileParams optionalFileParams){
if (this.hasError()){
return false;
@@ -1203,21 +1196,28 @@ private boolean step_030_createNewFilesViaIngest(){
//Don't repeatedly update the clone (losing changes) in multifile case
clone = workingVersion.cloneDatasetVersion();
}
+
+ Long suppliedFileSize = null;
+ String newCheckSum = null;
+ ChecksumType newCheckSumType = null;
+
+
+ if (optionalFileParams != null) {
+ if (optionalFileParams.hasCheckSum()) {
+ newCheckSum = optionalFileParams.getCheckSum();
+ newCheckSumType = optionalFileParams.getCheckSumType();
+ }
+ if (trustSuppliedFileSizes && optionalFileParams.hasFileSize()) {
+ suppliedFileSize = optionalFileParams.getFileSize();
+ }
+ }
+
try {
- /*CreateDataFileResult result = FileUtil.createDataFiles(workingVersion,
- this.newFileInputStream,
- this.newFileName,
- this.newFileContentType,
- this.newStorageIdentifier,
- this.newCheckSum,
- this.newCheckSumType,
- this.systemConfig);*/
-
UploadSessionQuotaLimit quota = null;
if (systemConfig.isStorageQuotasEnforced()) {
quota = fileService.getUploadSessionQuotaLimit(dataset);
}
- Command cmd = new CreateNewDataFilesCommand(dvRequest, workingVersion, newFileInputStream, newFileName, newFileContentType, newStorageIdentifier, quota, newCheckSum, newCheckSumType);
+ Command cmd = new CreateNewDataFilesCommand(dvRequest, workingVersion, newFileInputStream, newFileName, newFileContentType, newStorageIdentifier, quota, newCheckSum, newCheckSumType, suppliedFileSize);
CreateDataFileResult createDataFilesResult = commandEngine.submit(cmd);
initialFileList = createDataFilesResult.getDataFiles();
@@ -2033,9 +2033,15 @@ public void setDuplicateFileWarning(String duplicateFileWarning) {
* @param jsonData - an array of jsonData entries (one per file) using the single add file jsonData format
* @param dataset
* @param authUser
+ * @param trustSuppliedSizes - whether to accept the fileSize values passed
+ * in jsonData (we don't want to trust the users of the S3 direct
+ * upload API with that information - we will verify the status of
+ * the files in the S3 bucket and confirm the sizes in the process.
+ * we do want GlobusService to be able to pass the file sizes, since
+ * they are obtained and verified via a Globus API lookup).
* @return
*/
- public Response addFiles(String jsonData, Dataset dataset, User authUser) {
+ public Response addFiles(String jsonData, Dataset dataset, User authUser, boolean trustSuppliedFileSizes) {
msgt("(addFilesToDataset) jsonData: " + jsonData.toString());
JsonArrayBuilder jarr = Json.createArrayBuilder();
@@ -2044,6 +2050,7 @@ public Response addFiles(String jsonData, Dataset dataset, User authUser) {
int totalNumberofFiles = 0;
int successNumberofFiles = 0;
+ this.trustSuppliedFileSizes = trustSuppliedFileSizes;
// -----------------------------------------------------------
// Read jsonData and Parse files information from jsondata :
// -----------------------------------------------------------
@@ -2176,6 +2183,10 @@ public Response addFiles(String jsonData, Dataset dataset, User authUser) {
.add("data", Json.createObjectBuilder().add("Files", jarr).add("Result", result)).build() ).build();
}
+ public Response addFiles(String jsonData, Dataset dataset, User authUser) {
+ return addFiles(jsonData, dataset, authUser, false);
+ }
+
/**
* Replace multiple files with prepositioned replacements as listed in the
* jsonData. Works with direct upload, Globus, and other out-of-band methods.
diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java
index 959dbc4e262..54844160163 100644
--- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java
+++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java
@@ -39,6 +39,12 @@
* - Provenance related information
*
* @author rmp553
+ * @todo (?) We may want to consider renaming this class to DataFileParams or
+ * DataFileInfo... it was originally created to encode some bits of info -
+ * the file "tags" specifically, that didn't fit in elsewhere in the normal
+ * workflow; but it's been expanded to cover pretty much everything else associated
+ * with DataFiles and it's not really "optional" anymore when, for example, used
+ * in the direct upload workflow. (?)
*/
public class OptionalFileParams {
@@ -76,6 +82,8 @@ public class OptionalFileParams {
public static final String MIME_TYPE_ATTR_NAME = "mimeType";
private String checkSumValue;
private ChecksumType checkSumType;
+ public static final String FILE_SIZE_ATTR_NAME = "fileSize";
+ private Long fileSize;
public static final String LEGACY_CHECKSUM_ATTR_NAME = "md5Hash";
public static final String CHECKSUM_OBJECT_NAME = "checksum";
public static final String CHECKSUM_OBJECT_TYPE = "@type";
@@ -268,6 +276,18 @@ public String getCheckSum() {
public ChecksumType getCheckSumType() {
return checkSumType;
}
+
+ public boolean hasFileSize() {
+ return fileSize != null;
+ }
+
+ public Long getFileSize() {
+ return fileSize;
+ }
+
+ public void setFileSize(long fileSize) {
+ this.fileSize = fileSize;
+ }
/**
* Set tags
@@ -416,7 +436,13 @@ else if ((jsonObj.has(CHECKSUM_OBJECT_NAME)) && (!jsonObj.get(CHECKSUM_OBJECT_NA
this.checkSumType = ChecksumType.fromString(((JsonObject) jsonObj.get(CHECKSUM_OBJECT_NAME)).get(CHECKSUM_OBJECT_TYPE).getAsString());
}
-
+ // -------------------------------
+ // get file size as a Long, if supplied
+ // -------------------------------
+ if ((jsonObj.has(FILE_SIZE_ATTR_NAME)) && (!jsonObj.get(FILE_SIZE_ATTR_NAME).isJsonNull())){
+
+ this.fileSize = jsonObj.get(FILE_SIZE_ATTR_NAME).getAsLong();
+ }
// -------------------------------
// get tags
// -------------------------------
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java
index db9dc142506..b36a638956f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java
@@ -13,8 +13,10 @@
import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
import edu.harvard.iq.dataverse.pidproviders.PidProvider;
import static edu.harvard.iq.dataverse.util.StringUtil.isEmpty;
+import java.io.IOException;
import java.util.Objects;
import java.util.logging.Logger;
+import org.apache.solr.client.solrj.SolrServerException;
/**;
* An abstract base class for commands that creates {@link Dataset}s.
@@ -148,9 +150,19 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
//Use for code that requires database ids
postDBFlush(theDataset, ctxt);
-
- ctxt.index().asyncIndexDataset(theDataset, true);
-
+
+ if (harvested) {
+ try {
+ ctxt.index().indexDataset(theDataset, true);
+ } catch (SolrServerException | IOException solrEx) {
+ logger.warning("Failed to index harvested dataset. " + solrEx.getMessage());
+ }
+ } else {
+ // The asynchronous version does not throw any exceptions,
+ // logging them internally instead.
+ ctxt.index().asyncIndexDataset(theDataset, true);
+ }
+
return theDataset;
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java
new file mode 100644
index 00000000000..91f3a5b823c
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractWriteDataverseCommand.java
@@ -0,0 +1,106 @@
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.*;
+import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * An abstract base class for commands that perform write operations on {@link Dataverse}s.
+ */
+abstract class AbstractWriteDataverseCommand extends AbstractCommand {
+
+ protected Dataverse dataverse;
+ private final List inputLevels;
+ private final List facets;
+ protected final List metadataBlocks;
+ private final boolean resetRelationsOnNullValues;
+
+ public AbstractWriteDataverseCommand(Dataverse dataverse,
+ Dataverse affectedDataverse,
+ DataverseRequest request,
+ List facets,
+ List inputLevels,
+ List metadataBlocks,
+ boolean resetRelationsOnNullValues) {
+ super(request, affectedDataverse);
+ this.dataverse = dataverse;
+ if (facets != null) {
+ this.facets = new ArrayList<>(facets);
+ } else {
+ this.facets = null;
+ }
+ if (inputLevels != null) {
+ this.inputLevels = new ArrayList<>(inputLevels);
+ } else {
+ this.inputLevels = null;
+ }
+ if (metadataBlocks != null) {
+ this.metadataBlocks = new ArrayList<>(metadataBlocks);
+ } else {
+ this.metadataBlocks = null;
+ }
+ this.resetRelationsOnNullValues = resetRelationsOnNullValues;
+ }
+
+ @Override
+ public Dataverse execute(CommandContext ctxt) throws CommandException {
+ dataverse = innerExecute(ctxt);
+
+ processMetadataBlocks();
+ processFacets(ctxt);
+ processInputLevels(ctxt);
+
+ return ctxt.dataverses().save(dataverse);
+ }
+
+ private void processMetadataBlocks() {
+ if (metadataBlocks != null && !metadataBlocks.isEmpty()) {
+ dataverse.setMetadataBlockRoot(true);
+ dataverse.setMetadataBlocks(metadataBlocks);
+ } else if (resetRelationsOnNullValues) {
+ dataverse.setMetadataBlockRoot(false);
+ dataverse.clearMetadataBlocks();
+ }
+ }
+
+ private void processFacets(CommandContext ctxt) {
+ if (facets != null) {
+ ctxt.facets().deleteFacetsFor(dataverse);
+ dataverse.setDataverseFacets(new ArrayList<>());
+
+ if (!facets.isEmpty()) {
+ dataverse.setFacetRoot(true);
+ }
+
+ for (int i = 0; i < facets.size(); i++) {
+ ctxt.facets().create(i, facets.get(i), dataverse);
+ }
+ } else if (resetRelationsOnNullValues) {
+ ctxt.facets().deleteFacetsFor(dataverse);
+ dataverse.setFacetRoot(false);
+ }
+ }
+
+ private void processInputLevels(CommandContext ctxt) {
+ if (inputLevels != null) {
+ if (!inputLevels.isEmpty()) {
+ dataverse.addInputLevelsMetadataBlocksIfNotPresent(inputLevels);
+ }
+ ctxt.fieldTypeInputLevels().deleteFacetsFor(dataverse);
+ inputLevels.forEach(inputLevel -> {
+ inputLevel.setDataverse(dataverse);
+ ctxt.fieldTypeInputLevels().create(inputLevel);
+ });
+ } else if (resetRelationsOnNullValues) {
+ ctxt.fieldTypeInputLevels().deleteFacetsFor(dataverse);
+ }
+ }
+
+ abstract protected Dataverse innerExecute(CommandContext ctxt) throws IllegalCommandException;
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java
index 489b36e7cef..3728f3ee6ce 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java
@@ -6,11 +6,9 @@
import edu.harvard.iq.dataverse.authorization.groups.Group;
import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser;
import edu.harvard.iq.dataverse.authorization.users.User;
-import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
import edu.harvard.iq.dataverse.engine.command.CommandContext;
import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
-import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
@@ -27,48 +25,26 @@
* @author michael
*/
@RequiredPermissions(Permission.AddDataverse)
-public class CreateDataverseCommand extends AbstractCommand {
-
- private final Dataverse created;
- private final List inputLevelList;
- private final List facetList;
- private final List metadataBlocks;
+public class CreateDataverseCommand extends AbstractWriteDataverseCommand {
public CreateDataverseCommand(Dataverse created,
- DataverseRequest aRequest,
- List facetList,
- List inputLevelList) {
- this(created, aRequest, facetList, inputLevelList, null);
+ DataverseRequest request,
+ List facets,
+ List inputLevels) {
+ this(created, request, facets, inputLevels, null);
}
public CreateDataverseCommand(Dataverse created,
- DataverseRequest aRequest,
- List facetList,
- List inputLevelList,
+ DataverseRequest request,
+ List facets,
+ List inputLevels,
List metadataBlocks) {
- super(aRequest, created.getOwner());
- this.created = created;
- if (facetList != null) {
- this.facetList = new ArrayList<>(facetList);
- } else {
- this.facetList = null;
- }
- if (inputLevelList != null) {
- this.inputLevelList = new ArrayList<>(inputLevelList);
- } else {
- this.inputLevelList = null;
- }
- if (metadataBlocks != null) {
- this.metadataBlocks = new ArrayList<>(metadataBlocks);
- } else {
- this.metadataBlocks = null;
- }
+ super(created, created.getOwner(), request, facets, inputLevels, metadataBlocks, false);
}
@Override
- public Dataverse execute(CommandContext ctxt) throws CommandException {
-
- Dataverse owner = created.getOwner();
+ protected Dataverse innerExecute(CommandContext ctxt) throws IllegalCommandException {
+ Dataverse owner = dataverse.getOwner();
if (owner == null) {
if (ctxt.dataverses().isRootDataverseExists()) {
throw new IllegalCommandException("Root Dataverse already exists. Cannot create another one", this);
@@ -76,44 +52,44 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
}
if (metadataBlocks != null && !metadataBlocks.isEmpty()) {
- created.setMetadataBlockRoot(true);
- created.setMetadataBlocks(metadataBlocks);
+ dataverse.setMetadataBlockRoot(true);
+ dataverse.setMetadataBlocks(metadataBlocks);
}
- if (created.getCreateDate() == null) {
- created.setCreateDate(new Timestamp(new Date().getTime()));
+ if (dataverse.getCreateDate() == null) {
+ dataverse.setCreateDate(new Timestamp(new Date().getTime()));
}
- if (created.getCreator() == null) {
+ if (dataverse.getCreator() == null) {
final User user = getRequest().getUser();
if (user.isAuthenticated()) {
- created.setCreator((AuthenticatedUser) user);
+ dataverse.setCreator((AuthenticatedUser) user);
} else {
throw new IllegalCommandException("Guest users cannot create a Dataverse.", this);
}
}
- if (created.getDataverseType() == null) {
- created.setDataverseType(Dataverse.DataverseType.UNCATEGORIZED);
+ if (dataverse.getDataverseType() == null) {
+ dataverse.setDataverseType(Dataverse.DataverseType.UNCATEGORIZED);
}
- if (created.getDefaultContributorRole() == null) {
- created.setDefaultContributorRole(ctxt.roles().findBuiltinRoleByAlias(DataverseRole.EDITOR));
+ if (dataverse.getDefaultContributorRole() == null) {
+ dataverse.setDefaultContributorRole(ctxt.roles().findBuiltinRoleByAlias(DataverseRole.EDITOR));
}
// @todo for now we are saying all dataverses are permission root
- created.setPermissionRoot(true);
+ dataverse.setPermissionRoot(true);
- if (ctxt.dataverses().findByAlias(created.getAlias()) != null) {
- throw new IllegalCommandException("A dataverse with alias " + created.getAlias() + " already exists", this);
+ if (ctxt.dataverses().findByAlias(dataverse.getAlias()) != null) {
+ throw new IllegalCommandException("A dataverse with alias " + dataverse.getAlias() + " already exists", this);
}
- if (created.getFilePIDsEnabled() != null && !ctxt.settings().isTrueForKey(SettingsServiceBean.Key.AllowEnablingFilePIDsPerCollection, false)) {
+ if (dataverse.getFilePIDsEnabled() != null && !ctxt.settings().isTrueForKey(SettingsServiceBean.Key.AllowEnablingFilePIDsPerCollection, false)) {
throw new IllegalCommandException("File PIDs cannot be enabled per collection", this);
}
// Save the dataverse
- Dataverse managedDv = ctxt.dataverses().save(created);
+ Dataverse managedDv = ctxt.dataverses().save(dataverse);
// Find the built in admin role (currently by alias)
DataverseRole adminRole = ctxt.roles().findBuiltinRoleByAlias(DataverseRole.ADMIN);
@@ -160,33 +136,6 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
}
managedDv.setPermissionModificationTime(new Timestamp(new Date().getTime()));
-
- if (facetList != null) {
- ctxt.facets().deleteFacetsFor(managedDv);
-
- if (!facetList.isEmpty()) {
- managedDv.setFacetRoot(true);
- }
-
- int i = 0;
- for (DatasetFieldType df : facetList) {
- ctxt.facets().create(i++, df, managedDv);
- }
- }
-
- if (inputLevelList != null) {
- if (!inputLevelList.isEmpty()) {
- managedDv.addInputLevelsMetadataBlocksIfNotPresent(inputLevelList);
- }
- ctxt.fieldTypeInputLevels().deleteFacetsFor(managedDv);
- for (DataverseFieldTypeInputLevel inputLevel : inputLevelList) {
- inputLevel.setDataverse(managedDv);
- ctxt.fieldTypeInputLevels().create(inputLevel);
- }
- }
-
- // TODO: save is called here and above; we likely don't need both
- managedDv = ctxt.dataverses().save(managedDv);
return managedDv;
}
@@ -194,5 +143,4 @@ public Dataverse execute(CommandContext ctxt) throws CommandException {
public boolean onSuccess(CommandContext ctxt, Object r) {
return ctxt.dataverses().index((Dataverse) r);
}
-
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
index 3a21345448b..e9a2025b112 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDataFilesCommand.java
@@ -2,34 +2,29 @@
import edu.harvard.iq.dataverse.DataFile;
import edu.harvard.iq.dataverse.DatasetVersion;
+import edu.harvard.iq.dataverse.Dataverse;
import edu.harvard.iq.dataverse.authorization.Permission;
import edu.harvard.iq.dataverse.datasetutility.FileExceedsMaxSizeException;
import edu.harvard.iq.dataverse.datasetutility.FileSizeChecker;
-import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable;
import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
import edu.harvard.iq.dataverse.engine.command.CommandContext;
import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
-//import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
import edu.harvard.iq.dataverse.engine.command.exception.CommandExecutionException;
import edu.harvard.iq.dataverse.ingest.IngestServiceShapefileHelper;
-import edu.harvard.iq.dataverse.Dataverse;
import edu.harvard.iq.dataverse.storageuse.UploadSessionQuotaLimit;
-import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException;
import edu.harvard.iq.dataverse.util.BundleUtil;
import edu.harvard.iq.dataverse.util.FileUtil;
-import static edu.harvard.iq.dataverse.util.FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT;
-import static edu.harvard.iq.dataverse.util.FileUtil.createIngestFailureReport;
-import static edu.harvard.iq.dataverse.util.FileUtil.determineFileType;
-import static edu.harvard.iq.dataverse.util.FileUtil.determineFileTypeByNameAndExtension;
-import static edu.harvard.iq.dataverse.util.FileUtil.getFilesTempDirectory;
-import static edu.harvard.iq.dataverse.util.FileUtil.saveInputStreamInTempFile;
-import static edu.harvard.iq.dataverse.util.FileUtil.useRecognizedType;
import edu.harvard.iq.dataverse.util.ShapefileHandler;
import edu.harvard.iq.dataverse.util.StringUtil;
import edu.harvard.iq.dataverse.util.file.BagItFileHandler;
import edu.harvard.iq.dataverse.util.file.BagItFileHandlerFactory;
import edu.harvard.iq.dataverse.util.file.CreateDataFileResult;
+import edu.harvard.iq.dataverse.util.file.FileExceedsStorageQuotaException;
+import jakarta.enterprise.inject.spi.CDI;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang3.StringUtils;
+
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@@ -42,7 +37,7 @@
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Enumeration;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -51,12 +46,17 @@
import java.util.Set;
import java.util.logging.Logger;
import java.util.zip.GZIPInputStream;
-import java.util.zip.ZipFile;
import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-import jakarta.enterprise.inject.spi.CDI;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
+import java.util.zip.ZipFile;
+
+import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable;
+import static edu.harvard.iq.dataverse.util.FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT;
+import static edu.harvard.iq.dataverse.util.FileUtil.createIngestFailureReport;
+import static edu.harvard.iq.dataverse.util.FileUtil.determineFileType;
+import static edu.harvard.iq.dataverse.util.FileUtil.determineFileTypeByNameAndExtension;
+import static edu.harvard.iq.dataverse.util.FileUtil.getFilesTempDirectory;
+import static edu.harvard.iq.dataverse.util.FileUtil.saveInputStreamInTempFile;
+import static edu.harvard.iq.dataverse.util.FileUtil.useRecognizedType;
/**
*
@@ -93,6 +93,10 @@ public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion versi
this(aRequest, version, inputStream, fileName, suppliedContentType, newStorageIdentifier, quota, newCheckSum, newCheckSumType, null, null);
}
+ public CreateNewDataFilesCommand(DataverseRequest aRequest, DatasetVersion version, InputStream inputStream, String fileName, String suppliedContentType, String newStorageIdentifier, UploadSessionQuotaLimit quota, String newCheckSum, DataFile.ChecksumType newCheckSumType, Long newFileSize) {
+ this(aRequest, version, inputStream, fileName, suppliedContentType, newStorageIdentifier, quota, newCheckSum, newCheckSumType, newFileSize, null);
+ }
+
// This version of the command must be used when files are created in the
// context of creating a brand new dataset (from the Add Dataset page):
@@ -140,9 +144,10 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
if (newStorageIdentifier == null) {
- if (getFilesTempDirectory() != null) {
+ var filesTempDirectory = getFilesTempDirectory();
+ if (filesTempDirectory != null) {
try {
- tempFile = Files.createTempFile(Paths.get(getFilesTempDirectory()), "tmp", "upload");
+ tempFile = Files.createTempFile(Paths.get(filesTempDirectory), "tmp", "upload");
// "temporary" location is the key here; this is why we are not using
// the DataStore framework for this - the assumption is that
// temp files will always be stored on the local filesystem.
@@ -260,10 +265,6 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
// DataFile objects from its contents:
} else if (finalType.equals("application/zip")) {
- ZipFile zipFile = null;
- ZipInputStream unZippedIn = null;
- ZipEntry zipEntry = null;
-
int fileNumberLimit = ctxt.systemConfig().getZipUploadFilesLimit();
Long combinedUnzippedFileSize = 0L;
@@ -271,14 +272,14 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
Charset charset = null;
/*
TODO: (?)
- We may want to investigate somehow letting the user specify
+ We may want to investigate somehow letting the user specify
the charset for the filenames in the zip file...
- - otherwise, ZipInputStream bails out if it encounteres a file
- name that's not valid in the current charest (i.e., UTF-8, in
- our case). It would be a bit trickier than what we're doing for
- SPSS tabular ingests - with the lang. encoding pulldown menu -
+ - otherwise, ZipInputStream bails out if it encounteres a file
+ name that's not valid in the current charest (i.e., UTF-8, in
+ our case). It would be a bit trickier than what we're doing for
+ SPSS tabular ingests - with the lang. encoding pulldown menu -
because this encoding needs to be specified *before* we upload and
- attempt to unzip the file.
+ attempt to unzip the file.
-- L.A. 4.0 beta12
logger.info("default charset is "+Charset.defaultCharset().name());
if (Charset.isSupported("US-ASCII")) {
@@ -287,25 +288,21 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
if (charset != null) {
logger.info("was able to obtain charset for US-ASCII");
}
-
+
}
*/
- /**
- * Perform a quick check for how many individual files are
- * inside this zip archive. If it's above the limit, we can
- * give up right away, without doing any unpacking.
+ /**
+ * Perform a quick check for how many individual files are
+ * inside this zip archive. If it's above the limit, we can
+ * give up right away, without doing any unpacking.
* This should be a fairly inexpensive operation, we just need
- * to read the directory at the end of the file.
+ * to read the directory at the end of the file.
*/
-
- if (charset != null) {
- zipFile = new ZipFile(tempFile.toFile(), charset);
- } else {
- zipFile = new ZipFile(tempFile.toFile());
- }
+
+
/**
- * The ZipFile constructors above will throw ZipException -
+ * The ZipFile constructors in openZipFile will throw ZipException -
* a type of IOException - if there's something wrong
* with this file as a zip. There's no need to intercept it
* here, it will be caught further below, with other IOExceptions,
@@ -313,8 +310,8 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
* then attempt to save it as is.
*/
- int numberOfUnpackableFiles = 0;
-
+ int numberOfUnpackableFiles = 0;
+
/**
* Note that we can't just use zipFile.size(),
* unfortunately, since that's the total number of entries,
@@ -323,83 +320,46 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
* that are files.
*/
- for (Enumeration entries = zipFile.entries(); entries.hasMoreElements();) {
- ZipEntry entry = entries.nextElement();
- logger.fine("inside first zip pass; this entry: "+entry.getName());
- if (!entry.isDirectory()) {
- String shortName = entry.getName().replaceFirst("^.*[\\/]", "");
- // ... and, finally, check if it's a "fake" file - a zip archive entry
- // created for a MacOS X filesystem element: (these
- // start with "._")
- if (!shortName.startsWith("._") && !shortName.startsWith(".DS_Store") && !"".equals(shortName)) {
- numberOfUnpackableFiles++;
- if (numberOfUnpackableFiles > fileNumberLimit) {
- logger.warning("Zip upload - too many files in the zip to process individually.");
- warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit
- + "); please upload a zip archive with fewer files, if you want them to be ingested "
- + "as individual DataFiles.";
- throw new IOException();
- }
- // In addition to counting the files, we can
- // also check the file size while we're here,
- // provided the size limit is defined; if a single
- // file is above the individual size limit, unzipped,
- // we give up on unpacking this zip archive as well:
- if (fileSizeLimit != null && entry.getSize() > fileSizeLimit) {
- throw new FileExceedsMaxSizeException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(entry.getSize()), bytesToHumanReadable(fileSizeLimit)));
- }
- // Similarly, we want to check if saving all these unpacked
- // files is going to push the disk usage over the
- // quota:
- if (storageQuotaLimit != null) {
- combinedUnzippedFileSize = combinedUnzippedFileSize + entry.getSize();
- if (combinedUnzippedFileSize > storageQuotaLimit) {
- //throw new FileExceedsStorageQuotaException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(combinedUnzippedFileSize), bytesToHumanReadable(storageQuotaLimit)));
- // change of plans: if the unzipped content inside exceeds the remaining quota,
- // we reject the upload outright, rather than accepting the zip
- // file as is.
- throw new CommandExecutionException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.unzipped.quota_exceeded"), bytesToHumanReadable(storageQuotaLimit)), this);
- }
+ try (var zipFile = openZipFile(tempFile, charset)) {
+ var zipEntries = filteredZipEntries(zipFile);
+ for (var entry : zipEntries) {
+ logger.fine("inside first zip pass; this entry: " + entry.getName());
+ numberOfUnpackableFiles++;
+ if (numberOfUnpackableFiles > fileNumberLimit) {
+ logger.warning("Zip upload - too many files in the zip to process individually.");
+ warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit
+ + "); please upload a zip archive with fewer files, if you want them to be ingested "
+ + "as individual DataFiles.";
+ throw new IOException();
+ }
+ // In addition to counting the files, we can
+ // also check the file size while we're here,
+ // provided the size limit is defined; if a single
+ // file is above the individual size limit, unzipped,
+ // we give up on unpacking this zip archive as well:
+ if (fileSizeLimit != null && entry.getSize() > fileSizeLimit) {
+ throw new FileExceedsMaxSizeException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit"), bytesToHumanReadable(entry.getSize()), bytesToHumanReadable(fileSizeLimit)));
+ }
+ // Similarly, we want to check if saving all these unpacked
+ // files is going to push the disk usage over the
+ // quota:
+ if (storageQuotaLimit != null) {
+ combinedUnzippedFileSize = combinedUnzippedFileSize + entry.getSize();
+ if (combinedUnzippedFileSize > storageQuotaLimit) {
+ //throw new FileExceedsStorageQuotaException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.quota_exceeded"), bytesToHumanReadable(combinedUnzippedFileSize), bytesToHumanReadable(storageQuotaLimit)));
+ // change of plans: if the unzipped content inside exceeds the remaining quota,
+ // we reject the upload outright, rather than accepting the zip
+ // file as is.
+ throw new CommandExecutionException(MessageFormat.format(BundleUtil.getStringFromBundle("file.addreplace.error.unzipped.quota_exceeded"), bytesToHumanReadable(storageQuotaLimit)), this);
}
}
}
- }
-
- // OK we're still here - that means we can proceed unzipping.
-
- // Close the ZipFile, re-open as ZipInputStream:
- zipFile.close();
- // reset:
- combinedUnzippedFileSize = 0L;
-
- if (charset != null) {
- unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()), charset);
- } else {
- unZippedIn = new ZipInputStream(new FileInputStream(tempFile.toFile()));
- }
-
- while (true) {
- try {
- zipEntry = unZippedIn.getNextEntry();
- } catch (IllegalArgumentException iaex) {
- // Note:
- // ZipInputStream documentation doesn't even mention that
- // getNextEntry() throws an IllegalArgumentException!
- // but that's what happens if the file name of the next
- // entry is not valid in the current CharSet.
- // -- L.A.
- warningMessage = "Failed to unpack Zip file. (Unknown Character Set used in a file name?) Saving the file as is.";
- logger.warning(warningMessage);
- throw new IOException();
- }
+ // OK we're still here - that means we can proceed unzipping.
- if (zipEntry == null) {
- break;
- }
- // Note that some zip entries may be directories - we
- // simply skip them:
+ // reset:
+ combinedUnzippedFileSize = 0L;
- if (!zipEntry.isDirectory()) {
+ for (var entry : zipEntries) {
if (datafiles.size() > fileNumberLimit) {
logger.warning("Zip upload - too many files.");
warningMessage = "The number of files in the zip archive is over the limit (" + fileNumberLimit
@@ -407,72 +367,55 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
+ "as individual DataFiles.";
throw new IOException();
}
-
- String fileEntryName = zipEntry.getName();
+ var fileEntryName = entry.getName();
+ var shortName = getShortName(fileEntryName);
logger.fine("ZipEntry, file: " + fileEntryName);
+ String storageIdentifier = FileUtil.generateStorageIdentifier();
+ File unzippedFile = new File(getFilesTempDirectory() + "/" + storageIdentifier);
+ Files.copy(zipFile.getInputStream(entry), unzippedFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ // No need to check the size of this unpacked file against the size limit,
+ // since we've already checked for that in the first pass.
+ DataFile datafile = FileUtil.createSingleDataFile(version, null, storageIdentifier, shortName,
+ MIME_TYPE_UNDETERMINED_DEFAULT,
+ ctxt.systemConfig().getFileFixityChecksumAlgorithm(), null, false);
+
+ if (!fileEntryName.equals(shortName)) {
+ // If the filename looks like a hierarchical folder name (i.e., contains slashes and backslashes),
+ // we'll extract the directory name; then subject it to some "aggressive sanitizing" - strip all
+ // the leading, trailing and duplicate slashes; then replace all the characters that
+ // don't pass our validation rules.
+ String directoryName = fileEntryName.replaceFirst("[\\\\/][\\\\/]*[^\\\\/]*$", "");
+ directoryName = StringUtil.sanitizeFileDirectory(directoryName, true);
+ // if (!"".equals(directoryName)) {
+ if (!StringUtil.isEmpty(directoryName)) {
+ logger.fine("setting the directory label to " + directoryName);
+ datafile.getFileMetadata().setDirectoryLabel(directoryName);
+ }
+ }
- if (fileEntryName != null && !fileEntryName.equals("")) {
-
- String shortName = fileEntryName.replaceFirst("^.*[\\/]", "");
-
- // Check if it's a "fake" file - a zip archive entry
- // created for a MacOS X filesystem element: (these
- // start with "._")
- if (!shortName.startsWith("._") && !shortName.startsWith(".DS_Store") && !"".equals(shortName)) {
- // OK, this seems like an OK file entry - we'll try
- // to read it and create a DataFile with it:
-
- String storageIdentifier = FileUtil.generateStorageIdentifier();
- File unzippedFile = new File(getFilesTempDirectory() + "/" + storageIdentifier);
- Files.copy(unZippedIn, unzippedFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
- // No need to check the size of this unpacked file against the size limit,
- // since we've already checked for that in the first pass.
-
- DataFile datafile = FileUtil.createSingleDataFile(version, null, storageIdentifier, shortName,
- MIME_TYPE_UNDETERMINED_DEFAULT,
- ctxt.systemConfig().getFileFixityChecksumAlgorithm(), null, false);
-
- if (!fileEntryName.equals(shortName)) {
- // If the filename looks like a hierarchical folder name (i.e., contains slashes and backslashes),
- // we'll extract the directory name; then subject it to some "aggressive sanitizing" - strip all
- // the leading, trailing and duplicate slashes; then replace all the characters that
- // don't pass our validation rules.
- String directoryName = fileEntryName.replaceFirst("[\\\\/][\\\\/]*[^\\\\/]*$", "");
- directoryName = StringUtil.sanitizeFileDirectory(directoryName, true);
- // if (!"".equals(directoryName)) {
- if (!StringUtil.isEmpty(directoryName)) {
- logger.fine("setting the directory label to " + directoryName);
- datafile.getFileMetadata().setDirectoryLabel(directoryName);
- }
- }
+ if (datafile != null) {
+ // We have created this datafile with the mime type "unknown";
+ // Now that we have it saved in a temporary location,
+ // let's try and determine its real type:
- if (datafile != null) {
- // We have created this datafile with the mime type "unknown";
- // Now that we have it saved in a temporary location,
- // let's try and determine its real type:
-
- String tempFileName = getFilesTempDirectory() + "/" + datafile.getStorageIdentifier();
-
- try {
- recognizedType = determineFileType(unzippedFile, shortName);
- // null the File explicitly, to release any open FDs:
- unzippedFile = null;
- logger.fine("File utility recognized unzipped file as " + recognizedType);
- if (recognizedType != null && !recognizedType.equals("")) {
- datafile.setContentType(recognizedType);
- }
- } catch (Exception ex) {
- logger.warning("Failed to run the file utility mime type check on file " + fileName);
- }
-
- datafiles.add(datafile);
- combinedUnzippedFileSize += datafile.getFilesize();
+ String tempFileName = getFilesTempDirectory() + "/" + datafile.getStorageIdentifier();
+
+ try {
+ recognizedType = determineFileType(unzippedFile, shortName);
+ // null the File explicitly, to release any open FDs:
+ unzippedFile = null;
+ logger.fine("File utility recognized unzipped file as " + recognizedType);
+ if (recognizedType != null && !recognizedType.equals("")) {
+ datafile.setContentType(recognizedType);
}
+ } catch (Exception ex) {
+ logger.warning("Failed to run the file utility mime type check on file " + fileName);
}
+
+ datafiles.add(datafile);
+ combinedUnzippedFileSize += datafile.getFilesize();
}
}
- unZippedIn.closeEntry();
-
}
} catch (IOException ioex) {
@@ -494,18 +437,7 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
//warningMessage = BundleUtil.getStringFromBundle("file.addreplace.warning.unzip.failed.quota", Arrays.asList(FileSizeChecker.bytesToHumanReadable(storageQuotaLimit)));
//datafiles.clear();
throw new CommandExecutionException(fesqx.getMessage(), fesqx, this);
- }*/ finally {
- if (zipFile != null) {
- try {
- zipFile.close();
- } catch (Exception zEx) {}
- }
- if (unZippedIn != null) {
- try {
- unZippedIn.close();
- } catch (Exception zEx) {}
- }
- }
+ }*/
if (!datafiles.isEmpty()) {
// remove the uploaded zip file:
try {
@@ -591,7 +523,8 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
// The try-catch is due to error encountered in using NFS for stocking file,
// cf. https://github.com/IQSS/dataverse/issues/5909
try {
- FileUtils.deleteDirectory(rezipFolder);
+ if (rezipFolder!=null)
+ FileUtils.deleteDirectory(rezipFolder);
} catch (IOException ioex) {
// do nothing - it's a temp folder.
logger.warning("Could not remove temp folder, error message : " + ioex.getMessage());
@@ -730,7 +663,37 @@ public CreateDataFileResult execute(CommandContext ctxt) throws CommandException
return CreateDataFileResult.error(fileName, finalType);
} // end createDataFiles
-
+
+ private static List filteredZipEntries(ZipFile zipFile) {
+ var entries = Collections.list(zipFile.entries()).stream().filter(e -> {
+ var entryName = e.getName();
+ logger.fine("ZipEntry, file: " + entryName);
+ return !e.isDirectory() && !entryName.isEmpty() && !isFileToSkip(entryName);
+ }).toList();
+ return entries;
+ }
+
+ private static ZipFile openZipFile(Path tempFile, Charset charset) throws IOException {
+ if (charset != null) {
+ return new ZipFile(tempFile.toFile(), charset);
+ }
+ else {
+ return new ZipFile(tempFile.toFile());
+ }
+ }
+
+ private static boolean isFileToSkip(String fileName) {
+ // check if it's a "fake" file - a zip archive entry
+ // created for a MacOS X filesystem element: (these
+ // start with "._")
+ var shortName = getShortName(fileName);
+ return shortName.startsWith("._") || shortName.startsWith(".DS_Store") || "".equals(shortName);
+ }
+
+ private static String getShortName(String fileName) {
+ return fileName.replaceFirst("^.*[\\/]", "");
+ }
+
@Override
public Map> getRequiredPermissions() {
Map> ret = new HashMap<>();
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java
index e6e8279a314..e378e2e2ef7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CuratePublishedDatasetVersionCommand.java
@@ -131,7 +131,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException {
throw new IllegalCommandException(BundleUtil.getStringFromBundle("datasetversion.update.failure"), this);
} else {
- metadataUpdated = DatasetVersionDifference.compareFileMetadatas(publishedFmd, draftFmd);
+ metadataUpdated = !DatasetVersionDifference.compareFileMetadatas(publishedFmd, draftFmd).isEmpty();
publishedFmd.setLabel(draftFmd.getLabel());
publishedFmd.setDescription(draftFmd.getDescription());
publishedFmd.setCategories(draftFmd.getCategories());
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
index 1ac41105237..902bea7f833 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/PublishDatasetCommand.java
@@ -11,6 +11,9 @@
import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
import edu.harvard.iq.dataverse.workflow.Workflow;
import edu.harvard.iq.dataverse.workflow.WorkflowContext.TriggerType;
+
+import jakarta.persistence.OptimisticLockException;
+
import java.util.Optional;
import java.util.logging.Logger;
import static java.util.stream.Collectors.joining;
@@ -105,10 +108,15 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException
Optional prePubWf = ctxt.workflows().getDefaultWorkflow(TriggerType.PrePublishDataset);
if ( prePubWf.isPresent() ) {
// We start a workflow
- theDataset = ctxt.em().merge(theDataset);
- ctxt.em().flush();
- ctxt.workflows().start(prePubWf.get(), buildContext(theDataset, TriggerType.PrePublishDataset, datasetExternallyReleased), true);
- return new PublishDatasetResult(theDataset, Status.Workflow);
+ try {
+ theDataset = ctxt.em().merge(theDataset);
+ ctxt.em().flush();
+ ctxt.workflows().start(prePubWf.get(),
+ buildContext(theDataset, TriggerType.PrePublishDataset, datasetExternallyReleased), true);
+ return new PublishDatasetResult(theDataset, Status.Workflow);
+ } catch (OptimisticLockException e) {
+ throw new CommandException(e.getMessage(), e, this);
+ }
} else{
// We will skip trying to register the global identifiers for datafiles
@@ -157,7 +165,12 @@ public PublishDatasetResult execute(CommandContext ctxt) throws CommandException
lock.setInfo(info);
ctxt.datasets().addDatasetLock(theDataset, lock);
}
- theDataset = ctxt.em().merge(theDataset);
+ try {
+ theDataset = ctxt.em().merge(theDataset);
+ } catch (OptimisticLockException e) {
+ ctxt.datasets().removeDatasetLocks(theDataset, DatasetLock.Reason.finalizePublication);
+ throw new CommandException(e.getMessage(), e, this);
+ }
// The call to FinalizePublicationCommand has been moved to the new @onSuccess()
// method:
//ctxt.datasets().callFinalizePublishCommandAsynchronously(theDataset.getId(), ctxt, request, datasetExternallyReleased);
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseAttributeCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseAttributeCommand.java
new file mode 100644
index 00000000000..57ac20fcee6
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseAttributeCommand.java
@@ -0,0 +1,110 @@
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
+import edu.harvard.iq.dataverse.engine.command.exception.PermissionException;
+import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
+
+import java.util.Collections;
+
+/**
+ * Command to update an existing Dataverse attribute.
+ */
+@RequiredPermissions(Permission.EditDataverse)
+public class UpdateDataverseAttributeCommand extends AbstractCommand {
+
+ private static final String ATTRIBUTE_ALIAS = "alias";
+ private static final String ATTRIBUTE_NAME = "name";
+ private static final String ATTRIBUTE_DESCRIPTION = "description";
+ private static final String ATTRIBUTE_AFFILIATION = "affiliation";
+ private static final String ATTRIBUTE_FILE_PIDS_ENABLED = "filePIDsEnabled";
+
+ private final Dataverse dataverse;
+ private final String attributeName;
+ private final Object attributeValue;
+
+ public UpdateDataverseAttributeCommand(DataverseRequest request, Dataverse dataverse, String attributeName, Object attributeValue) {
+ super(request, dataverse);
+ this.dataverse = dataverse;
+ this.attributeName = attributeName;
+ this.attributeValue = attributeValue;
+ }
+
+ @Override
+ public Dataverse execute(CommandContext ctxt) throws CommandException {
+ switch (attributeName) {
+ case ATTRIBUTE_ALIAS:
+ case ATTRIBUTE_NAME:
+ case ATTRIBUTE_DESCRIPTION:
+ case ATTRIBUTE_AFFILIATION:
+ setStringAttribute(attributeName, attributeValue);
+ break;
+ case ATTRIBUTE_FILE_PIDS_ENABLED:
+ setBooleanAttributeForFilePIDs(ctxt);
+ break;
+ default:
+ throw new IllegalCommandException("'" + attributeName + "' is not a supported attribute", this);
+ }
+
+ return ctxt.engine().submit(new UpdateDataverseCommand(dataverse, null, null, getRequest(), null));
+ }
+
+ /**
+ * Helper method to set a string attribute.
+ *
+ * @param attributeName The name of the attribute.
+ * @param attributeValue The value of the attribute (must be a String).
+ * @throws IllegalCommandException if the provided attribute value is not of String type.
+ */
+ private void setStringAttribute(String attributeName, Object attributeValue) throws IllegalCommandException {
+ if (!(attributeValue instanceof String stringValue)) {
+ throw new IllegalCommandException("'" + attributeName + "' requires a string value", this);
+ }
+
+ switch (attributeName) {
+ case ATTRIBUTE_ALIAS:
+ dataverse.setAlias(stringValue);
+ break;
+ case ATTRIBUTE_NAME:
+ dataverse.setName(stringValue);
+ break;
+ case ATTRIBUTE_DESCRIPTION:
+ dataverse.setDescription(stringValue);
+ break;
+ case ATTRIBUTE_AFFILIATION:
+ dataverse.setAffiliation(stringValue);
+ break;
+ default:
+ throw new IllegalCommandException("Unsupported string attribute: " + attributeName, this);
+ }
+ }
+
+ /**
+ * Helper method to handle the "filePIDsEnabled" boolean attribute.
+ *
+ * @param ctxt The command context.
+ * @throws PermissionException if the user doesn't have permission to modify this attribute.
+ */
+ private void setBooleanAttributeForFilePIDs(CommandContext ctxt) throws CommandException {
+ if (!getRequest().getUser().isSuperuser()) {
+ throw new PermissionException("You must be a superuser to change this setting",
+ this, Collections.singleton(Permission.EditDataset), dataverse);
+ }
+ if (!ctxt.settings().isTrueForKey(SettingsServiceBean.Key.AllowEnablingFilePIDsPerCollection, false)) {
+ throw new PermissionException("Changing File PID policy per collection is not enabled on this server",
+ this, Collections.singleton(Permission.EditDataset), dataverse);
+ }
+
+ if (!(attributeValue instanceof Boolean)) {
+ throw new IllegalCommandException("'" + ATTRIBUTE_FILE_PIDS_ENABLED + "' requires a boolean value", this);
+ }
+
+ dataverse.setFilePIDsEnabled((Boolean) attributeValue);
+ }
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
index bdb69dc918f..6dc4ab4d00d 100644
--- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDataverseCommand.java
@@ -1,141 +1,144 @@
package edu.harvard.iq.dataverse.engine.command.impl;
-import edu.harvard.iq.dataverse.Dataset;
-import edu.harvard.iq.dataverse.DatasetFieldType;
-import edu.harvard.iq.dataverse.Dataverse;
+import edu.harvard.iq.dataverse.*;
import edu.harvard.iq.dataverse.Dataverse.DataverseType;
-import edu.harvard.iq.dataverse.DataverseFieldTypeInputLevel;
+import edu.harvard.iq.dataverse.api.dto.DataverseDTO;
import edu.harvard.iq.dataverse.authorization.Permission;
import static edu.harvard.iq.dataverse.dataverse.DataverseUtil.validateDataverseMetadataExternally;
-import edu.harvard.iq.dataverse.engine.command.AbstractCommand;
+
import edu.harvard.iq.dataverse.engine.command.CommandContext;
import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
-import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
import java.util.ArrayList;
import java.util.List;
-import java.util.logging.Logger;
/**
* Update an existing dataverse.
+ *
* @author michael
*/
-@RequiredPermissions( Permission.EditDataverse )
-public class UpdateDataverseCommand extends AbstractCommand {
- private static final Logger logger = Logger.getLogger(UpdateDataverseCommand.class.getName());
-
- private final Dataverse editedDv;
- private final List facetList;
+@RequiredPermissions(Permission.EditDataverse)
+public class UpdateDataverseCommand extends AbstractWriteDataverseCommand {
private final List featuredDataverseList;
- private final List inputLevelList;
+ private final DataverseDTO updatedDataverseDTO;
private boolean datasetsReindexRequired = false;
- public UpdateDataverseCommand(Dataverse editedDv, List facetList, List featuredDataverseList,
- DataverseRequest aRequest, List inputLevelList ) {
- super(aRequest, editedDv);
- this.editedDv = editedDv;
- // add update template uses this command but does not
- // update facet list or featured dataverses
- if (facetList != null){
- this.facetList = new ArrayList<>(facetList);
- } else {
- this.facetList = null;
- }
- if (featuredDataverseList != null){
- this.featuredDataverseList = new ArrayList<>(featuredDataverseList);
- } else {
- this.featuredDataverseList = null;
- }
- if (inputLevelList != null){
- this.inputLevelList = new ArrayList<>(inputLevelList);
- } else {
- this.inputLevelList = null;
- }
- }
-
- @Override
- public Dataverse execute(CommandContext ctxt) throws CommandException {
- logger.fine("Entering update dataverse command");
-
- // Perform any optional validation steps, if defined:
- if (ctxt.systemConfig().isExternalDataverseValidationEnabled()) {
- // For admins, an override of the external validation step may be enabled:
- if (!(getUser().isSuperuser() && ctxt.systemConfig().isExternalValidationAdminOverrideEnabled())) {
- String executable = ctxt.systemConfig().getDataverseValidationExecutable();
- boolean result = validateDataverseMetadataExternally(editedDv, executable, getRequest());
-
- if (!result) {
- String rejectionMessage = ctxt.systemConfig().getDataverseUpdateValidationFailureMsg();
- throw new IllegalCommandException(rejectionMessage, this);
- }
- }
- }
-
- Dataverse oldDv = ctxt.dataverses().find(editedDv.getId());
-
- DataverseType oldDvType = oldDv.getDataverseType();
- String oldDvAlias = oldDv.getAlias();
- String oldDvName = oldDv.getName();
- oldDv = null;
-
- Dataverse result = ctxt.dataverses().save(editedDv);
-
- if ( facetList != null ) {
- ctxt.facets().deleteFacetsFor(result);
- int i=0;
- for ( DatasetFieldType df : facetList ) {
- ctxt.facets().create(i++, df.getId(), result.getId());
- }
- }
- if ( featuredDataverseList != null ) {
- ctxt.featuredDataverses().deleteFeaturedDataversesFor(result);
- int i=0;
- for ( Object obj : featuredDataverseList ) {
- Dataverse dv = (Dataverse) obj;
- ctxt.featuredDataverses().create(i++, dv.getId(), result.getId());
+ public UpdateDataverseCommand(Dataverse dataverse,
+ List facets,
+ List featuredDataverses,
+ DataverseRequest request,
+ List inputLevels) {
+ this(dataverse, facets, featuredDataverses, request, inputLevels, null, null, false);
+ }
+
+ public UpdateDataverseCommand(Dataverse dataverse,
+ List facets,
+ List featuredDataverses,
+ DataverseRequest request,
+ List inputLevels,
+ List metadataBlocks,
+ DataverseDTO updatedDataverseDTO,
+ boolean resetRelationsOnNullValues) {
+ super(dataverse, dataverse, request, facets, inputLevels, metadataBlocks, resetRelationsOnNullValues);
+ if (featuredDataverses != null) {
+ this.featuredDataverseList = new ArrayList<>(featuredDataverses);
+ } else {
+ this.featuredDataverseList = null;
+ }
+ this.updatedDataverseDTO = updatedDataverseDTO;
+ }
+
+ @Override
+ protected Dataverse innerExecute(CommandContext ctxt) throws IllegalCommandException {
+ // Perform any optional validation steps, if defined:
+ if (ctxt.systemConfig().isExternalDataverseValidationEnabled()) {
+ // For admins, an override of the external validation step may be enabled:
+ if (!(getUser().isSuperuser() && ctxt.systemConfig().isExternalValidationAdminOverrideEnabled())) {
+ String executable = ctxt.systemConfig().getDataverseValidationExecutable();
+ boolean result = validateDataverseMetadataExternally(dataverse, executable, getRequest());
+
+ if (!result) {
+ String rejectionMessage = ctxt.systemConfig().getDataverseUpdateValidationFailureMsg();
+ throw new IllegalCommandException(rejectionMessage, this);
}
}
- if ( inputLevelList != null ) {
- ctxt.fieldTypeInputLevels().deleteFacetsFor(result);
- for ( DataverseFieldTypeInputLevel obj : inputLevelList ) {
- ctxt.fieldTypeInputLevels().create(obj);
- }
+ }
+
+ Dataverse oldDv = ctxt.dataverses().find(dataverse.getId());
+
+ DataverseType oldDvType = oldDv.getDataverseType();
+ String oldDvAlias = oldDv.getAlias();
+ String oldDvName = oldDv.getName();
+
+ // We don't want to reindex the children datasets unnecessarily:
+ // When these values are changed we need to reindex all children datasets
+ // This check is not recursive as all the values just report the immediate parent
+ if (!oldDvType.equals(dataverse.getDataverseType())
+ || !oldDvName.equals(dataverse.getName())
+ || !oldDvAlias.equals(dataverse.getAlias())) {
+ datasetsReindexRequired = true;
+ }
+
+ if (featuredDataverseList != null) {
+ ctxt.featuredDataverses().deleteFeaturedDataversesFor(dataverse);
+ int i = 0;
+ for (Object obj : featuredDataverseList) {
+ Dataverse dv = (Dataverse) obj;
+ ctxt.featuredDataverses().create(i++, dv.getId(), dataverse.getId());
}
-
- // We don't want to reindex the children datasets unnecessarily:
- // When these values are changed we need to reindex all children datasets
- // This check is not recursive as all the values just report the immediate parent
- if (!oldDvType.equals(editedDv.getDataverseType())
- || !oldDvName.equals(editedDv.getName())
- || !oldDvAlias.equals(editedDv.getAlias())) {
- datasetsReindexRequired = true;
+ }
+
+ if (updatedDataverseDTO != null) {
+ updateDataverseFromDTO(dataverse, updatedDataverseDTO);
+ }
+
+ return dataverse;
+ }
+
+ private void updateDataverseFromDTO(Dataverse dataverse, DataverseDTO dto) {
+ if (dto.getAlias() != null) {
+ dataverse.setAlias(dto.getAlias());
+ }
+ if (dto.getName() != null) {
+ dataverse.setName(dto.getName());
+ }
+ if (dto.getDescription() != null) {
+ dataverse.setDescription(dto.getDescription());
+ }
+ if (dto.getAffiliation() != null) {
+ dataverse.setAffiliation(dto.getAffiliation());
+ }
+ if (dto.getDataverseContacts() != null) {
+ dataverse.setDataverseContacts(dto.getDataverseContacts());
+ for (DataverseContact dc : dataverse.getDataverseContacts()) {
+ dc.setDataverse(dataverse);
}
-
- return result;
- }
-
+ }
+ if (dto.getDataverseType() != null) {
+ dataverse.setDataverseType(dto.getDataverseType());
+ }
+ }
+
@Override
public boolean onSuccess(CommandContext ctxt, Object r) {
-
+
// first kick of async index of datasets
// TODO: is this actually needed? Is there a better way to handle
// It appears that we at some point lost some extra logic here, where
// we only reindex the underlying datasets if one or more of the specific set
- // of fields have been changed (since these values are included in the
- // indexed solr documents for dataasets). So I'm putting that back. -L.A.
+ // of fields have been changed (since these values are included in the
+ // indexed solr documents for datasets). So I'm putting that back. -L.A.
Dataverse result = (Dataverse) r;
-
+
if (datasetsReindexRequired) {
List datasets = ctxt.datasets().findByOwnerId(result.getId());
ctxt.index().asyncIndexDatasetList(datasets, true);
}
-
- return ctxt.dataverses().index((Dataverse) r);
- }
+ return ctxt.dataverses().index((Dataverse) r);
+ }
}
-
diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java
new file mode 100644
index 00000000000..09563686299
--- /dev/null
+++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateHarvestedDatasetCommand.java
@@ -0,0 +1,202 @@
+package edu.harvard.iq.dataverse.engine.command.impl;
+
+import edu.harvard.iq.dataverse.authorization.Permission;
+import edu.harvard.iq.dataverse.engine.command.CommandContext;
+import edu.harvard.iq.dataverse.engine.command.DataverseRequest;
+import edu.harvard.iq.dataverse.engine.command.RequiredPermissions;
+import edu.harvard.iq.dataverse.engine.command.exception.CommandException;
+import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException;
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.DatasetVersion;
+import edu.harvard.iq.dataverse.DataFile;
+import edu.harvard.iq.dataverse.FileMetadata;
+import static edu.harvard.iq.dataverse.search.IndexServiceBean.solrDocIdentifierFile;
+import edu.harvard.iq.dataverse.util.StringUtil;
+import java.io.IOException;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.apache.solr.client.solrj.SolrServerException;
+
+/**
+ *
+ * @author landreev
+ *
+ * Much simplified version of UpdateDatasetVersionCommand,
+ * but with some extra twists. The goal is to avoid creating new Dataset and
+ * DataFile objects, and to instead preserve the database ids of the re-harvested
+ * datasets and files, whenever possible. This in turn allows us to avoid deleting
+ * and rebuilding from scratch the Solr documents for these objects.
+ */
+@RequiredPermissions(Permission.EditDataset)
+public class UpdateHarvestedDatasetCommand extends AbstractDatasetCommand {
+
+ private static final Logger logger = Logger.getLogger(UpdateHarvestedDatasetCommand.class.getCanonicalName());
+ private final DatasetVersion newHarvestedVersion;
+ final private boolean validateLenient = true;
+
+ public UpdateHarvestedDatasetCommand(Dataset theDataset, DatasetVersion newHarvestedVersion, DataverseRequest aRequest) {
+ super(aRequest, theDataset);
+ this.newHarvestedVersion = newHarvestedVersion;
+ }
+
+ public boolean isValidateLenient() {
+ return validateLenient;
+ }
+
+ @Override
+ public Dataset execute(CommandContext ctxt) throws CommandException {
+
+ Dataset existingDataset = getDataset();
+
+ if (existingDataset == null
+ || existingDataset.getId() == null
+ || !existingDataset.isHarvested()
+ || existingDataset.getVersions().size() != 1) {
+ throw new IllegalCommandException("The command can only be called on an existing harvested dataset with only 1 version", this);
+ }
+ DatasetVersion existingVersion = existingDataset.getVersions().get(0);
+
+ if (newHarvestedVersion == null || newHarvestedVersion.getId() != null) {
+ throw new IllegalCommandException("The command can only be called with a newly-harvested, not yet saved DatasetVersion supplied", this);
+ }
+
+ newHarvestedVersion.setCreateTime(getTimestamp());
+ newHarvestedVersion.setLastUpdateTime(getTimestamp());
+
+
+ Map existingFilesIndex = new HashMap<>();
+
+ /*
+ Create a map of the files that are currently part of this existing
+ harvested dataset. We assume that a harvested file can be uniquely
+ defined by its storageidentifier. Which, in the case of a datafile
+ harvested from another Dataverse should be its data access api url.
+ */
+ for (int i = 0; i < existingDataset.getFiles().size(); i++) {
+ String storageIdentifier = existingDataset.getFiles().get(i).getStorageIdentifier();
+ if (!StringUtil.isEmpty(storageIdentifier)) {
+ existingFilesIndex.put(storageIdentifier, i);
+ }
+ }
+
+ /*
+ Go through the files in the newly-harvested version and check if any of
+ them are (potentially new/updated) versions of files that we already
+ have, harvested previously from the same archive location.
+ */
+ for (FileMetadata newFileMetadata : newHarvestedVersion.getFileMetadatas()) {
+ // is it safe to assume that each new FileMetadata will be
+ // pointing to a non-null DataFile here?
+ String storageIdentifier = newFileMetadata.getDataFile().getStorageIdentifier();
+ if (!StringUtil.isEmpty(storageIdentifier) && existingFilesIndex.containsKey(storageIdentifier)) {
+ newFileMetadata.getDataFile().setFileMetadatas(new ArrayList<>());
+
+ int fileIndex = existingFilesIndex.get(storageIdentifier);
+
+ // Make sure to update the existing DataFiles that we are going
+ // to keep in case their newly-harvested versions have different
+ // checksums, mime types etc. These values are supposed to be
+ // immutable, normally - but who knows, errors happen, the source
+ // Dataverse may have had to fix these in their database to
+ // correct a data integrity issue (for ex.):
+ existingDataset.getFiles().get(fileIndex).setContentType(newFileMetadata.getDataFile().getContentType());
+ existingDataset.getFiles().get(fileIndex).setFilesize(newFileMetadata.getDataFile().getFilesize());
+ existingDataset.getFiles().get(fileIndex).setChecksumType(newFileMetadata.getDataFile().getChecksumType());
+ existingDataset.getFiles().get(fileIndex).setChecksumValue(newFileMetadata.getDataFile().getChecksumValue());
+
+ // Point the newly-harvested filemetadata to the existing datafile:
+ newFileMetadata.setDataFile(existingDataset.getFiles().get(fileIndex));
+
+ // Make sure this new FileMetadata is the only one attached to this existing file:
+ existingDataset.getFiles().get(fileIndex).setFileMetadatas(new ArrayList<>(1));
+ existingDataset.getFiles().get(fileIndex).getFileMetadatas().add(newFileMetadata);
+ // (we don't want any cascade relationships left between this existing
+ // dataset and this version, since we are going to attemp to delete it).
+
+ // Drop the file from the index map:
+ existingFilesIndex.remove(storageIdentifier);
+ }
+ }
+
+ // @todo? check if there's anything special that needs to be done for things
+ // like file categories
+
+ List solrIdsOfDocumentsToDelete = new ArrayList<>();
+
+ // Go through the existing files and delete the ones that are
+ // no longer present in the version that we have just harvesed:
+ for (FileMetadata oldFileMetadata : existingDataset.getVersions().get(0).getFileMetadatas()) {
+ DataFile oldDataFile = oldFileMetadata.getDataFile();
+ String storageIdentifier = oldDataFile.getStorageIdentifier();
+ // Is it still in the existing files map? - that means it is no longer
+ // present in the newly-harvested version
+ if (StringUtil.isEmpty(storageIdentifier) || existingFilesIndex.containsKey(storageIdentifier)) {
+ solrIdsOfDocumentsToDelete.add(solrDocIdentifierFile + oldDataFile.getId());
+ existingDataset.getFiles().remove(oldDataFile);
+ // Files from harvested datasets are removed unceremoniously,
+ // directly in the database. No need to bother calling the
+ // DeleteFileCommand on them. We'll just need to remember to purge
+ // them from Solr as well (right below)
+ ctxt.em().remove(ctxt.em().merge(oldDataFile));
+ // (no need to explicitly remove the oldFileMetadata; it will be
+ // removed with the entire old version is deleted)
+ }
+ }
+
+ // purge all the SOLR documents associated with the files
+ // we have just deleted:
+ if (!solrIdsOfDocumentsToDelete.isEmpty()) {
+ ctxt.index().deleteHarvestedDocuments(solrIdsOfDocumentsToDelete);
+ }
+
+ // ... And now delete the existing version itself:
+ existingDataset.setVersions(new ArrayList<>());
+ existingVersion.setDataset(null);
+
+ existingVersion = ctxt.em().merge(existingVersion);
+ ctxt.em().remove(existingVersion);
+
+ // Now attach the newly-harvested version to the dataset:
+ existingDataset.getVersions().add(newHarvestedVersion);
+ newHarvestedVersion.setDataset(existingDataset);
+
+ // ... There's one more thing to do - go through the new files,
+ // that are not in the database yet, and make sure they are
+ // attached to this existing dataset, instead of the dummy temp
+ // dataset into which they were originally imported:
+ for (FileMetadata newFileMetadata : newHarvestedVersion.getFileMetadatas()) {
+ if (newFileMetadata.getDataFile().getId() == null) {
+ existingDataset.getFiles().add(newFileMetadata.getDataFile());
+ newFileMetadata.getDataFile().setOwner(existingDataset);
+ }
+ }
+
+ ctxt.em().persist(newHarvestedVersion);
+
+ Dataset savedDataset = ctxt.em().merge(existingDataset);
+ ctxt.em().flush();
+
+ return savedDataset;
+ }
+
+ @Override
+ public boolean onSuccess(CommandContext ctxt, Object r) {
+ boolean retVal = true;
+ Dataset d = (Dataset) r;
+
+ try {
+ // Note that we index harvested datasets synchronously:
+ ctxt.index().indexDataset(d, true);
+ } catch (SolrServerException|IOException solrServerEx) {
+ logger.log(Level.WARNING, "Exception while trying to index the updated Harvested dataset " + d.getGlobalId().asString(), solrServerEx.getMessage());
+ retVal = false;
+ }
+
+ return retVal;
+ }
+}
diff --git a/src/main/java/edu/harvard/iq/dataverse/export/DDIExportServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/export/DDIExportServiceBean.java
index edd01ae98a3..d76020cb8d8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/export/DDIExportServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/export/DDIExportServiceBean.java
@@ -98,8 +98,10 @@ public class DDIExportServiceBean {
public static final String LEVEL_FILE = "file";
public static final String NOTE_TYPE_UNF = "VDC:UNF";
public static final String NOTE_TYPE_TAG = "DATAVERSE:TAG";
+ public static final String NOTE_TYPE_FILEDESCRIPTION = "DATAVERSE:FILEDESC";
public static final String NOTE_SUBJECT_UNF = "Universal Numeric Fingerprint";
public static final String NOTE_SUBJECT_TAG = "Data File Tag";
+ public static final String NOTE_SUBJECT_FILEDESCRIPTION = "DataFile Description";
/*
* Internal service objects:
@@ -742,11 +744,6 @@ private void createFileDscr(XMLStreamWriter xmlw, Set excludedFieldSet,
xmlw.writeEndElement(); // fileName
}
- /*
- xmlw.writeStartElement("fileCont");
- xmlw.writeCharacters( df.getContentType() );
- xmlw.writeEndElement(); // fileCont
- */
// dimensions
if (checkField("dimensns", excludedFieldSet, includedFieldSet)) {
if (dt.getCaseQuantity() != null || dt.getVarQuantity() != null || dt.getRecordsPerCase() != null) {
@@ -801,26 +798,6 @@ private void createFileDscr(XMLStreamWriter xmlw, Set excludedFieldSet,
xmlw.writeEndElement(); // notes
}
- /*
- xmlw.writeStartElement("notes");
- writeAttribute( xmlw, "type", "vdc:category" );
- xmlw.writeCharacters( fm.getCategory() );
- xmlw.writeEndElement(); // notes
- */
- // A special note for LOCKSS crawlers indicating the restricted
- // status of the file:
-
- /*
- if (tdf != null && isRestrictedFile(tdf)) {
- xmlw.writeStartElement("notes");
- writeAttribute( xmlw, "type", NOTE_TYPE_LOCKSS_CRAWL );
- writeAttribute( xmlw, "level", LEVEL_FILE );
- writeAttribute( xmlw, "subject", NOTE_SUBJECT_LOCKSS_PERM );
- xmlw.writeCharacters( "restricted" );
- xmlw.writeEndElement(); // notes
-
- }
- */
if (checkField("tags", excludedFieldSet, includedFieldSet) && df.getTags() != null) {
for (int i = 0; i < df.getTags().size(); i++) {
xmlw.writeStartElement("notes");
@@ -831,6 +808,17 @@ private void createFileDscr(XMLStreamWriter xmlw, Set excludedFieldSet,
xmlw.writeEndElement(); // notes
}
}
+
+ // A dedicated node for the Description entry
+ if (!StringUtilisEmpty(fm.getDescription())) {
+ xmlw.writeStartElement("notes");
+ xmlw.writeAttribute("level", LEVEL_FILE);
+ xmlw.writeAttribute("type", NOTE_TYPE_FILEDESCRIPTION);
+ xmlw.writeAttribute("subject", NOTE_SUBJECT_FILEDESCRIPTION);
+ xmlw.writeCharacters(fm.getDescription());
+ xmlw.writeEndElement(); // notes
+ }
+
xmlw.writeEndElement(); // fileDscr
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
index f5efc448090..05ddbe83e78 100644
--- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java
@@ -14,8 +14,10 @@
import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.LEVEL_FILE;
import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.NOTE_SUBJECT_TAG;
import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.NOTE_SUBJECT_UNF;
+import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.NOTE_SUBJECT_FILEDESCRIPTION;
import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.NOTE_TYPE_TAG;
import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.NOTE_TYPE_UNF;
+import static edu.harvard.iq.dataverse.export.DDIExportServiceBean.NOTE_TYPE_FILEDESCRIPTION;
import edu.harvard.iq.dataverse.export.DDIExporter;
import edu.harvard.iq.dataverse.pidproviders.PidUtil;
import edu.harvard.iq.dataverse.settings.SettingsServiceBean;
@@ -1901,6 +1903,8 @@ private static void createFileDscr(XMLStreamWriter xmlw, JsonArray fileDetails)
xmlw.writeEndElement(); // notes
}
+ // If any tabular tags are present, each is formatted in a
+ // dedicated note:
if (fileJson.containsKey("tabularTags")) {
JsonArray tags = fileJson.getJsonArray("tabularTags");
for (int j = 0; j < tags.size(); j++) {
@@ -1912,6 +1916,17 @@ private static void createFileDscr(XMLStreamWriter xmlw, JsonArray fileDetails)
xmlw.writeEndElement(); // notes
}
}
+
+ // Adding a dedicated node for the description entry (for
+ // non-tabular files we format it under the field)
+ if (fileJson.containsKey("description")) {
+ xmlw.writeStartElement("notes");
+ xmlw.writeAttribute("level", LEVEL_FILE);
+ xmlw.writeAttribute("type", NOTE_TYPE_FILEDESCRIPTION);
+ xmlw.writeAttribute("subject", NOTE_SUBJECT_FILEDESCRIPTION);
+ xmlw.writeCharacters(fileJson.getString("description"));
+ xmlw.writeEndElement(); // notes
+ }
// TODO: add the remaining fileDscr elements!
xmlw.writeEndElement(); // fileDscr
diff --git a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
index ac3c81622fc..58992805dc8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/globus/GlobusServiceBean.java
@@ -74,6 +74,7 @@
import edu.harvard.iq.dataverse.util.URLTokenUtil;
import edu.harvard.iq.dataverse.util.UrlSignerUtil;
import edu.harvard.iq.dataverse.util.json.JsonUtil;
+import jakarta.json.JsonNumber;
import jakarta.json.JsonReader;
import jakarta.persistence.EntityManager;
import jakarta.persistence.PersistenceContext;
@@ -284,6 +285,52 @@ private int makeDir(GlobusEndpoint endpoint, String dir) {
return result.status;
}
+ private Map lookupFileSizes(GlobusEndpoint endpoint, String dir) {
+ MakeRequestResponse result;
+
+ try {
+ logger.fine("Attempting to look up the contents of the Globus folder "+dir);
+ URL url = new URL(
+ "https://transfer.api.globusonline.org/v0.10/operation/endpoint/" + endpoint.getId()
+ + "/ls?path=" + dir);
+ result = makeRequest(url, "Bearer", endpoint.getClientToken(), "GET", null);
+
+ switch (result.status) {
+ case 200:
+ logger.fine("Looked up directory " + dir + " successfully.");
+ break;
+ default:
+ logger.warning("Status " + result.status + " received when looking up dir " + dir);
+ logger.fine("Response: " + result.jsonResponse);
+ return null;
+ }
+ } catch (MalformedURLException ex) {
+ // Misconfiguration
+ logger.warning("Failed to list the contents of the directory "+ dir + " on endpoint " + endpoint.getId());
+ return null;
+ }
+
+ Map ret = new HashMap<>();
+
+ JsonObject listObject = JsonUtil.getJsonObject(result.jsonResponse);
+ JsonArray dataArray = listObject.getJsonArray("DATA");
+
+ if (dataArray != null && !dataArray.isEmpty()) {
+ for (int i = 0; i < dataArray.size(); i++) {
+ String dataType = dataArray.getJsonObject(i).getString("DATA_TYPE", null);
+ if (dataType != null && dataType.equals("file")) {
+ // is it safe to assume that any entry with a valid "DATA_TYPE": "file"
+ // will also have valid "name" and "size" entries?
+ String fileName = dataArray.getJsonObject(i).getString("name");
+ long fileSize = dataArray.getJsonObject(i).getJsonNumber("size").longValueExact();
+ ret.put(fileName, fileSize);
+ }
+ }
+ }
+
+ return ret;
+ }
+
private int requestPermission(GlobusEndpoint endpoint, Dataset dataset, Permissions permissions) {
Gson gson = new GsonBuilder().create();
MakeRequestResponse result = null;
@@ -938,9 +985,20 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut
inputList.add(fileId + "IDsplit" + fullPath + "IDsplit" + fileName);
}
+
+ Map fileSizeMap = null;
+
+ if (filesJsonArray.size() >= systemConfig.getGlobusBatchLookupSize()) {
+ // Look up the sizes of all the files in the dataset folder, to avoid
+ // looking them up one by one later:
+ // @todo: we should only be doing this if this is a managed store, probably (?)
+ GlobusEndpoint endpoint = getGlobusEndpoint(dataset);
+ fileSizeMap = lookupFileSizes(endpoint, endpoint.getBasePath());
+ }
// calculateMissingMetadataFields: checksum, mimetype
JsonObject newfilesJsonObject = calculateMissingMetadataFields(inputList, myLogger);
+
JsonArray newfilesJsonArray = newfilesJsonObject.getJsonArray("files");
logger.fine("Size: " + newfilesJsonArray.size());
logger.fine("Val: " + JsonUtil.prettyPrint(newfilesJsonArray.getJsonObject(0)));
@@ -964,20 +1022,26 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut
if (newfileJsonObject != null) {
logger.fine("List Size: " + newfileJsonObject.size());
// if (!newfileJsonObject.get(0).getString("hash").equalsIgnoreCase("null")) {
- JsonPatch path = Json.createPatchBuilder()
+ JsonPatch patch = Json.createPatchBuilder()
.add("/md5Hash", newfileJsonObject.get(0).getString("hash")).build();
- fileJsonObject = path.apply(fileJsonObject);
- path = Json.createPatchBuilder()
+ fileJsonObject = patch.apply(fileJsonObject);
+ patch = Json.createPatchBuilder()
.add("/mimeType", newfileJsonObject.get(0).getString("mime")).build();
- fileJsonObject = path.apply(fileJsonObject);
+ fileJsonObject = patch.apply(fileJsonObject);
+ // If we already know the size of this file on the Globus end,
+ // we'll pass it to /addFiles, to avoid looking up file sizes
+ // one by one:
+ if (fileSizeMap != null && fileSizeMap.get(fileId) != null) {
+ Long uploadedFileSize = fileSizeMap.get(fileId);
+ myLogger.info("Found size for file " + fileId + ": " + uploadedFileSize + " bytes");
+ patch = Json.createPatchBuilder()
+ .add("/fileSize", Json.createValue(uploadedFileSize)).build();
+ fileJsonObject = patch.apply(fileJsonObject);
+ } else {
+ logger.fine("No file size entry found for file "+fileId);
+ }
addFilesJsonData.add(fileJsonObject);
countSuccess++;
- // } else {
- // globusLogger.info(fileName
- // + " will be skipped from adding to dataset by second API due to missing
- // values ");
- // countError++;
- // }
} else {
myLogger.info(fileName
+ " will be skipped from adding to dataset in the final AddReplaceFileHelper.addFiles() call. ");
@@ -1029,7 +1093,7 @@ private void processUploadedFiles(JsonArray filesJsonArray, Dataset dataset, Aut
// The old code had 2 sec. of sleep, so ...
Thread.sleep(2000);
- Response addFilesResponse = addFileHelper.addFiles(newjsonData, dataset, authUser);
+ Response addFilesResponse = addFileHelper.addFiles(newjsonData, dataset, authUser, true);
if (addFilesResponse == null) {
logger.info("null response from addFiles call");
@@ -1211,7 +1275,7 @@ private GlobusTaskState globusStatusCheck(GlobusEndpoint endpoint, String taskId
return task;
}
- public JsonObject calculateMissingMetadataFields(List inputList, Logger globusLogger)
+ private JsonObject calculateMissingMetadataFields(List inputList, Logger globusLogger)
throws InterruptedException, ExecutionException, IOException {
List> hashvalueCompletableFutures = inputList.stream()
@@ -1230,7 +1294,7 @@ public JsonObject calculateMissingMetadataFields(List inputList, Logger
});
JsonArrayBuilder filesObject = (JsonArrayBuilder) completableFuture.get();
-
+
JsonObject output = Json.createObjectBuilder().add("files", filesObject).build();
return output;
diff --git a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
index 0667f5594ce..7280b6af129 100644
--- a/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
+++ b/src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java
@@ -252,8 +252,16 @@ public void setAllowHarvestingMissingCVV(boolean allowHarvestingMissingCVV) {
this.allowHarvestingMissingCVV = allowHarvestingMissingCVV;
}
- // TODO: do we need "orphanRemoval=true"? -- L.A. 4.4
- // TODO: should it be @OrderBy("startTime")? -- L.A. 4.4
+ private boolean useOaiIdAsPid;
+
+ public boolean isUseOaiIdentifiersAsPids() {
+ return useOaiIdAsPid;
+ }
+
+ public void setUseOaiIdentifiersAsPids(boolean useOaiIdAsPid) {
+ this.useOaiIdAsPid = useOaiIdAsPid;
+ }
+
@OneToMany(mappedBy="harvestingClient", cascade={CascadeType.REMOVE, CascadeType.MERGE, CascadeType.PERSIST})
@OrderBy("id")
private List harvestHistory;
diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
index b42fd950528..71c498a4d0b 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java
@@ -344,10 +344,20 @@ public List saveAndAddFilesToDataset(DatasetVersion version,
try {
StorageIO dataAccess = DataAccess.getStorageIO(dataFile);
//Populate metadata
- dataAccess.open(DataAccessOption.READ_ACCESS);
- // (the .open() above makes a remote call to check if
- // the file exists and obtains its size)
- confirmedFileSize = dataAccess.getSize();
+
+ // There are direct upload sub-cases where the file size
+ // is already known at this point. For example, direct uploads
+ // to S3 that go through the jsf dataset page. Or the Globus
+ // uploads, where the file sizes are looked up in bulk on
+ // the completion of the remote upload task.
+ if (dataFile.getFilesize() >= 0) {
+ confirmedFileSize = dataFile.getFilesize();
+ } else {
+ dataAccess.open(DataAccessOption.READ_ACCESS);
+ // (the .open() above makes a remote call to check if
+ // the file exists and obtains its size)
+ confirmedFileSize = dataAccess.getSize();
+ }
// For directly-uploaded files, we will perform the file size
// limit and quota checks here. Perform them *again*, in
@@ -362,13 +372,16 @@ public List saveAndAddFilesToDataset(DatasetVersion version,
if (fileSizeLimit == null || confirmedFileSize < fileSizeLimit) {
//set file size
- logger.fine("Setting file size: " + confirmedFileSize);
- dataFile.setFilesize(confirmedFileSize);
+ if (dataFile.getFilesize() < 0) {
+ logger.fine("Setting file size: " + confirmedFileSize);
+ dataFile.setFilesize(confirmedFileSize);
+ }
if (dataAccess instanceof S3AccessIO) {
((S3AccessIO) dataAccess).removeTempTag();
}
savedSuccess = true;
+ logger.info("directly uploaded file successfully saved. file size: "+dataFile.getFilesize());
}
} catch (IOException ioex) {
logger.warning("Failed to get file size, storage id, or failed to remove the temp tag on the saved S3 object" + dataFile.getStorageIdentifier() + " ("
diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceShapefileHelper.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceShapefileHelper.java
index 8c5dad237b1..27a2ab99376 100644
--- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceShapefileHelper.java
+++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceShapefileHelper.java
@@ -100,71 +100,48 @@ public IngestServiceShapefileHelper(File zippedShapefile, File rezipFolder){
//this.processFile(zippedShapefile, rezipFolder);
}
-
- private FileInputStream getFileInputStream(File fileObject){
- if (fileObject==null){
- return null;
- }
- try {
+
+ private FileInputStream getFileInputStream(File fileObject){
+ if (fileObject==null){
+ return null;
+ }
+ try {
return new FileInputStream(fileObject);
} catch (FileNotFoundException ex) {
logger.severe("Failed to create FileInputStream from File: " + fileObject.getAbsolutePath());
return null;
}
- }
-
- private void closeFileInputStream(FileInputStream fis){
- if (fis==null){
- return;
- }
+ }
+
+ private void closeFileInputStream(FileInputStream fis){
+ if (fis==null){
+ return;
+ }
try {
- fis.close();
+ fis.close();
} catch (IOException ex) {
logger.info("Failed to close FileInputStream");
}
- }
-
+ }
+
public boolean processFile() {
if ((!isValidFile(this.zippedShapefile))||(!isValidFolder(this.rezipFolder))){
return false;
}
-
- // (1) Use the ShapefileHandler to the .zip for a shapefile
- //
- FileInputStream shpfileInputStream = this.getFileInputStream(zippedShapefile);
- if (shpfileInputStream==null){
- return false;
- }
-
- this.shpHandler = new ShapefileHandler(shpfileInputStream);
- if (!shpHandler.containsShapefile()){
- logger.severe("Shapefile was incorrectly detected upon Ingest (FileUtil) and passed here");
- return false;
- }
-
- this.closeFileInputStream(shpfileInputStream);
-
- // (2) Rezip the shapefile pieces
- logger.info("rezipFolder: " + rezipFolder.getAbsolutePath());
- shpfileInputStream = this.getFileInputStream(zippedShapefile);
- if (shpfileInputStream==null){
- return false;
- }
-
- boolean rezipSuccess;
try {
- rezipSuccess = shpHandler.rezipShapefileSets(shpfileInputStream, rezipFolder);
+ this.shpHandler = new ShapefileHandler(zippedShapefile);
+ if (!shpHandler.containsShapefile()){
+ logger.severe("Shapefile was incorrectly detected upon Ingest (FileUtil) and passed here");
+ return false;
+ }
+ logger.info("rezipFolder: " + rezipFolder.getAbsolutePath());
+ return shpHandler.rezipShapefileSets(rezipFolder);
} catch (IOException ex) {
logger.severe("Shapefile was not correctly unpacked/repacked");
logger.severe("shpHandler message: " + shpHandler.errorMessage);
return false;
}
-
- this.closeFileInputStream(shpfileInputStream);
-
- return rezipSuccess;
-
// return createDataFiles(rezipFolder);
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetExternalCitationsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetExternalCitationsServiceBean.java
index 50c24274bb2..fa56432cc3c 100644
--- a/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetExternalCitationsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/makedatacount/DatasetExternalCitationsServiceBean.java
@@ -7,6 +7,9 @@
import edu.harvard.iq.dataverse.Dataset;
import edu.harvard.iq.dataverse.DatasetServiceBean;
+import edu.harvard.iq.dataverse.GlobalId;
+import edu.harvard.iq.dataverse.pidproviders.PidUtil;
+
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -40,7 +43,8 @@ public class DatasetExternalCitationsServiceBean implements java.io.Serializable
Arrays.asList(
"cites",
"references",
- "supplements"));
+ "supplements",
+ "is-supplement-to"));
static ArrayList outboundRelationships = new ArrayList(
Arrays.asList(
"is-cited-by",
@@ -59,12 +63,11 @@ public List parseCitations(JsonArray citations) {
if (inboundRelationships.contains(relationship)) {
Dataset localDs = null;
if (objectUri.contains("doi")) {
- String globalId = objectUri.replace("https://", "").replace("doi.org/", "doi:").toUpperCase().replace("DOI:", "doi:");
- localDs = datasetService.findByGlobalId(globalId);
+ localDs = datasetService.findByGlobalId(objectUri);
exCit.setDataset(localDs);
}
exCit.setCitedByUrl(subjectUri);
-
+
if (localDs != null && !exCit.getCitedByUrl().isEmpty()) {
datasetExternalCitations.add(exCit);
}
@@ -72,9 +75,9 @@ public List parseCitations(JsonArray citations) {
if (outboundRelationships.contains(relationship)) {
Dataset localDs = null;
if (subjectUri.contains("doi")) {
- String globalId = subjectUri.replace("https://", "").replace("doi.org/", "doi:").toUpperCase().replace("DOI:", "doi:");
- localDs = datasetService.findByGlobalId(globalId);
+ localDs = datasetService.findByGlobalId(subjectUri);
exCit.setDataset(localDs);
+
}
exCit.setCitedByUrl(objectUri);
diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java
index f6d142aac96..250eae7e5fc 100644
--- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/AbstractPidProvider.java
@@ -36,9 +36,9 @@ public abstract class AbstractPidProvider implements PidProvider {
private String datafilePidFormat = null;
- private HashSet managedSet;
+ protected HashSet managedSet = new HashSet();
- private HashSet excludedSet;
+ protected HashSet excludedSet = new HashSet();
private String id;
private String label;
@@ -47,8 +47,6 @@ protected AbstractPidProvider(String id, String label, String protocol) {
this.id = id;
this.label = label;
this.protocol = protocol;
- this.managedSet = new HashSet();
- this.excludedSet = new HashSet();
}
protected AbstractPidProvider(String id, String label, String protocol, String authority, String shoulder,
@@ -60,8 +58,12 @@ protected AbstractPidProvider(String id, String label, String protocol, String a
this.shoulder = shoulder;
this.identifierGenerationStyle = identifierGenerationStyle;
this.datafilePidFormat = datafilePidFormat;
- this.managedSet = new HashSet(Arrays.asList(managedList.split(",\\s")));
- this.excludedSet = new HashSet(Arrays.asList(excludedList.split(",\\s")));
+ if(!managedList.isEmpty()) {
+ this.managedSet.addAll(Arrays.asList(managedList.split(",\\s")));
+ }
+ if(!excludedList.isEmpty()) {
+ this.excludedSet.addAll(Arrays.asList(excludedList.split(",\\s")));
+ }
if (logger.isLoggable(Level.FINE)) {
Iterator iter = managedSet.iterator();
while (iter.hasNext()) {
@@ -313,10 +315,17 @@ protected GlobalId parsePersistentId(String protocol, String identifierString) {
}
public GlobalId parsePersistentId(String protocol, String authority, String identifier) {
+ return parsePersistentId(protocol, authority, identifier, false);
+ }
+
+ public GlobalId parsePersistentId(String protocol, String authority, String identifier, boolean isCaseInsensitive) {
logger.fine("Parsing: " + protocol + ":" + authority + getSeparator() + identifier + " in " + getId());
if (!PidProvider.isValidGlobalId(protocol, authority, identifier)) {
return null;
}
+ if(isCaseInsensitive) {
+ identifier = identifier.toUpperCase();
+ }
// Check authority/identifier if this is a provider that manages specific
// identifiers
// /is not one of the unmanaged providers that has null authority
@@ -333,7 +342,7 @@ public GlobalId parsePersistentId(String protocol, String authority, String iden
logger.fine("managed in " + getId() + ": " + getManagedSet().contains(cleanIdentifier));
logger.fine("excluded from " + getId() + ": " + getExcludedSet().contains(cleanIdentifier));
- if (!(((authority.equals(getAuthority()) && identifier.startsWith(getShoulder()))
+ if (!(((authority.equals(getAuthority()) && identifier.startsWith(getShoulder().toUpperCase()))
|| getManagedSet().contains(cleanIdentifier)) && !getExcludedSet().contains(cleanIdentifier))) {
return null;
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java
index 02a7dedce47..70ce1ec4c14 100644
--- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/AbstractDOIProvider.java
@@ -1,6 +1,7 @@
package edu.harvard.iq.dataverse.pidproviders.doi;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.Map;
import java.util.logging.Level;
import java.util.logging.Logger;
@@ -26,9 +27,24 @@ public abstract class AbstractDOIProvider extends AbstractPidProvider {
public AbstractDOIProvider(String id, String label, String providerAuthority, String providerShoulder, String identifierGenerationStyle, String datafilePidFormat, String managedList, String excludedList) {
super(id, label, DOI_PROTOCOL, providerAuthority, providerShoulder, identifierGenerationStyle, datafilePidFormat, managedList, excludedList);
+ //Create case insensitive (converted toUpperCase) managedSet and excludedSet
+ managedSet = clean(managedSet, "managed");
+ excludedSet = clean(excludedSet, "excluded");
}
- //For Unmanged provider
+ private HashSet clean(HashSet originalSet, String setName) {
+ HashSet cleanSet = new HashSet();
+ for(String entry: originalSet) {
+ if(entry.startsWith(DOI_PROTOCOL)) {
+ cleanSet.add(DOI_PROTOCOL + entry.substring(DOI_PROTOCOL.length()).toUpperCase());
+ } else {
+ logger.warning("Non-DOI found in " + setName + " set of pidProvider id: " + getId() + ": " + entry + ". Entry is being dropped.");
+ }
+ }
+ return cleanSet;
+ }
+
+ //For Unmanaged provider
public AbstractDOIProvider(String name, String label) {
super(name, label, DOI_PROTOCOL);
}
@@ -67,7 +83,7 @@ public GlobalId parsePersistentId(String protocol, String authority, String iden
if (!DOI_PROTOCOL.equals(protocol)) {
return null;
}
- return super.parsePersistentId(protocol, authority, identifier);
+ return super.parsePersistentId(protocol, authority, identifier, true);
}
public String getUrlPrefix() {
diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java
index 5630844fb32..b07cd027a01 100644
--- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/doi/datacite/DataCiteDOIProvider.java
@@ -226,8 +226,7 @@ protected String getProviderKeyName() {
@Override
public String getProviderType() {
- // TODO Auto-generated method stub
- return null;
+ return TYPE;
}
public String getMdsUrl() {
diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/handle/HandlePidProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/handle/HandlePidProvider.java
index 9d61663d034..1f03d8a6cfb 100644
--- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/handle/HandlePidProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/handle/HandlePidProvider.java
@@ -59,6 +59,11 @@
* service.
* As of now, it only does the registration updates, to accommodate
* the modifyRegistration datasets API sub-command.
+ *
+ * Note that while Handles are nominally case sensitive, handle.net is
+ * configured to be case-insensitive and Dataverse makes case-insensitve
+ * database look-ups to find Handles (See #11003). That said, database
+ * entries are stored in the case matching the configuration of the provider.
*/
public class HandlePidProvider extends AbstractPidProvider {
diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkPidProvider.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkPidProvider.java
index 7b55292350f..2cc0d41ede7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkPidProvider.java
+++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/perma/PermaLinkPidProvider.java
@@ -24,6 +24,9 @@
* overridable by a configurable parameter to support use of an external
* resolver.
*
+ * Note that while PermaLinks are nominally case sensitive, Dataverse makes
+ * case-insensitve database look-ups to find them (See #11003). That said, database
+ * entries are stored in the case matching the configuration of the provider.
*/
public class PermaLinkPidProvider extends AbstractPidProvider {
diff --git a/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrl.java b/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrl.java
index beb676f60d1..63b5bf03ea7 100644
--- a/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrl.java
+++ b/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrl.java
@@ -30,7 +30,7 @@ public class PrivateUrl {
public PrivateUrl(RoleAssignment roleAssignment, Dataset dataset, String dataverseSiteUrl) {
this.token = roleAssignment.getPrivateUrlToken();
- this.link = dataverseSiteUrl + "/privateurl.xhtml?token=" + token;
+ this.link = dataverseSiteUrl + "/previewurl.xhtml?token=" + token;
this.dataset = dataset;
this.roleAssignment = roleAssignment;
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlPage.java b/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlPage.java
index 9af4bb6af9e..17c622be9e2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlPage.java
+++ b/src/main/java/edu/harvard/iq/dataverse/privateurl/PrivateUrlPage.java
@@ -1,6 +1,10 @@
package edu.harvard.iq.dataverse.privateurl;
+import edu.harvard.iq.dataverse.Dataset;
+import edu.harvard.iq.dataverse.DatasetServiceBean;
+import edu.harvard.iq.dataverse.DataverseRequestServiceBean;
import edu.harvard.iq.dataverse.DataverseSession;
+import edu.harvard.iq.dataverse.PermissionsWrapper;
import edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser;
import java.io.Serializable;
import java.util.logging.Logger;
@@ -20,8 +24,14 @@ public class PrivateUrlPage implements Serializable {
@EJB
PrivateUrlServiceBean privateUrlService;
+ @EJB
+ DatasetServiceBean datasetServiceBean;
@Inject
DataverseSession session;
+ @Inject
+ PermissionsWrapper permissionsWrapper;
+ @Inject
+ DataverseRequestServiceBean dvRequestService;
/**
* The unique string used to look up a PrivateUrlUser and the associated
@@ -34,7 +44,16 @@ public String init() {
PrivateUrlRedirectData privateUrlRedirectData = privateUrlService.getPrivateUrlRedirectDataFromToken(token);
String draftDatasetPageToBeRedirectedTo = privateUrlRedirectData.getDraftDatasetPageToBeRedirectedTo() + "&faces-redirect=true";
PrivateUrlUser privateUrlUser = privateUrlRedirectData.getPrivateUrlUser();
- session.setUser(privateUrlUser);
+ boolean sessionUserCanViewUnpublishedDataset = false;
+ if (session.getUser().isAuthenticated()){
+ Long datasetId = privateUrlUser.getDatasetId();
+ Dataset dataset = datasetServiceBean.find(datasetId);
+ sessionUserCanViewUnpublishedDataset = permissionsWrapper.canViewUnpublishedDataset(dvRequestService.getDataverseRequest(), dataset);
+ }
+ if(!sessionUserCanViewUnpublishedDataset){
+ //Only Reset if user cannot view this Draft Version
+ session.setUser(privateUrlUser);
+ }
logger.info("Redirecting PrivateUrlUser '" + privateUrlUser.getIdentifier() + "' to " + draftDatasetPageToBeRedirectedTo);
return draftDatasetPageToBeRedirectedTo;
} catch (Exception ex) {
diff --git a/src/main/java/edu/harvard/iq/dataverse/privateurl/package-info.java b/src/main/java/edu/harvard/iq/dataverse/privateurl/package-info.java
index 6e939c1bb6d..1310e0eb199 100644
--- a/src/main/java/edu/harvard/iq/dataverse/privateurl/package-info.java
+++ b/src/main/java/edu/harvard/iq/dataverse/privateurl/package-info.java
@@ -1,19 +1,19 @@
/**
- * Private URL for unpublished datasets.
+ * Preview URL for unpublished datasets.
*
- * The Private URL feature has been implemented as a specialized role assignment
+ * The Preview (formerly Private) URL feature has been implemented as a specialized role assignment
* with an associated token that permits read-only access to the metadata and
* all files (regardless of if the files are restricted or not) of a draft
* version of a dataset.
*
- * As of this note, a second option - to create a Private URL that provides an
+ * As of this note, a second option - to create a Preview URL that provides an
* anonymized view of the dataset has been added. This option works the same as
* the original except that it hides author names in the citation block, hides
* the values for an admin specified list of metadata fields, disables citation
* downloads, and disables API access (except for file and file thumbnail
* downloads which are used by the UI).
*
- * The primary use case for a Private URL is for journal editors to send a link
+ * The primary use case for a Preview URL is for journal editors to send a link
* to reviewers of a dataset before publication. In most cases, these journal
* editors do not permit depositors to publish on their own, which is to say
* they only allow depositors to have the "Contributor" role on the datasets
@@ -24,42 +24,42 @@
* the depositor, who is in charge of both the security of the dataset and the
* timing of when the dataset is published.
*
- * A secondary use case for a Private URL is for depositors who have the ability
+ * A secondary use case for a Preview URL is for depositors who have the ability
* to manage permissions on their dataset (depositors who have the "Curator" or
* "Admin" role, which grants much more power than the "Contributor" role) to
* send a link to coauthors or other trusted parties to preview the dataset
* before the depositors publish the dataset on their own. For better security,
* these depositors could ask their coauthors to create Dataverse accounts and
- * assign roles to them directly, rather than using a Private URL which requires
+ * assign roles to them directly, rather than using a Preview URL which requires
* no username or password.
*
* As of this note, a second option aimed specifically at the review use case -
- * to create a Private URL that provides an anonymized view of the dataset - has
+ * to create a Preview URL that provides an anonymized view of the dataset - has
* been added. This option works the same as the original except that it hides
* author names in the citation block, hides the values for an admin specified
* list of metadata fields, disables citation downloads, and disables API access
* (except for file and file thumbnail downloads which are used by the UI).
*
- * The token associated with the Private URL role assignment that can be used
+ * The token associated with the Preview URL role assignment that can be used
* either in the GUI or, for the non-anonymized-access option, via the API to
* elevate privileges beyond what a "Guest" can see. The ability to use a
- * Private URL token via API was added mostly to facilitate automated testing of
- * the feature but the far more common case is expected to be use of the Private
+ * Preview URL token via API was added mostly to facilitate automated testing of
+ * the feature but the far more common case is expected to be use of the Preview
* URL token in a link that is clicked to open a browser, similar to links
* shared via Dropbox, Google, etc.
*
- * When reviewers click a Private URL their browser sessions are set to the
+ * When reviewers click a Preview URL their browser sessions are set to the
* "{@link edu.harvard.iq.dataverse.authorization.users.PrivateUrlUser}" that
* has the "Member" role only on the dataset in question and redirected to that
* dataset, where they will see an indication in blue at the top of the page
* that they are viewing an unpublished dataset. If the reviewer happens to be
* logged into Dataverse already, clicking the link will log them out because
* the review is meant to be blind. Because the dataset is always in draft when
- * a Private URL is in effect, no downloads or any other activity by the
- * reviewer are logged to the guestbook. All reviewers click the same Private
+ * a Preview URL is in effect, no downloads or any other activity by the
+ * reviewer are logged to the guestbook. All reviewers click the same Preview
* URL containing the same token, and with the exception of an IP address being
* logged, it should be impossible to trace which reviewers have clicked a
- * Private URL. If the reviewer navigates to the home page, the session is set
+ * Preview URL. If the reviewer navigates to the home page, the session is set
* to the Guest user and they will see what a Guest would see.
*
* The "Member" role is used because it contains the necessary read-only
@@ -76,51 +76,51 @@
* version. A Member can also download restricted files that have been deleted
* from previously published versions.
*
- * Likewise, when a Private URL token is used via API, commands are executed
+ * Likewise, when a Preview URL token is used via API, commands are executed
* using the "PrivateUrlUser" that has the "Member" role only on the dataset in
* question. This means that read-only operations such as downloads of the
- * dataset's files are permitted. The Search API does not respect the Private
+ * dataset's files are permitted. The Search API does not respect the Preview
* URL token but you can download files using the Access API, and, with the
* non-anonymized-access option, download unpublished metadata using the Native
* API.
*
- * A Private URL cannot be created for a published version of a dataset. In the
+ * A Preview URL cannot be created for a published version of a dataset. In the
* GUI, you will be reminded of this fact with a popup. The API will explain
* this as well.
*
- * An anonymized-access Private URL can't be created if any published dataset
+ * An anonymized-access Preview URL can't be created if any published dataset
* version exists. The primary reason for this is that, since datasets have
* DOIs, the full metadata about published versions is available directly from
* the DOI provider. (While the metadata for that version could be somewhat
* different, in practice it would probably provide a means of identifying
* some/all of the authors).
*
- * If a draft dataset containing a Private URL is
- * published, the Private URL is deleted. This means that reviewers who click
+ * If a draft dataset containing a Preview URL is
+ * published, the Preview URL is deleted. This means that reviewers who click
* the link after publication will see a 404.
*
- * If a post-publication draft containing a Private URL is deleted, the Private
+ * If a post-publication draft containing a Preview URL is deleted, the Preview
* URL is deleted. This is to ensure that if a new draft is created in the
* future, a new token will be used.
*
- * The creation and deletion of a Private URL are limited to the "Curator" and
+ * The creation and deletion of a Preview URL are limited to the "Curator" and
* "Admin" roles because only those roles have the permission called
* "ManageDatasetPermissions", which is the permission used by the
* "AssignRoleCommand" and "RevokeRoleCommand" commands. If you have the
- * permission to create or delete a Private URL, the fact that a Private URL is
+ * permission to create or delete a Preview URL, the fact that a Preview URL is
* enabled for a dataset will be indicated in blue at the top of the page.
* Success messages are shown at the top of the page when you create or delete a
- * Private URL. In the GUI, deleting a Private URL is called "disabling" and you
+ * Preview URL. In the GUI, deleting a Preview URL is called "disabling" and you
* will be prompted for a confirmation. No matter what you call it the role is
- * revoked. You can also delete a Private URL by revoking the role.
+ * revoked. You can also delete a Preview URL by revoking the role.
*
* A "Contributor" does not have the "ManageDatasetPermissions" permission and
- * cannot see "Permissions" nor "Private URL" under the "Edit" menu of their
- * dataset. When a Curator or Admin has enabled a Private URL on a Contributor's
- * dataset, the Contributor does not see a visual indication that a Private URL
+ * cannot see "Permissions" nor "Preview URL" under the "Edit" menu of their
+ * dataset. When a Curator or Admin has enabled a Preview URL on a Contributor's
+ * dataset, the Contributor does not see a visual indication that a Preview URL
* has been enabled for their dataset.
*
- * There is no way for an "Admin" or "Curator" to see when a Private URL was
+ * There is no way for an "Admin" or "Curator" to see when a Preview URL was
* created or deleted for a dataset but someone who has access to the database
* can see that the following commands are logged to the "actionlogrecord"
* database table:
@@ -129,7 +129,7 @@
*
{@link edu.harvard.iq.dataverse.engine.command.impl.CreatePrivateUrlCommand}
* {@link edu.harvard.iq.dataverse.engine.command.impl.DeletePrivateUrlCommand}
*
- * See also the Private URL To Unpublished Dataset BRD at
* https://docs.google.com/document/d/1FT47QkZKcmjSgRnePaJO2g1nzcotLyN3Yb2ORvBr6cs/edit?usp=sharing
*/
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
index a8cf9ed519b..4efd339ee46 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/IndexServiceBean.java
@@ -135,6 +135,9 @@ public class IndexServiceBean {
@EJB
DatasetFieldServiceBean datasetFieldService;
+ @Inject
+ DatasetVersionFilesServiceBean datasetVersionFilesServiceBean;
+
public static final String solrDocIdentifierDataverse = "dataverse_";
public static final String solrDocIdentifierFile = "datafile_";
public static final String solrDocIdentifierDataset = "dataset_";
@@ -420,7 +423,7 @@ synchronized private static Dataset getNextToIndex(Long id, Dataset d) {
public void asyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) {
try {
acquirePermitFromSemaphore();
- doAyncIndexDataset(dataset, doNormalSolrDocCleanUp);
+ doAsyncIndexDataset(dataset, doNormalSolrDocCleanUp);
} catch (InterruptedException e) {
String failureLogText = "Indexing failed: interrupted. You can kickoff a re-index of this dataset with: \r\n curl http://localhost:8080/api/admin/index/datasets/" + dataset.getId().toString();
failureLogText += "\r\n" + e.getLocalizedMessage();
@@ -430,7 +433,7 @@ public void asyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) {
}
}
- private void doAyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) {
+ private void doAsyncIndexDataset(Dataset dataset, boolean doNormalSolrDocCleanUp) {
Long id = dataset.getId();
Dataset next = getNextToIndex(id, dataset); // if there is an ongoing index job for this dataset, next is null (ongoing index job will reindex the newest version after current indexing finishes)
while (next != null) {
@@ -451,7 +454,7 @@ public void asyncIndexDatasetList(List datasets, boolean doNormalSolrDo
for(Dataset dataset : datasets) {
try {
acquirePermitFromSemaphore();
- doAyncIndexDataset(dataset, true);
+ doAsyncIndexDataset(dataset, true);
} catch (InterruptedException e) {
String failureLogText = "Indexing failed: interrupted. You can kickoff a re-index of this dataset with: \r\n curl http://localhost:8080/api/admin/index/datasets/" + dataset.getId().toString();
failureLogText += "\r\n" + e.getLocalizedMessage();
@@ -1018,6 +1021,8 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, SetFeature Request/Idea: Harvest metadata values that aren't from a list of controlled values #9992
@@ -1296,7 +1299,6 @@ public SolrInputDocuments toSolrDocs(IndexableDataset indexableDataset, Set findPermissionsInSolrOnly() throws SearchException {
String dtype = dvObjectService.getDtype(id);
if (dtype == null) {
permissionInSolrOnly.add(docId);
- }
- if (dtype.equals(DType.Dataset.getDType())) {
+ }else if (dtype.equals(DType.Dataset.getDType())) {
List states = datasetService.getVersionStates(id);
if (states != null) {
String latestState = states.get(states.size() - 1);
@@ -2252,7 +2253,7 @@ public List findPermissionsInSolrOnly() throws SearchException {
} else if (dtype.equals(DType.DataFile.getDType())) {
List states = dataFileService.findVersionStates(id);
Set strings = states.stream().map(VersionState::toString).collect(Collectors.toSet());
- logger.fine("States for " + docId + ": " + String.join(", ", strings));
+ logger.finest("States for " + docId + ": " + String.join(", ", strings));
if (docId.endsWith("draft_permission")) {
if (!states.contains(VersionState.DRAFT)) {
permissionInSolrOnly.add(docId);
@@ -2266,7 +2267,7 @@ public List findPermissionsInSolrOnly() throws SearchException {
permissionInSolrOnly.add(docId);
} else {
if (!dataFileService.isInReleasedVersion(id)) {
- logger.fine("Adding doc " + docId + " to list of permissions in Solr only");
+ logger.finest("Adding doc " + docId + " to list of permissions in Solr only");
permissionInSolrOnly.add(docId);
}
}
@@ -2407,6 +2408,11 @@ public void deleteHarvestedDocuments(Dataset harvestedDataset) {
solrIdsOfDocumentsToDelete.add(solrDocIdentifierFile + datafile.getId());
}
+ deleteHarvestedDocuments(solrIdsOfDocumentsToDelete);
+ }
+
+ public void deleteHarvestedDocuments(List solrIdsOfDocumentsToDelete) {
+
logger.fine("attempting to delete the following documents from the index: " + StringUtils.join(solrIdsOfDocumentsToDelete, ","));
IndexResponse resultOfAttemptToDeleteDocuments = solrIndexService.deleteMultipleSolrIds(solrIdsOfDocumentsToDelete);
logger.fine("result of attempt to delete harvested documents: " + resultOfAttemptToDeleteDocuments + "\n");
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java
index ef27a5eefaf..1f1137016f2 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchFields.java
@@ -291,5 +291,6 @@ more targeted results for just datasets. The format is YYYY (i.e.
public static final String DATASET_VALID = "datasetValid";
public static final String DATASET_LICENSE = "license";
+ public static final String FILE_COUNT = "fileCount";
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
index 245f778da23..3fd97d418c0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SearchServiceBean.java
@@ -332,7 +332,7 @@ public SolrQueryResponse search(
// PERMISSION FILTER QUERY
// -----------------------------------
String permissionFilterQuery = this.getPermissionFilterQuery(dataverseRequest, solrQuery, onlyDatatRelatedToMe, addFacets);
- if (permissionFilterQuery != null) {
+ if (!StringUtils.isBlank(permissionFilterQuery)) {
solrQuery.addFilterQuery(permissionFilterQuery);
}
@@ -497,7 +497,8 @@ public SolrQueryResponse search(
Long retentionEndDate = (Long) solrDocument.getFieldValue(SearchFields.RETENTION_END_DATE);
//
Boolean datasetValid = (Boolean) solrDocument.getFieldValue(SearchFields.DATASET_VALID);
-
+ Long fileCount = (Long) solrDocument.getFieldValue(SearchFields.FILE_COUNT);
+
List matchedFields = new ArrayList<>();
SolrSearchResult solrSearchResult = new SolrSearchResult(query, name);
@@ -570,6 +571,7 @@ public SolrQueryResponse search(
solrSearchResult.setDeaccessionReason(deaccessionReason);
solrSearchResult.setDvTree(dvTree);
solrSearchResult.setDatasetValid(datasetValid);
+ solrSearchResult.setFileCount(fileCount);
if (Boolean.TRUE.equals((Boolean) solrDocument.getFieldValue(SearchFields.IS_HARVESTED))) {
solrSearchResult.setHarvested(true);
@@ -1047,7 +1049,7 @@ private String getPermissionFilterQuery(DataverseRequest dataverseRequest, SolrQ
// add joins on all the non-public groups that may exist for the
// user:
- // Authenticated users and GuestUser may be part of one or more groups; such
+ // Authenticated users, *and the GuestUser*, may be part of one or more groups; such
// as IP Groups.
groups = groupService.collectAncestors(groupService.groupsFor(dataverseRequest));
diff --git a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java
index 27900bac63f..8802555affd 100644
--- a/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java
+++ b/src/main/java/edu/harvard/iq/dataverse/search/SolrSearchResult.java
@@ -78,6 +78,10 @@ public class SolrSearchResult {
private String citation;
private String citationHtml;
private String datasetType;
+ /**
+ * Only Dataset can have a file count
+ */
+ private Long fileCount;
/**
* Files and datasets might have a UNF. Dataverses don't.
*/
@@ -456,10 +460,10 @@ public JsonObjectBuilder getJsonForMyData(boolean isValid) {
} // getJsonForMydata
public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, boolean showApiUrls) {
- return json(showRelevance, showEntityIds, showApiUrls, null, null);
+ return json(showRelevance, showEntityIds, showApiUrls, null);
}
- public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, boolean showApiUrls, List metadataFields, Long datasetFileCount) {
+ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, boolean showApiUrls, List metadataFields) {
if (this.type == null) {
return jsonObjectBuilder();
}
@@ -597,7 +601,7 @@ public JsonObjectBuilder json(boolean showRelevance, boolean showEntityIds, bool
subjects.add(subject);
}
nullSafeJsonBuilder.add("subjects", subjects);
- nullSafeJsonBuilder.add("fileCount", datasetFileCount);
+ nullSafeJsonBuilder.add("fileCount", this.fileCount);
nullSafeJsonBuilder.add("versionId", dv.getId());
nullSafeJsonBuilder.add("versionState", dv.getVersionState().toString());
if (this.isPublishedState()) {
@@ -1348,4 +1352,12 @@ public boolean isValid(Predicate canUpdateDataset) {
}
return !canUpdateDataset.test(this);
}
+
+ public Long getFileCount() {
+ return fileCount;
+ }
+
+ public void setFileCount(Long fileCount) {
+ this.fileCount = fileCount;
+ }
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java
index 33e828e619d..20632c170e4 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/FeatureFlags.java
@@ -97,12 +97,16 @@ public enum FeatureFlags {
* for the dataset.
*
* @apiNote Raise flag by setting
- * "dataverse.feature.enable-dataset-thumbnail-autoselect"
+ * "dataverse.feature.disable-dataset-thumbnail-autoselect"
* @since Dataverse 6.4
*/
DISABLE_DATASET_THUMBNAIL_AUTOSELECT("disable-dataset-thumbnail-autoselect"),
/**
* Feature flag for the new Globus upload framework.
+ *
+ * @apiNote Raise flag by setting
+ * "dataverse.feature.globus-use-experimental-async-framework"
+ * @since Dataverse 6.4
*/
GLOBUS_USE_EXPERIMENTAL_ASYNC_FRAMEWORK("globus-use-experimental-async-framework"),
;
diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
index 8ed96690e84..b5eb483c2c8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
+++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java
@@ -539,6 +539,12 @@ Whether Harvesting (OAI) service is enabled
*
*/
GlobusSingleFileTransfer,
+ /** Lower limit of the number of files in a Globus upload task where
+ * the batch mode should be utilized in looking up the file information
+ * on the remote end node (file sizes, primarily), instead of individual
+ * lookups.
+ */
+ GlobusBatchLookupSize,
/**
* Optional external executables to run on the metadata for dataverses
* and datasets being published; as an extra validation step, to
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/BundleUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/BundleUtil.java
index 922e6ff5d28..771cf5fd0f0 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/BundleUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/BundleUtil.java
@@ -111,7 +111,7 @@ public static ResourceBundle getResourceBundle(String propertyFileName, Locale c
ClassLoader loader = getClassLoader(filesRootDirectory);
bundle = ResourceBundle.getBundle(propertyFileName, currentLocale, loader);
} catch (MissingResourceException mre) {
- logger.warning("No property file named " + propertyFileName + "_" + currentLocale.getLanguage()
+ logger.fine("No property file named " + propertyFileName + "_" + currentLocale.getLanguage()
+ " found in " + filesRootDirectory + ", using untranslated values");
bundle = ResourceBundle.getBundle("propertyFiles/" + propertyFileName, currentLocale);
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
index a0c32d5c8ce..991682ec8e8 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java
@@ -525,15 +525,18 @@ public static String determineFileType(File f, String fileName) throws IOExcepti
// Check for shapefile extensions as described here: http://en.wikipedia.org/wiki/Shapefile
//logger.info("Checking for shapefile");
- ShapefileHandler shp_handler = new ShapefileHandler(new FileInputStream(f));
+ ShapefileHandler shp_handler = new ShapefileHandler(f);
if (shp_handler.containsShapefile()){
// logger.info("------- shapefile FOUND ----------");
fileType = ShapefileHandler.SHAPEFILE_FILE_TYPE; //"application/zipped-shapefile";
}
-
- Optional bagItFileHandler = CDI.current().select(BagItFileHandlerFactory.class).get().getBagItFileHandler();
- if(bagItFileHandler.isPresent() && bagItFileHandler.get().isBagItPackage(fileName, f)) {
- fileType = BagItFileHandler.FILE_TYPE;
+ try {
+ Optional bagItFileHandler = CDI.current().select(BagItFileHandlerFactory.class).get().getBagItFileHandler();
+ if (bagItFileHandler.isPresent() && bagItFileHandler.get().isBagItPackage(fileName, f)) {
+ fileType = BagItFileHandler.FILE_TYPE;
+ }
+ } catch (Exception e) {
+ logger.warning("Error checking for BagIt package: " + e.getMessage());
}
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java
index bb916cc3906..2b54f7a3bfe 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/ShapefileHandler.java
@@ -1,23 +1,21 @@
package edu.harvard.iq.dataverse.util;
import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
import java.util.Date;
import java.util.ArrayList;
import java.util.List;
-import java.util.zip.ZipEntry;
-import java.util.zip.ZipInputStream;
-import java.util.zip.ZipException;
+import java.util.zip.ZipFile;
import java.util.HashMap;
import java.util.*;
import java.nio.file.Files;
import java.nio.file.Paths;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
-import java.util.logging.Level;
+
import java.util.logging.Logger;
import org.apache.commons.io.FileUtils;
@@ -43,11 +41,10 @@
* "shape1.pdf", "README.md", "shape_notes.txt"
*
* Code Example:
- * FileInputStream shp_file_input_stream = new FileInputStream(new File("zipped_shapefile.zip"))
- * ShapefileHandler shp_handler = new ShapefileHandler(shp_file_input_stream);
+ * ShapefileHandler shp_handler = new ShapefileHandler(new File("zipped_shapefile.zip"));
* if (shp_handler.containsShapefile()){
* File rezip_folder = new File("~/folder_for_rezipping");
- * boolean rezip_success = shp_handler.rezipShapefileSets(shp_file_input_stream, rezip_folder );
+ * boolean rezip_success = shp_handler.rezipShapefileSets(rezip_folder );
* if (!rezip_success){
* // rezip failed, should be an error message (String) available
System.out.println(shp_handler.error_message);
@@ -74,7 +71,7 @@ public class ShapefileHandler{
public static final String SHP_XML_EXTENSION = "shp.xml";
public static final String BLANK_EXTENSION = "__PLACEHOLDER-FOR-BLANK-EXTENSION__";
public static final List SHAPEFILE_ALL_EXTENSIONS = Arrays.asList("shp", "shx", "dbf", "prj", "sbn", "sbx", "fbn", "fbx", "ain", "aih", "ixs", "mxs", "atx", "cpg", "qpj", "qmd", SHP_XML_EXTENSION);
-
+ private final File zipFile;
public boolean DEBUG = false;
private boolean zipFileProcessed = false;
@@ -97,9 +94,6 @@ public class ShapefileHandler{
private Map> fileGroups = new HashMap<>();
private List finalRezippedFiles = new ArrayList<>();
-
- private String outputFolder = "unzipped";
- private String rezippedFolder = "rezipped";
// Debug helper
private void msg(String s){
@@ -116,40 +110,28 @@ private void msgt(String s){
}
/*
- Constructor, start with filename
- */
- public ShapefileHandler(String filename){
-
- if (filename==null){
- this.addErrorMessage("The filename was null");
- return;
- }
-
- FileInputStream zip_file_stream;
- try {
- zip_file_stream = new FileInputStream(new File(filename));
- } catch (FileNotFoundException ex) {
- this.addErrorMessage("The file was not found");
+ Constructor, start with File
+ */
+ public ShapefileHandler(File zip_file) throws IOException {
+ zipFile = zip_file;
+ if (zip_file == null) {
+ this.addErrorMessage("The file was null");
return;
}
-
- this.examineZipfile(zip_file_stream);
- }
-
-
- /*
- Constructor, start with FileInputStream
- */
- public ShapefileHandler(FileInputStream zip_file_stream){
-
- if (zip_file_stream==null){
- this.addErrorMessage("The zip_file_stream was null");
- return;
+ try (var zip_file_object = new ZipFile(zip_file)) {
+ this.examineZipfile(zip_file_object);
+ }
+ catch (FileNotFoundException ex) {
+ // While this constructor had a FileInputStream as argument:
+ // FileUtil.determineFileType threw this exception before calling the constructor with a FileInputStream
+ // IngestServiceShapefileHelper.processFile won´t call this constructor if the file is not valid hence does not exist.
+ // When the file would have disappeared in the meantime, it would have produced a slightly different error message.
+ logger.severe("File not found: " + zip_file.getAbsolutePath());
+ throw ex;
}
- this.examineZipfile(zip_file_stream);
}
-
+
public List getFinalRezippedFiles(){
return this.finalRezippedFiles;
}
@@ -291,26 +273,19 @@ inside the uploaded zip file (issue #6873). To achieve this, we recreate
subfolders in the FileMetadata of the newly created DataFiles.
(-- L.A. 09/2020)
*/
- private boolean unzipFilesToDirectory(FileInputStream zipfile_input_stream, File target_directory){
+ private boolean unzipFilesToDirectory(ZipFile zipfileInput, File target_directory){
logger.fine("unzipFilesToDirectory: " + target_directory.getAbsolutePath() );
- if (zipfile_input_stream== null){
- this.addErrorMessage("unzipFilesToDirectory. The zipfile_input_stream is null.");
- return false;
- }
if (!target_directory.isDirectory()){
this.addErrorMessage("This directory does not exist: " + target_directory.getAbsolutePath());
return false;
}
- List unzippedFileNames = new ArrayList<>();
-
- ZipInputStream zipStream = new ZipInputStream(zipfile_input_stream);
+ List unzippedFileNames = new ArrayList<>();
+
- ZipEntry origEntry;
- byte[] buffer = new byte[2048];
try {
- while((origEntry = zipStream.getNextEntry())!=null){
+ for(var origEntry : Collections.list(zipfileInput.entries())){
String zentryFileName = origEntry.getName();
logger.fine("\nOriginal entry name: " + origEntry);
@@ -360,15 +335,10 @@ private boolean unzipFilesToDirectory(FileInputStream zipfile_input_stream, File
unzippedFileNames.add(outpath);
}
logger.fine("Write zip file: " + outpath);
- FileOutputStream fileOutputStream;
- long fsize = 0;
- fileOutputStream = new FileOutputStream(outpath);
- int len;// = 0;
- while ((len = zipStream.read(buffer)) > 0){
- fileOutputStream.write(buffer, 0, len);
- fsize+=len;
- } // end while
- fileOutputStream.close();
+ try(var inputStream = zipfileInput.getInputStream(origEntry)) {
+ Files.createDirectories(new File(outpath).getParentFile().toPath());
+ Files.copy(inputStream, Path.of(outpath), StandardCopyOption.REPLACE_EXISTING);
+ }
} // end outer while
} catch (IOException ex) {
for (StackTraceElement el : ex.getStackTrace()){
@@ -377,19 +347,13 @@ private boolean unzipFilesToDirectory(FileInputStream zipfile_input_stream, File
this.addErrorMessage("Failed to open ZipInputStream entry" + ex.getMessage());
return false;
}
-
- try {
- zipStream.close();
- } catch (IOException ex) {
- Logger.getLogger(ShapefileHandler.class.getName()).log(Level.SEVERE, null, ex);
- }
- return true;
+ return true;
}
/*
Rezip the shapefile(s) into a given directory
Assumes that the zipfile_input_stream has already been checked!
*/
- public boolean rezipShapefileSets(FileInputStream zipfile_input_stream, File rezippedFolder) throws IOException{
+ public boolean rezipShapefileSets(File rezippedFolder) throws IOException{
logger.fine("rezipShapefileSets");
//msgt("rezipShapefileSets");
if (!this.zipFileProcessed){
@@ -400,10 +364,6 @@ public boolean rezipShapefileSets(FileInputStream zipfile_input_stream, File rez
this.addErrorMessage("There are no shapefiles here!");
return false;
}
- if (zipfile_input_stream== null){
- this.addErrorMessage("The zipfile_input_stream is null.");
- return false;
- }
if (rezippedFolder == null){
this.addErrorMessage("The rezippedFolder is null.");
return false;
@@ -433,9 +393,11 @@ public boolean rezipShapefileSets(FileInputStream zipfile_input_stream, File rez
// Unzip files!
- if (!this.unzipFilesToDirectory(zipfile_input_stream, dir_for_unzipping)){
- this.addErrorMessage("Failed to unzip files.");
- return false;
+ try(var zipfileObject = new ZipFile(zipFile)) {
+ if (!this.unzipFilesToDirectory(zipfileObject, dir_for_unzipping)) {
+ this.addErrorMessage("Failed to unzip files.");
+ return false;
+ }
}
// Redistribute files!
String target_dirname = rezippedFolder.getAbsolutePath();
@@ -681,27 +643,19 @@ private boolean isFileToSkip(String fname){
/**************************************
* Iterate through the zip file contents.
* Does it contain any shapefiles?
- *
- * @param FileInputStream zip_file_stream
*/
- private boolean examineZipfile(FileInputStream zip_file_stream){
+ private boolean examineZipfile(ZipFile zip_file){
// msgt("examineZipfile");
-
- if (zip_file_stream==null){
- this.addErrorMessage("The zip file stream was null");
- return false;
- }
-
+
// Clear out file lists
this.filesListInDir.clear();
this.filesizeHash.clear();
this.fileGroups.clear();
- try{
- ZipInputStream zipStream = new ZipInputStream(zip_file_stream);
- ZipEntry entry;
- List hiddenDirectories = new ArrayList<>();
- while((entry = zipStream.getNextEntry())!=null){
+ try{
+ List hiddenDirectories = new ArrayList<>();
+ for(var entry : Collections.list(zip_file.entries())){
+
String zentryFileName = entry.getName();
boolean isDirectory = entry.isDirectory();
@@ -748,8 +702,6 @@ private boolean examineZipfile(FileInputStream zip_file_stream){
this.filesizeHash.put(unzipFilePath, entry.getSize());
}
} // end while
-
- zipStream.close();
if (this.filesListInDir.isEmpty()){
errorMessage = "No files in zipStream";
@@ -759,13 +711,8 @@ private boolean examineZipfile(FileInputStream zip_file_stream){
this.zipFileProcessed = true;
return true;
- }catch(ZipException ex){
- this.addErrorMessage("ZipException");
- msgt("ZipException");
- return false;
-
}catch(IOException ex){
- //ex.printStackTrace();
+ //ex.printStackTrace();
this.addErrorMessage("IOException File name");
msgt("IOException");
return false;
@@ -773,9 +720,6 @@ private boolean examineZipfile(FileInputStream zip_file_stream){
this.addErrorMessage("IllegalArgumentException when parsing zipfile");
msgt("IllegalArgumentException when parsing zipfile");
return false;
-
- }finally{
-
}
} // end examineFile
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
index 434b3bd8f8f..e769cacfdb1 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java
@@ -78,6 +78,7 @@ public class SystemConfig {
public static final long defaultZipDownloadLimit = 104857600L; // 100MB
private static final int defaultMultipleUploadFilesLimit = 1000;
private static final int defaultLoginSessionTimeout = 480; // = 8 hours
+ private static final int defaultGlobusBatchLookupSize = 50;
private String buildNumber = null;
@@ -954,6 +955,11 @@ public boolean isGlobusFileDownload() {
return (isGlobusDownload() && settingsService.isTrueForKey(SettingsServiceBean.Key.GlobusSingleFileTransfer, false));
}
+ public int getGlobusBatchLookupSize() {
+ String batchSizeOption = settingsService.getValueForKey(SettingsServiceBean.Key.GlobusBatchLookupSize);
+ return getIntLimitFromStringOrDefault(batchSizeOption, defaultGlobusBatchLookupSize);
+ }
+
private Boolean getMethodAvailable(String method, boolean upload) {
String methods = settingsService.getValueForKey(
upload ? SettingsServiceBean.Key.UploadMethods : SettingsServiceBean.Key.DownloadMethods);
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
index 2f01c9bc2f2..232b7431a24 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonParser.java
@@ -19,6 +19,7 @@
import edu.harvard.iq.dataverse.MetadataBlockServiceBean;
import edu.harvard.iq.dataverse.TermsOfUseAndAccess;
import edu.harvard.iq.dataverse.api.Util;
+import edu.harvard.iq.dataverse.api.dto.DataverseDTO;
import edu.harvard.iq.dataverse.api.dto.FieldDTO;
import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.IpGroup;
import edu.harvard.iq.dataverse.authorization.groups.impl.ipaddress.ip.IpAddress;
@@ -48,8 +49,10 @@
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.function.Consumer;
import java.util.logging.Logger;
import java.util.stream.Collectors;
+
import jakarta.json.Json;
import jakarta.json.JsonArray;
import jakarta.json.JsonObject;
@@ -128,7 +131,7 @@ public Dataverse parseDataverse(JsonObject jobj) throws JsonParseException {
dv.setPermissionRoot(jobj.getBoolean("permissionRoot", false));
dv.setFacetRoot(jobj.getBoolean("facetRoot", false));
dv.setAffiliation(jobj.getString("affiliation", null));
-
+
if (jobj.containsKey("dataverseContacts")) {
JsonArray dvContacts = jobj.getJsonArray("dataverseContacts");
int i = 0;
@@ -141,7 +144,7 @@ public Dataverse parseDataverse(JsonObject jobj) throws JsonParseException {
}
dv.setDataverseContacts(dvContactList);
}
-
+
if (jobj.containsKey("theme")) {
DataverseTheme theme = parseDataverseTheme(jobj.getJsonObject("theme"));
dv.setDataverseTheme(theme);
@@ -149,21 +152,21 @@ public Dataverse parseDataverse(JsonObject jobj) throws JsonParseException {
}
dv.setDataverseType(Dataverse.DataverseType.UNCATEGORIZED); // default
- if (jobj.containsKey("dataverseType")) {
- for (Dataverse.DataverseType dvtype : Dataverse.DataverseType.values()) {
- if (dvtype.name().equals(jobj.getString("dataverseType"))) {
- dv.setDataverseType(dvtype);
- }
- }
+ String receivedDataverseType = jobj.getString("dataverseType", null);
+ if (receivedDataverseType != null) {
+ Arrays.stream(Dataverse.DataverseType.values())
+ .filter(type -> type.name().equals(receivedDataverseType))
+ .findFirst()
+ .ifPresent(dv::setDataverseType);
}
-
+
if (jobj.containsKey("filePIDsEnabled")) {
dv.setFilePIDsEnabled(jobj.getBoolean("filePIDsEnabled"));
}
/* We decided that subject is not user set, but gotten from the subject of the dataverse's
datasets - leavig this code in for now, in case we need to go back to it at some point
-
+
if (jobj.containsKey("dataverseSubjects")) {
List dvSubjectList = new LinkedList<>();
DatasetFieldType subjectType = datasetFieldSvc.findByName(DatasetFieldConstant.subject);
@@ -186,10 +189,49 @@ public Dataverse parseDataverse(JsonObject jobj) throws JsonParseException {
dv.setDataverseSubjects(dvSubjectList);
}
*/
-
+
return dv;
}
-
+
+ public DataverseDTO parseDataverseDTO(JsonObject jsonObject) throws JsonParseException {
+ DataverseDTO dataverseDTO = new DataverseDTO();
+
+ setDataverseDTOPropertyIfPresent(jsonObject, "alias", dataverseDTO::setAlias);
+ setDataverseDTOPropertyIfPresent(jsonObject, "name", dataverseDTO::setName);
+ setDataverseDTOPropertyIfPresent(jsonObject, "description", dataverseDTO::setDescription);
+ setDataverseDTOPropertyIfPresent(jsonObject, "affiliation", dataverseDTO::setAffiliation);
+
+ String dataverseType = jsonObject.getString("dataverseType", null);
+ if (dataverseType != null) {
+ Arrays.stream(Dataverse.DataverseType.values())
+ .filter(type -> type.name().equals(dataverseType))
+ .findFirst()
+ .ifPresent(dataverseDTO::setDataverseType);
+ }
+
+ if (jsonObject.containsKey("dataverseContacts")) {
+ JsonArray dvContacts = jsonObject.getJsonArray("dataverseContacts");
+ List contacts = new ArrayList<>();
+ for (int i = 0; i < dvContacts.size(); i++) {
+ JsonObject contactObj = dvContacts.getJsonObject(i);
+ DataverseContact contact = new DataverseContact();
+ contact.setContactEmail(getMandatoryString(contactObj, "contactEmail"));
+ contact.setDisplayOrder(i);
+ contacts.add(contact);
+ }
+ dataverseDTO.setDataverseContacts(contacts);
+ }
+
+ return dataverseDTO;
+ }
+
+ private void setDataverseDTOPropertyIfPresent(JsonObject jsonObject, String key, Consumer setter) {
+ String value = jsonObject.getString(key, null);
+ if (value != null) {
+ setter.accept(value);
+ }
+ }
+
public DataverseTheme parseDataverseTheme(JsonObject obj) {
DataverseTheme theme = new DataverseTheme();
@@ -965,7 +1007,6 @@ public void parseControlledVocabularyValue(DatasetField dsf, DatasetFieldType cv
if (cvv == null) {
if (allowHarvestingMissingCVV) {
// we need to process this as a primitive value
- logger.warning(">>>> Value '" + strValue + "' does not exist in type '" + cvvType.getName() + "'. Processing as primitive per setting override.");
parsePrimitiveValue(dsf, cvvType , json);
return;
} else {
@@ -1011,6 +1052,7 @@ public String parseHarvestingClient(JsonObject obj, HarvestingClient harvestingC
harvestingClient.setHarvestingSet(obj.getString("set",null));
harvestingClient.setCustomHttpHeaders(obj.getString("customHeaders", null));
harvestingClient.setAllowHarvestingMissingCVV(obj.getBoolean("allowHarvestingMissingCVV", false));
+ harvestingClient.setUseOaiIdentifiersAsPids(obj.getBoolean("useOaiIdentifiersAsPids", false));
return dataverseAlias;
}
diff --git a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
index 3f08850b91a..e8ca525ca4f 100644
--- a/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
+++ b/src/main/java/edu/harvard/iq/dataverse/util/json/JsonPrinter.java
@@ -277,7 +277,9 @@ public static JsonObjectBuilder json(Dataverse dv, Boolean hideEmail, Boolean re
}
bld.add("permissionRoot", dv.isPermissionRoot())
.add("description", dv.getDescription())
- .add("dataverseType", dv.getDataverseType().name());
+ .add("dataverseType", dv.getDataverseType().name())
+ .add("isMetadataBlockRoot", dv.isMetadataBlockRoot())
+ .add("isFacetRoot", dv.isFacetRoot());
if (dv.getOwner() != null) {
bld.add("ownerId", dv.getOwner().getId());
}
@@ -1021,6 +1023,7 @@ public static JsonObjectBuilder json(HarvestingClient harvestingClient) {
add("status", harvestingClient.isHarvestingNow() ? "inProgress" : "inActive").
add("customHeaders", harvestingClient.getCustomHttpHeaders()).
add("allowHarvestingMissingCVV", harvestingClient.getAllowHarvestingMissingCVV()).
+ add("useOaiIdentifiersAsPids", harvestingClient.isUseOaiIdentifiersAsPids()).
add("lastHarvest", harvestingClient.getLastHarvestTime() == null ? null : harvestingClient.getLastHarvestTime().toString()).
add("lastResult", harvestingClient.getLastResult()).
add("lastSuccessful", harvestingClient.getLastSuccessfulHarvestTime() == null ? null : harvestingClient.getLastSuccessfulHarvestTime().toString()).
diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties
index 149e6a7e828..c47356008ff 100644
--- a/src/main/java/propertyFiles/Bundle.properties
+++ b/src/main/java/propertyFiles/Bundle.properties
@@ -307,7 +307,13 @@ notification.typeDescription.WORKFLOW_FAILURE=External workflow run has failed
notification.typeDescription.STATUSUPDATED=Status of dataset has been updated
notification.typeDescription.DATASETCREATED=Dataset was created by user
notification.typeDescription.DATASETMENTIONED=Dataset was referenced in remote system
-
+notification.typeDescription.GLOBUSUPLOADCOMPLETED=Globus upload is completed
+notification.typeDescription.GLOBUSUPLOADCOMPLETEDWITHERRORS=Globus upload completed with errors
+notification.typeDescription.GLOBUSDOWNLOADCOMPLETED=Globus download is completed
+notification.typeDescription.GLOBUSDOWNLOADCOMPLETEDWITHERRORS=Globus download completed with errors
+notification.typeDescription.GLOBUSUPLOADLOCALFAILURE=Globus upload failed, internal error
+notification.typeDescription.GLOBUSUPLOADREMOTEFAILURE=Globus upload failed, remote transfer error
+notification.typeDescription.REQUESTEDFILEACCESS=File access requested
groupAndRoles.manageTips=Here is where you can access and manage all the groups you belong to, and the roles you have been assigned.
user.message.signup.label=Create Account
user.message.signup.tip=Why have a Dataverse account? To create your own dataverse and customize it, add datasets, or request access to restricted files.
@@ -837,7 +843,8 @@ notification.email.datasetWasMentioned=Hello {0},
The {1} has just been
notification.email.datasetWasMentioned.subject={0}: A Dataset Relationship has been reported!
notification.email.globus.uploadCompleted.subject={0}: Files uploaded successfully via Globus and verified
notification.email.globus.downloadCompleted.subject={0}: Files downloaded successfully via Globus
-notification.email.globus.uploadCompletedWithErrors.subject={0}: Uploaded files via Globus with errors
+notification.email.globus.downloadCompletedWithErrors.subject={0}: Globus download task completed, errors encountered
+notification.email.globus.uploadCompletedWithErrors.subject={0}: Globus upload task completed with errors
notification.email.globus.uploadFailedRemotely.subject={0}: Failed to upload files via Globus
notification.email.globus.uploadFailedLocally.subject={0}: Failed to add files uploaded via Globus to dataset
# dataverse.xhtml
@@ -1457,14 +1464,14 @@ dataset.editBtn.itemLabel.metadata=Metadata
dataset.editBtn.itemLabel.terms=Terms
dataset.editBtn.itemLabel.permissions=Permissions
dataset.editBtn.itemLabel.thumbnailsAndWidgets=Thumbnails + Widgets
-dataset.editBtn.itemLabel.privateUrl=Private URL
+dataset.editBtn.itemLabel.privateUrl=Preview URL
dataset.editBtn.itemLabel.permissionsDataset=Dataset
dataset.editBtn.itemLabel.permissionsFile=Restricted Files
dataset.editBtn.itemLabel.deleteDataset=Delete Dataset
dataset.editBtn.itemLabel.deleteDraft=Delete Draft Version
dataset.editBtn.itemLabel.deaccession=Deaccession Dataset
dataset.exportBtn=Export Metadata
-dataset.exportBtn.itemLabel.ddi=DDI
+dataset.exportBtn.itemLabel.ddi=DDI Codebook v2
dataset.exportBtn.itemLabel.dublinCore=Dublin Core
dataset.exportBtn.itemLabel.schemaDotOrg=Schema.org JSON-LD
dataset.exportBtn.itemLabel.datacite=DataCite
@@ -1669,6 +1676,8 @@ dataset.message.createFailure=The dataset could not be created.
dataset.message.termsFailure=The dataset terms could not be updated.
dataset.message.label.fileAccess=Publicly-accessible storage
dataset.message.publicInstall=Files in this dataset may be readable outside Dataverse, restricted and embargoed access are disabled
+dataset.message.parallelUpdateError=Changes cannot be saved. This dataset has been edited since this page was opened. To continue, copy your changes, refresh the page to see the recent updates, and re-enter any changes you want to save.
+dataset.message.parallelPublishError=Publishing is blocked. This dataset has been edited since this page was opened. To publish it, refresh the page to see the recent updates, and publish again.
dataset.metadata.publicationDate=Publication Date
dataset.metadata.publicationDate.tip=The publication date of a Dataset.
dataset.metadata.citationDate=Citation Date
@@ -1721,23 +1730,36 @@ dataset.transferUnrestricted=Click Continue to transfer the elligible files.
dataset.requestAccessToRestrictedFiles=You may request access to any restricted file(s) by clicking the Request Access button.
dataset.requestAccessToRestrictedFilesWithEmbargo=Embargoed files cannot be accessed during the embargo period. If your selection contains restricted files, you may request access to them by clicking the Request Access button.
dataset.privateurl.infoMessageAuthor=Privately share this dataset before it is published: {0}
-dataset.privateurl.infoMessageReviewer=This unpublished dataset is being privately shared.
-dataset.privateurl.header=Unpublished Dataset Private URL
-dataset.privateurl.tip=Use a Private URL to allow those without Dataverse accounts to access your unpublished dataset. For more information about the Private URL feature, please refer to the User Guide.
-dataset.privateurl.absent=Private URL has not been created.
-dataset.privateurl.createPrivateUrl=Create Private URL
+dataset.privateurl.infoMessageReviewer=You are viewing a preview of this unpublished dataset version.
+dataset.privateurl.header=Unpublished Dataset Preview URL
+dataset.privateurl.tip=To cite this data in publications, use the dataset's persistent ID instead of this URL. For more information about the Preview URL feature, please refer to the User Guide.
+dataset.privateurl.onlyone=Only one Preview URL can be active for a single dataset.
+dataset.privateurl.absent=Preview URL has not been created.
+dataset.privateurl.general.button.label=Create General Preview URL
+dataset.privateurl.general.description=Create a URL that others can use to review this dataset version before it is published. They will be able to access all files in the dataset and see all metadata, including metadata that may identify the dataset's authors.
+dataset.privateurl.general.title=General Preview
+dataset.privateurl.anonymous.title=Anonymous Preview
+dataset.privateurl.anonymous.button.label=Create Anonymous Preview URL
+dataset.privateurl.anonymous.description=Create a URL that others can use to access an anonymized view of this unpublished dataset version. Metadata that could identify the dataset author will not be displayed. Non-identifying metadata will be visible.
+dataset.privateurl.anonymous.description.paragraph.two=The dataset's files are not changed and will be accessible if they're not restricted. Users of the Anonymous Preview URL will not be able to see the name of the Dataverse that this dataset is in but will be able to see the name of the repository, which might expose the dataset authors' identities.
+dataset.privateurl.createPrivateUrl=Create Preview URL
+dataset.privateurl.introduction=You can create a Preview URL to copy and share with others who will not need a repository account to review this unpublished dataset version. Once the dataset is published or if the URL is disabled, the URL will no longer work and will point to a "Page not found" page.
dataset.privateurl.createPrivateUrl.anonymized=Create URL for Anonymized Access
dataset.privateurl.createPrivateUrl.anonymized.unavailable=Anonymized Access is not available once a version of the dataset has been published
-dataset.privateurl.disablePrivateUrl=Disable Private URL
-dataset.privateurl.disablePrivateUrlConfirm=Yes, Disable Private URL
-dataset.privateurl.disableConfirmationText=Are you sure you want to disable the Private URL? If you have shared the Private URL with others they will no longer be able to use it to access your unpublished dataset.
-dataset.privateurl.cannotCreate=Private URL can only be used with unpublished versions of datasets.
-dataset.privateurl.roleassigeeTitle=Private URL Enabled
+dataset.privateurl.disablePrivateUrl=Disable Preview URL
+dataset.privateurl.disableGeneralPreviewUrl=Disable General Preview URL
+dataset.privateurl.disableAnonPreviewUrl=Disable Anonymous Preview URL
+dataset.privateurl.disablePrivateUrlConfirm=Yes, Disable Preview URL
+dataset.privateurl.disableGeneralPreviewUrlConfirm=Yes, Disable General Preview URL
+dataset.privateurl.disableAnonPreviewUrlConfirm=Yes, Disable Anonymous Preview URL
+dataset.privateurl.disableConfirmationText=Are you sure you want to disable the Preview URL? If you have shared the Preview URL with others they will no longer be able to use it to access your unpublished dataset.
+dataset.privateurl.cannotCreate=Preview URL can only be used with unpublished versions of datasets.
+dataset.privateurl.roleassigeeTitle=Preview URL Enabled
dataset.privateurl.createdSuccess=Success!
-dataset.privateurl.full=This Private URL provides full read access to the dataset
-dataset.privateurl.anonymized=This Private URL provides access to the anonymized dataset
-dataset.privateurl.disabledSuccess=You have successfully disabled the Private URL for this unpublished dataset.
-dataset.privateurl.noPermToCreate=To create a Private URL you must have the following permissions: {0}.
+dataset.privateurl.full=This Preview URL provides full read access to the dataset
+dataset.privateurl.anonymized=This Preview URL provides access to the anonymized dataset
+dataset.privateurl.disabledSuccess=You have successfully disabled the Preview URL for this unpublished dataset.
+dataset.privateurl.noPermToCreate=To create a Preview URL you must have the following permissions: {0}.
dataset.externalstatus.header=Curation Status Changed
dataset.externalstatus.removed=Curation Status Removed
dataset.externalstatus.info=Curation Status is now "{0}"
@@ -1934,7 +1956,7 @@ file.downloadBtn.format.all=All File Formats + Information
file.downloadBtn.format.tab=Tab-Delimited
file.downloadBtn.format.original={0} (Original File Format)
file.downloadBtn.format.rdata=RData
-file.downloadBtn.format.var=Variable Metadata
+file.downloadBtn.format.var=DDI Codebook v2
file.downloadBtn.format.citation=Data File Citation
file.download.filetype.unknown=Original File Format
file.more.information.link=Link to more file information for
@@ -2498,6 +2520,7 @@ dataset.version.file.changed=Files (Changed File Metadata: {0}
dataset.version.file.changed2=; Changed File Metadata: {0}
dataset.version.variablemetadata.changed=Variable Metadata (Changed Variable Metadata: {0}
dataset.version.variablemetadata.changed2=; Changed Variable Metadata: {0}
+dataset.version.compare.incorrect.order=Compare requires the older dataset version to be listed first.
#DataversePage.java
dataverse.item.required=Required
@@ -2719,8 +2742,8 @@ datasets.api.grant.role.assignee.has.role.error=User already has this role for t
datasets.api.revoke.role.not.found.error="Role assignment {0} not found"
datasets.api.revoke.role.success=Role {0} revoked for assignee {1} in {2}
datasets.api.privateurl.error.datasetnotfound=Could not find dataset.
-datasets.api.privateurl.error.alreadyexists=Private URL already exists for this dataset.
-datasets.api.privateurl.error.notdraft=Can't create Private URL because the latest version of this dataset is not a draft.
+datasets.api.privateurl.error.alreadyexists=Preview URL already exists for this dataset.
+datasets.api.privateurl.error.notdraft=Can't create Preview URL because the latest version of this dataset is not a draft.
datasets.api.privateurl.anonymized.error.released=Can't create a URL for anonymized access because this dataset has been published.
datasets.api.creationdate=Date Created
datasets.api.modificationdate=Last Modified Date
diff --git a/src/main/java/propertyFiles/MimeTypeDetectionByFileExtension.properties b/src/main/java/propertyFiles/MimeTypeDetectionByFileExtension.properties
index 630539d912e..4507c22fdf8 100644
--- a/src/main/java/propertyFiles/MimeTypeDetectionByFileExtension.properties
+++ b/src/main/java/propertyFiles/MimeTypeDetectionByFileExtension.properties
@@ -15,6 +15,7 @@ m=text/x-matlab
mat=application/matlab-mat
md=text/markdown
mp3=audio/mp3
+m4a=audio/mp4
nii=image/nii
nc=application/netcdf
ods=application/vnd.oasis.opendocument.spreadsheet
diff --git a/src/main/resources/db/migration/V6.4.0.1.sql b/src/main/resources/db/migration/V6.4.0.1.sql
new file mode 100644
index 00000000000..0bcd87dd736
--- /dev/null
+++ b/src/main/resources/db/migration/V6.4.0.1.sql
@@ -0,0 +1,4 @@
+-- Adding a case-insensitive index related to #11003
+--
+
+CREATE UNIQUE INDEX IF NOT EXISTS INDEX_DVOBJECT_authority_protocol_upper_identifier ON dvobject (authority, protocol, UPPER(identifier));
\ No newline at end of file
diff --git a/src/main/resources/db/migration/V6.4.0.2.sql b/src/main/resources/db/migration/V6.4.0.2.sql
new file mode 100644
index 00000000000..bc4a85b278f
--- /dev/null
+++ b/src/main/resources/db/migration/V6.4.0.2.sql
@@ -0,0 +1,2 @@
+-- #10118
+ALTER TABLE customquestion ALTER COLUMN questionstring TYPE text;
diff --git a/src/main/resources/db/migration/V6.4.0.3.sql b/src/main/resources/db/migration/V6.4.0.3.sql
new file mode 100644
index 00000000000..307d8ed206c
--- /dev/null
+++ b/src/main/resources/db/migration/V6.4.0.3.sql
@@ -0,0 +1,2 @@
+-- Add this boolean flag to accommodate a new harvesting client feature
+ALTER TABLE harvestingclient ADD COLUMN IF NOT EXISTS useOaiIdAsPid BOOLEAN DEFAULT FALSE;
diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml
index 6de0f00e94e..9426884d349 100644
--- a/src/main/webapp/dataset.xhtml
+++ b/src/main/webapp/dataset.xhtml
@@ -20,6 +20,7 @@
+
-
+
@@ -834,7 +835,6 @@
/>
@@ -995,7 +995,6 @@
+
#{bundle['dataset.privateurl.introduction']}
+
+
+ #{bundle['dataset.privateurl.onlyone']}
+
+
+
+
+
+ #{bundle['dataset.privateurl.general.description']}
+
+
+
+
+
+
+ #{privateUrlLink}
+
+
+