From 3468bac511a367dd2b456f0cc8fc0d6a1e92528a Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 10 Nov 2023 13:44:32 -0500 Subject: [PATCH 01/54] restore changes --- .../10117-guestbook-question-size-limit-raised.md | 1 + src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java | 4 ++-- src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java | 2 +- .../V6.0.0.4__10017-failure-with-long-custom-question.sql | 1 + src/main/webapp/resources/iqbs/messages.xhtml | 2 +- 5 files changed, 6 insertions(+), 4 deletions(-) create mode 100644 doc/release-notes/10117-guestbook-question-size-limit-raised.md create mode 100644 src/main/resources/db/migration/V6.0.0.4__10017-failure-with-long-custom-question.sql diff --git a/doc/release-notes/10117-guestbook-question-size-limit-raised.md b/doc/release-notes/10117-guestbook-question-size-limit-raised.md new file mode 100644 index 00000000000..ab5e84d78fe --- /dev/null +++ b/doc/release-notes/10117-guestbook-question-size-limit-raised.md @@ -0,0 +1 @@ +Custom questions in Guestbooks can now be more than 255 characters and the bug causing a silent failure when questions were longer than this limit has been fixed. \ No newline at end of file diff --git a/src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java b/src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java index 2cb6f27c3e4..d880da5b4a8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java +++ b/src/main/java/edu/harvard/iq/dataverse/CustomQuestion.java @@ -2,7 +2,7 @@ import java.io.Serializable; import java.util.List; import jakarta.persistence.*; -import org.hibernate.validator.constraints.NotBlank; +import jakarta.validation.constraints.NotBlank; /** * @@ -41,7 +41,7 @@ public void setId(Long id) { private String questionType; @NotBlank(message = "{custom.questiontext}") - @Column( nullable = false ) + @Column( nullable = false, columnDefinition = "TEXT") private String questionString; private boolean required; diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java index 9fb584a9133..8b09291d052 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookPage.java @@ -320,7 +320,7 @@ public String save() { logger.info("Guestbook Page Command Exception. Dataverse: " + dataverse.getName()); logger.info(ex.toString()); FacesContext.getCurrentInstance().addMessage(null, new FacesMessage(FacesMessage.SEVERITY_FATAL, BundleUtil.getStringFromBundle("guestbook.save.fail"), " - " + ex.toString())); - //logger.severe(ex.getMessage()); + return null; } editMode = null; String msg = (create)? BundleUtil.getStringFromBundle("guestbook.create"): BundleUtil.getStringFromBundle("guestbook.save"); diff --git a/src/main/resources/db/migration/V6.0.0.4__10017-failure-with-long-custom-question.sql b/src/main/resources/db/migration/V6.0.0.4__10017-failure-with-long-custom-question.sql new file mode 100644 index 00000000000..9a3002378b3 --- /dev/null +++ b/src/main/resources/db/migration/V6.0.0.4__10017-failure-with-long-custom-question.sql @@ -0,0 +1 @@ +ALTER TABLE customquestion ALTER COLUMN questionstring TYPE text; diff --git a/src/main/webapp/resources/iqbs/messages.xhtml b/src/main/webapp/resources/iqbs/messages.xhtml index bd17cf34d21..f8e1f5e8e9d 100644 --- a/src/main/webapp/resources/iqbs/messages.xhtml +++ b/src/main/webapp/resources/iqbs/messages.xhtml @@ -63,7 +63,7 @@ Server: - #{systemConfig.dataverseServer} + #{systemConfig.dataverseSiteUrl} #{msg.rendered()} From 446d7f18183745d07365fd7741a33f28a0082b2f Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Wed, 20 Dec 2023 13:50:59 -0500 Subject: [PATCH 02/54] #10193 document configuring SELinux for Shibboleth --- .../source/installation/shibboleth.rst | 44 +++++-------------- 1 file changed, 11 insertions(+), 33 deletions(-) diff --git a/doc/sphinx-guides/source/installation/shibboleth.rst b/doc/sphinx-guides/source/installation/shibboleth.rst index 9f7c04c1534..d8bba1f0b49 100644 --- a/doc/sphinx-guides/source/installation/shibboleth.rst +++ b/doc/sphinx-guides/source/installation/shibboleth.rst @@ -199,43 +199,21 @@ The first and easiest option is to set ``SELINUX=permisive`` in ``/etc/selinux/c Reconfigure SELinux to Accommodate Shibboleth ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The second (more involved) option is to use the ``checkmodule``, ``semodule_package``, and ``semodule`` tools to apply a local policy to make Shibboleth work with SELinux. Let's get started. +Issue the following commands to allow Shibboleth to function when SELinux is enabled: -Put Type Enforcement (TE) File in misc directory -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Copy and paste or download the :download:`shibboleth.te <../_static/installation/files/etc/selinux/targeted/src/policy/domains/misc/shibboleth.te>` Type Enforcement (TE) file below and put it at ``/etc/selinux/targeted/src/policy/domains/misc/shibboleth.te``. - -.. literalinclude:: ../_static/installation/files/etc/selinux/targeted/src/policy/domains/misc/shibboleth.te - :language: text - -(If you would like to know where the ``shibboleth.te`` came from and how to hack on it, please see the :doc:`/developers/selinux` section of the Developer Guide. Pull requests are welcome!) - -Navigate to misc directory -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``cd /etc/selinux/targeted/src/policy/domains/misc`` - -Run checkmodule -^^^^^^^^^^^^^^^ - -``checkmodule -M -m -o shibboleth.mod shibboleth.te`` - -Run semodule_package -^^^^^^^^^^^^^^^^^^^^ - -``semodule_package -o shibboleth.pp -m shibboleth.mod`` - -Silent is golden. No output is expected. - -Run semodule -^^^^^^^^^^^^ +.. code-block:: none -``semodule -i shibboleth.pp`` + # Allow httpd to connect to network and read content + sudo /usr/sbin/setsebool -P httpd_can_network_connect 1 + sudo /usr/sbin/setsebool -P httpd_read_user_content 1 -Silent is golden. No output is expected. This will place a file in ``/etc/selinux/targeted/modules/active/modules/shibboleth.pp`` and include "shibboleth" in the output of ``semodule -l``. See the ``semodule`` man page if you ever want to remove or disable the module you just added. + # Allow httpd to connect to Shib socket + sudo grep httpd_t /var/log/audit/audit.log |/usr/bin/audit2allow -M allow_httpd_shibd_sock + sudo /usr/sbin/semodule -i allow_httpd_shibd_sock.pp -Congrats! You've made the creator of https://stopdisablingselinux.com proud. :) + # Allow httpd to read /var/cache/shibboleth + sudo /usr/sbin/semanage fcontext -a -t httpd_sys_content_t "/var/cache/shibboleth(/.*)?" + sudo /usr/sbin/restorecon -vR /var/cache/shibboleth Restart Apache and Shibboleth ----------------------------- From 9c3ac152ee3eaa3af7834ac6c17ac9bbee3570c2 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 12 Mar 2024 17:43:41 -0400 Subject: [PATCH 03/54] update flyway version --- ....sql => V6.1.0.6__10017-failure-with-long-custom-question.sql} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/resources/db/migration/{V6.0.0.4__10017-failure-with-long-custom-question.sql => V6.1.0.6__10017-failure-with-long-custom-question.sql} (100%) diff --git a/src/main/resources/db/migration/V6.0.0.4__10017-failure-with-long-custom-question.sql b/src/main/resources/db/migration/V6.1.0.6__10017-failure-with-long-custom-question.sql similarity index 100% rename from src/main/resources/db/migration/V6.0.0.4__10017-failure-with-long-custom-question.sql rename to src/main/resources/db/migration/V6.1.0.6__10017-failure-with-long-custom-question.sql From 709ecf735d85b5e7c57d2e4c952ff26021e5e768 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 29 Aug 2024 16:03:41 -0400 Subject: [PATCH 04/54] Updates to speed difference calcs and cleanup for terms differencing --- .../dataverse/DatasetVersionDifference.java | 438 +++++------------- 1 file changed, 118 insertions(+), 320 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java index eca0c84ae84..888c920b279 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java @@ -8,6 +8,8 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.logging.Logger; @@ -15,6 +17,7 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import java.util.Arrays; import java.util.Date; +import java.util.HashMap; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedList; @@ -106,50 +109,69 @@ public DatasetVersionDifference(DatasetVersion newVersion, DatasetVersion origin addToSummary(null, dsfn); } } - - // TODO: ? - // It looks like we are going through the filemetadatas in both versions, - // *sequentially* (i.e. at the cost of O(N*M)), to select the lists of - // changed, deleted and added files between the 2 versions... But why - // are we doing it, if we are doing virtually the same thing inside - // the initDatasetFilesDifferenceList(), below - but in a more efficient - // way (sorting both lists, then goint through them in parallel, at the - // cost of (N+M) max.? - // -- 4.6 Nov. 2016 - + logger.info("Start alt loop: " + System.currentTimeMillis()); + Map originalFileMetadataMap = new HashMap<>(); + Map previousIDtoFileMetadataMap = new HashMap<>(); for (FileMetadata fmdo : originalVersion.getFileMetadatas()) { - boolean deleted = true; - for (FileMetadata fmdn : newVersion.getFileMetadatas()) { - if (fmdo.getDataFile().equals(fmdn.getDataFile())) { - deleted = false; - if (!compareFileMetadatas(fmdo, fmdn)) { - changedFileMetadata.add(fmdo); - changedFileMetadata.add(fmdn); - } - if (!variableMetadataUtil.compareVariableMetadata(fmdo,fmdn) || !compareVarGroup(fmdo, fmdn)) { - changedVariableMetadata.add(fmdo); - changedVariableMetadata.add(fmdn); - } - break; - } - } - if (deleted) { - removedFiles.add(fmdo); - } + originalFileMetadataMap.put(fmdo.getDataFile().getId(), fmdo); } + logger.info("End getorigids loop: " + System.currentTimeMillis()); for (FileMetadata fmdn : newVersion.getFileMetadatas()) { - boolean added = true; - for (FileMetadata fmdo : originalVersion.getFileMetadatas()) { - if (fmdo.getDataFile().equals(fmdn.getDataFile())) { - added = false; - break; + DataFile ndf = fmdn.getDataFile(); + Long id = ndf.getId(); + FileMetadata fmdo = originalFileMetadataMap.get(id); + //If this file was in the original version + if(fmdo!= null) { + //Check for differences + if (!compareFileMetadatas(fmdo, fmdn)) { + logger.info("Adding file metadata diff: " + fmdo.getId()); + logger.info("Adding file metadata diff: " + fmdn.getId()); + changedFileMetadata.add(fmdo); + changedFileMetadata.add(fmdn); + } + if (!VariableMetadataUtil.compareVariableMetadata(fmdo,fmdn) || !compareVarGroup(fmdo, fmdn)) { + changedVariableMetadata.add(fmdo); + changedVariableMetadata.add(fmdn); + } + // And drop it from the list since it can't be a deleted file + originalFileMetadataMap.remove(id); + } else { + //It wasn't in the original version + Long prevID = ndf.getPreviousDataFileId(); + //It might be a replacement file or an added file + if(prevID != null) { + //Add it to a map so we can check later to see if it's a replacement + previousIDtoFileMetadataMap.put(prevID, fmdn); + } else { + //Otherwise make it an added file now + addedFiles.add(fmdn); } } - if (added) { - addedFiles.add(fmdn); + } + //Finally check any remaining files from the original version that weren't in the new version' + for (Long removedId : originalFileMetadataMap.keySet()) { + //See if it has been replaced + FileMetadata replacingFmd = previousIDtoFileMetadataMap.get(removedId); + FileMetadata fmdRemoved = originalFileMetadataMap.get(removedId); + if (replacingFmd != null) { + //This is a replacement + replacedFiles.add(new FileMetadata[] { fmdRemoved, replacingFmd }); + //Drop if from the map + previousIDtoFileMetadataMap.remove(removedId); + } else { + //This is a removed file + removedFiles.add(fmdRemoved); } - } - getReplacedFiles(); + } + // Any fms left are not updating existing files and aren't replacing a file, but + // they are claiming a previous file id. That shouldn't be possible, but this will + // make sure they get listed in the difference if they do + for (Entry entry : previousIDtoFileMetadataMap.entrySet()) { + logger.warning("Previous file id claimed for a new file: fmd id: " + entry.getValue() + ", previous file id: " + entry.getKey()); + addedFiles.add(entry.getValue()); + } + + logger.info("End alt loop: " + System.currentTimeMillis()); initDatasetFilesDifferencesList(); //Sort within blocks by datasetfieldtype dispaly order then.... @@ -173,294 +195,62 @@ public DatasetVersionDifference(DatasetVersion newVersion, DatasetVersion origin getTermsDifferences(); } - private void getReplacedFiles() { - if (addedFiles.isEmpty() || removedFiles.isEmpty()) { - return; - } - List addedToReplaced = new ArrayList<>(); - List removedToReplaced = new ArrayList<>(); - for (FileMetadata added : addedFiles) { - DataFile addedDF = added.getDataFile(); - Long replacedId = addedDF.getPreviousDataFileId(); - if (added.getDataFile().getPreviousDataFileId() != null){ - } - for (FileMetadata removed : removedFiles) { - DataFile test = removed.getDataFile(); - if (test.getId().equals(replacedId)) { - addedToReplaced.add(added); - removedToReplaced.add(removed); - FileMetadata[] replacedArray = new FileMetadata[2]; - replacedArray[0] = removed; - replacedArray[1] = added; - replacedFiles.add(replacedArray); - } - } - } - if(addedToReplaced.isEmpty()){ - } else{ - addedToReplaced.stream().forEach((delete) -> { - addedFiles.remove(delete); - }); - removedToReplaced.stream().forEach((delete) -> { - removedFiles.remove(delete); - }); - } - } + private void getTermsDifferences() { - changedTermsAccess = new ArrayList<>(); - if (newVersion.getTermsOfUseAndAccess() != null && originalVersion.getTermsOfUseAndAccess() != null) { - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getTermsOfUse()).equals(StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getTermsOfUse()))) { - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.header"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getTermsOfUse()), StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getTermsOfUse())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getConfidentialityDeclaration()).equals(StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getConfidentialityDeclaration()))) { - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.declaration"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getConfidentialityDeclaration()), - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getConfidentialityDeclaration())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getSpecialPermissions()).equals(StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getSpecialPermissions()))) { - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.permissions"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getSpecialPermissions()), - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getSpecialPermissions())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getRestrictions()).equals(StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getRestrictions()))) { - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.restrictions"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getRestrictions()), - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getRestrictions())); - - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getCitationRequirements()).equals(StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getCitationRequirements()))) { - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.citationRequirements"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getCitationRequirements()), - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getCitationRequirements())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getDepositorRequirements()).equals(StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getDepositorRequirements()))) { - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.depositorRequirements"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getDepositorRequirements()), - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getDepositorRequirements())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getConditions()).equals(StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getConditions()))) { - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.conditions"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getConditions()), - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getConditions())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getDisclaimer()).equals(StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getDisclaimer()))) { - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.disclaimer"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getDisclaimer()), StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getDisclaimer())); - } - - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getTermsOfAccess()).equals(StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getTermsOfAccess()))) { - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.termsOfsAccess"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getTermsOfAccess()), - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getTermsOfAccess())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getDataAccessPlace()).equals(StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getDataAccessPlace()))) { - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.dataAccessPlace"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getDataAccessPlace()), - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getDataAccessPlace())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getOriginalArchive()).equals(StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getOriginalArchive()))) { - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.originalArchive"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getOriginalArchive()), - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getOriginalArchive())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getAvailabilityStatus()).equals(StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getAvailabilityStatus()))) { - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.availabilityStatus"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getAvailabilityStatus()), - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getAvailabilityStatus())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getContactForAccess()).equals(StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getContactForAccess()))) { - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.contactForAccess"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getContactForAccess()), - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getContactForAccess())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getSizeOfCollection()).equals(StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getSizeOfCollection()))) { - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.sizeOfCollection"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getSizeOfCollection()), - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getSizeOfCollection())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getStudyCompletion()).equals(StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getStudyCompletion()))) { - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.studyCompletion"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getStudyCompletion()), - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getStudyCompletion())); - } + TermsOfUseAndAccess originalTerms = originalVersion.getTermsOfUseAndAccess(); + if(originalTerms == null) { + originalTerms = new TermsOfUseAndAccess(); + } + // newTerms should never be null + TermsOfUseAndAccess newTerms = newVersion.getTermsOfUseAndAccess(); + if(newTerms == null) { + logger.warning("New version does not have TermsOfUseAndAccess"); + newTerms = new TermsOfUseAndAccess(); } - - if (newVersion.getTermsOfUseAndAccess() != null && originalVersion.getTermsOfUseAndAccess() == null) { - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getTermsOfUse()).isEmpty()) { - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.header"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, "", StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getTermsOfUse())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getConfidentialityDeclaration()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.declaration"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, "", - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getConfidentialityDeclaration())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getSpecialPermissions()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.permissions"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, "", - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getSpecialPermissions())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getRestrictions()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.restrictions"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, "", - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getRestrictions())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getCitationRequirements()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.citationRequirements"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, "", - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getCitationRequirements())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getDepositorRequirements()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.depositorRequirements"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, "", - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getDepositorRequirements())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getConditions()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.conditions"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, "", - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getConditions())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getDisclaimer()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.disclaimer"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, "", StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getDisclaimer())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getTermsOfAccess()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.termsOfsAccess"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, "", - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getTermsOfAccess())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getDataAccessPlace()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.dataAccessPlace"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, "", - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getDataAccessPlace())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getOriginalArchive()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.originalArchive"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, "", - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getOriginalArchive())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getAvailabilityStatus()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.availabilityStatus"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, "", - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getAvailabilityStatus())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getContactForAccess()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.contactForAccess"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, "", - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getContactForAccess())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getSizeOfCollection()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.sizeOfCollection"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, "", - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getSizeOfCollection())); - } - if (!StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getStudyCompletion()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.studyCompletion"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, "", - StringUtil.nullToEmpty(newVersion.getTermsOfUseAndAccess().getStudyCompletion())); - } - } - - if (newVersion.getTermsOfUseAndAccess() == null && originalVersion.getTermsOfUseAndAccess() != null) { - if (!StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getTermsOfUse()).isEmpty()) { - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.header"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getTermsOfUse()), ""); - } - if (!StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getConfidentialityDeclaration()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.declaration"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, - StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getConfidentialityDeclaration()), ""); - } - if (!StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getSpecialPermissions()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.permissions"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, - StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getSpecialPermissions()), ""); - } - if (!StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getRestrictions()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.restrictions"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, - StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getRestrictions()), ""); - } - if (!StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getCitationRequirements()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.citationRequirements"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, - StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getCitationRequirements()), ""); - } - if (!StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getDepositorRequirements()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.depositorRequirements"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, - StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getDepositorRequirements()), ""); - } - if (!StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getConditions()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.conditions"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, - StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getConditions()), ""); - } - if (!StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getDisclaimer()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.disclaimer"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getDisclaimer()), ""); - } - if (!StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getTermsOfAccess()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.termsOfsAccess"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, - StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getTermsOfAccess()), ""); - } - if (!StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getDataAccessPlace()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.dataAccessPlace"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, - StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getDataAccessPlace()), ""); - } - if (!StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getOriginalArchive()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.originalArchive"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, - StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getOriginalArchive()), ""); - } - if (!StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getAvailabilityStatus()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.availabilityStatus"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, - StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getAvailabilityStatus()), ""); - } - if (!StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getContactForAccess()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.contactForAccess"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, - StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getContactForAccess()), ""); - } - if (!StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getSizeOfCollection()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.sizeOfCollection"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, - StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getSizeOfCollection()), ""); - } - if (!StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getStudyCompletion()).isEmpty()){ - String diffLabel = BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.studyCompletion"); - changedTermsAccess = addToTermsChangedList(changedTermsAccess, diffLabel, - StringUtil.nullToEmpty(originalVersion.getTermsOfUseAndAccess().getStudyCompletion()), ""); - } - } - } - - private DifferenceSummaryItem createSummaryItem(){ - return null; - } - - private List addToSummaryGroup(String displayName, DifferenceSummaryItem differenceSummaryItem){ - return null; + checkAndAddToChangeList(originalTerms.getTermsOfUse(), newTerms.getTermsOfUse(), + BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.header")); + checkAndAddToChangeList(originalTerms.getConfidentialityDeclaration(), newTerms.getConfidentialityDeclaration(), + BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.declaration")); + checkAndAddToChangeList(originalTerms.getSpecialPermissions(), newTerms.getSpecialPermissions(), + BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.permissions")); + checkAndAddToChangeList(originalTerms.getRestrictions(), newTerms.getRestrictions(), + BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.restrictions")); + checkAndAddToChangeList(originalTerms.getCitationRequirements(), newTerms.getCitationRequirements(), + BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.citationRequirements")); + checkAndAddToChangeList(originalTerms.getDepositorRequirements(), newTerms.getDepositorRequirements(), + BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.depositorRequirements")); + checkAndAddToChangeList(originalTerms.getConditions(), newTerms.getConditions(), + BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.conditions")); + checkAndAddToChangeList(originalTerms.getDisclaimer(), newTerms.getDisclaimer(), + BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.disclaimer")); + checkAndAddToChangeList(originalTerms.getTermsOfAccess(), newTerms.getTermsOfAccess(), + BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.termsOfsAccess")); + checkAndAddToChangeList(originalTerms.getDataAccessPlace(), newTerms.getDataAccessPlace(), + BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.dataAccessPlace")); + checkAndAddToChangeList(originalTerms.getOriginalArchive(), newTerms.getOriginalArchive(), + BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.originalArchive")); + checkAndAddToChangeList(originalTerms.getAvailabilityStatus(), newTerms.getAvailabilityStatus(), + BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.availabilityStatus")); + checkAndAddToChangeList(originalTerms.getContactForAccess(), newTerms.getContactForAccess(), + BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.contactForAccess")); + checkAndAddToChangeList(originalTerms.getSizeOfCollection(), newTerms.getSizeOfCollection(), + BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.sizeOfCollection")); + checkAndAddToChangeList(originalTerms.getStudyCompletion(), newTerms.getStudyCompletion(), + BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfAccess.addInfo.studyCompletion")); } - - private List addToTermsChangedList(List listIn, String label, String origVal, String newVal) { - String[] diffArray; - diffArray = new String[3]; - diffArray[0] = label; - diffArray[1] = origVal; - diffArray[2] = newVal; - listIn.add(diffArray); - return listIn; + + private void checkAndAddToChangeList(String originalTerm, String newTerm, + String termLabel) { + originalTerm = StringUtil.nullToEmpty(originalTerm); + newTerm = StringUtil.nullToEmpty(newTerm); + if(!originalTerm.equals(newTerm)) { + changedTermsAccess.add(new String[]{termLabel, originalTerm, newTerm}); + } } - private void addToList(List listIn, DatasetField dsfo, DatasetField dsfn) { DatasetField[] dsfArray; dsfArray = new DatasetField[2]; @@ -523,7 +313,7 @@ private void addToNoteSummary(DatasetField dsfo, int added, int deleted, int cha summaryDataForNote.add(noteArray); } - private boolean compareVarGroup(FileMetadata fmdo, FileMetadata fmdn) { + static boolean compareVarGroup(FileMetadata fmdo, FileMetadata fmdn) { List vglo = fmdo.getVarGroups(); List vgln = fmdn.getVarGroups(); @@ -533,7 +323,7 @@ private boolean compareVarGroup(FileMetadata fmdo, FileMetadata fmdn) { int count = 0; for (VarGroup vgo : vglo) { for (VarGroup vgn : vgln) { - if (!variableMetadataUtil.checkDiff(vgo.getLabel(), vgn.getLabel())) { + if (!VariableMetadataUtil.checkDiff(vgo.getLabel(), vgn.getLabel())) { Set dvo = vgo.getVarsInGroup(); Set dvn = vgn.getVarsInGroup(); if (dvo.equals(dvn)) { @@ -1819,4 +1609,12 @@ private static boolean fieldsAreDifferent(DatasetField originalField, DatasetFie } return false; } + + List getgetChangedVariableMetadata() { + return changedVariableMetadata; + } + + List getReplacedFiles() { + return replacedFiles; + } } From 4bb39433aa7c3ba8435143127e0a7b2e85e1dab1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 29 Aug 2024 16:04:02 -0400 Subject: [PATCH 05/54] tests for metadata, varmetadata, replace,add, remove cases --- .../DatasetVersionDifferenceTest.java | 299 ++++++++++++++++++ 1 file changed, 299 insertions(+) create mode 100644 src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java new file mode 100644 index 00000000000..36c69605dda --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java @@ -0,0 +1,299 @@ +package edu.harvard.iq.dataverse; + +import edu.harvard.iq.dataverse.DatasetVersionDifference.datasetFileDifferenceItem; +import edu.harvard.iq.dataverse.DatasetVersionDifference.datasetReplaceFileItem; +import edu.harvard.iq.dataverse.branding.BrandingUtilTest; +import edu.harvard.iq.dataverse.datavariable.VariableMetadata; +import edu.harvard.iq.dataverse.datavariable.VariableMetadataUtil; +import edu.harvard.iq.dataverse.license.License; +import java.net.URI; +import java.sql.Timestamp; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.List; +import java.util.logging.Logger; +import static org.junit.jupiter.api.Assertions.assertEquals; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Test; + +public class DatasetVersionDifferenceTest { + + private static final Logger logger = Logger.getLogger(DatasetVersion.class.getCanonicalName()); + + private static List addedFiles; + private static List removedFiles; + private static List changedFileMetadata; + private static List changedVariableMetadata; + private static List replacedFiles; + + @BeforeAll + public static void setUp() { + BrandingUtilTest.setupMocks(); + } + + @AfterAll + public static void tearDown() { + BrandingUtilTest.setupMocks(); + } + + @Test + public void testDifferencing() { + Dataset dataset = new Dataset(); + License license = new License("CC0 1.0", + "You can copy, modify, distribute and perform the work, even for commercial purposes, all without asking permission.", + URI.create("http://creativecommons.org/publicdomain/zero/1.0"), URI.create("/resources/images/cc0.png"), + true, 1l); + license.setDefault(true); + dataset.setProtocol("doi"); + dataset.setAuthority("10.5072/FK2"); + dataset.setIdentifier("LK0D1H"); + DatasetVersion datasetVersion = new DatasetVersion(); + datasetVersion.setDataset(dataset); + datasetVersion.setVersionState(DatasetVersion.VersionState.RELEASED); + datasetVersion.setVersionNumber(1L); + DatasetVersion datasetVersion2 = new DatasetVersion(); + datasetVersion2.setDataset(dataset); + datasetVersion2.setVersionState(DatasetVersion.VersionState.DRAFT); + + // Published version's two files + DataFile dataFile = new DataFile(); + dataFile.setId(1L); + DataFile dataFile2 = new DataFile(); + dataFile2.setId(2L); + + FileMetadata fileMetadata1 = createFileMetadata(10L, datasetVersion, dataFile, "file1.txt"); + fileMetadata1.setLabel("file1.txt"); + + FileMetadata fileMetadata2 = createFileMetadata(20L, datasetVersion, dataFile2, "file2.txt"); + + // Draft version - same two files with one label change + FileMetadata fileMetadata3 = fileMetadata1.createCopy(); + fileMetadata3.setId(30L); + + FileMetadata fileMetadata4 = fileMetadata2.createCopy(); + fileMetadata4.setLabel("file3.txt"); + fileMetadata4.setId(40L); + + List fileMetadatas = new ArrayList<>(Arrays.asList(fileMetadata1, fileMetadata2)); + datasetVersion.setFileMetadatas(fileMetadatas); + List fileMetadatas2 = new ArrayList<>(Arrays.asList(fileMetadata3, fileMetadata4)); + datasetVersion2.setFileMetadatas(fileMetadatas2); + + SimpleDateFormat dateFmt = new SimpleDateFormat("yyyyMMdd"); + Date publicationDate; + try { + publicationDate = dateFmt.parse("19551105"); + datasetVersion.setReleaseTime(publicationDate); + dataset.setPublicationDate(new Timestamp(publicationDate.getTime())); + } catch (ParseException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + List versionList = new ArrayList<>(Arrays.asList(datasetVersion, datasetVersion2)); + dataset.setVersions(versionList); + + // One file has a changed label + List expectedAddedFiles = new ArrayList<>(); + List expectedRemovedFiles = new ArrayList<>(); + ; + List expectedChangedFileMetadata = Arrays.asList(fileMetadata2, fileMetadata4); + List expectedChangedVariableMetadata = new ArrayList<>(); + List expectedReplacedFiles = new ArrayList<>(); + + compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, + expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles); + // change label for first file as well + fileMetadata3.setLabel("file1_updated.txt"); + expectedChangedFileMetadata = Arrays.asList(fileMetadata1, fileMetadata3, fileMetadata2, fileMetadata4); + compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, + expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles); + // Add one change to variable metadata + fileMetadata3.setVariableMetadatas(Arrays.asList(new VariableMetadata())); + expectedChangedVariableMetadata = Arrays.asList(fileMetadata1, fileMetadata3); + compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, + expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles); + // Replaced File + DataFile replacingFile = new DataFile(); + replacingFile.setId(3L); + replacingFile.setPreviousDataFileId(1L); + fileMetadata3.setDataFile(replacingFile); + expectedChangedFileMetadata = Arrays.asList(fileMetadata2, fileMetadata4); + expectedChangedVariableMetadata = new ArrayList<>(); + + FileMetadata[] filePair = new FileMetadata[2]; + filePair[0] = fileMetadata1; + filePair[1] = fileMetadata3; + expectedReplacedFiles = new ArrayList<>(); + expectedReplacedFiles.add(filePair); + compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, + expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles); + + // Add a new file + DataFile newFile = new DataFile(); + newFile.setId(3L); + FileMetadata fileMetadata5 = createFileMetadata(50L, datasetVersion2, newFile, "newFile.txt"); + datasetVersion2.getFileMetadatas().add(fileMetadata5); + expectedAddedFiles = Arrays.asList(fileMetadata5); + compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, + expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles); + + // Remove a file + datasetVersion2.getFileMetadatas().remove(fileMetadata4); + expectedRemovedFiles = Arrays.asList(fileMetadata2); + expectedChangedFileMetadata = new ArrayList<>(); + compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, + expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles); + } + + private FileMetadata createFileMetadata(long id, DatasetVersion datasetVersion, DataFile dataFile, String label) { + FileMetadata fileMetadata = new FileMetadata(); + fileMetadata.setId(id); + fileMetadata.setDatasetVersion(datasetVersion); + fileMetadata.setDataFile(dataFile); + fileMetadata.setLabel(label); + fileMetadata.setCategories(new ArrayList<>()); + return fileMetadata; + } + + /** + * CompareResults is currently testing the output of the + * DatasetVersionDifference class with the manually created expected results + * included as parameters and with the results of the less efficient algorithm + * it is replacing. Once we're collectively convinced that the tests here are + * correct (i.e. the manually created expected* parameters are set correctly for + * each use case), we could drop running the originalCalculateDifference method + * and just compare with the expected* results. + */ + private void compareResults(DatasetVersion datasetVersion, DatasetVersion datasetVersion2, + List expectedAddedFiles, List expectedRemovedFiles, + List expectedChangedFileMetadata, List expectedChangedVariableMetadata, + List expectedReplacedFiles) { + DatasetVersionDifference diff = new DatasetVersionDifference(datasetVersion2, datasetVersion); + // Run the original algorithm + originalCalculateDifference(datasetVersion2, datasetVersion); + // Compare the old and new algorithms + assertEquals(addedFiles, diff.getAddedFiles()); + assertEquals(removedFiles, diff.getRemovedFiles()); + assertEquals(changedFileMetadata, diff.getChangedFileMetadata()); + assertEquals(changedVariableMetadata, diff.getgetChangedVariableMetadata()); + assertEquals(replacedFiles.size(), diff.getReplacedFiles().size()); + for (int i = 0; i < replacedFiles.size(); i++) { + assertEquals(replacedFiles.get(i)[0], diff.getReplacedFiles().get(i)[0]); + assertEquals(replacedFiles.get(i)[1], diff.getReplacedFiles().get(i)[1]); + } + + // Also compare the new algorithm with the manually created expected* values for + // the test cases + assertEquals(expectedAddedFiles, diff.getAddedFiles()); + assertEquals(expectedRemovedFiles, diff.getRemovedFiles()); + assertEquals(expectedChangedFileMetadata, diff.getChangedFileMetadata()); + assertEquals(expectedChangedVariableMetadata, diff.getgetChangedVariableMetadata()); + assertEquals(expectedReplacedFiles.size(), diff.getReplacedFiles().size()); + for (int i = 0; i < expectedReplacedFiles.size(); i++) { + assertEquals(expectedReplacedFiles.get(i)[0], diff.getReplacedFiles().get(i)[0]); + assertEquals(expectedReplacedFiles.get(i)[1], diff.getReplacedFiles().get(i)[1]); + } + + } + + @Deprecated + // This is the "Original" difference calculation from DatasetVersionDifference + // It is included here to help verify that the new implementation is the same as + // the original + private static void originalCalculateDifference(DatasetVersion newVersion, DatasetVersion originalVersion) { + + addedFiles = new ArrayList<>(); + removedFiles = new ArrayList<>(); + changedFileMetadata = new ArrayList<>(); + changedVariableMetadata = new ArrayList<>(); + replacedFiles = new ArrayList<>(); + logger.info("Start orig loops: " + System.currentTimeMillis()); + // TODO: ? + // It looks like we are going through the filemetadatas in both versions, + // *sequentially* (i.e. at the cost of O(N*M)), to select the lists of + // changed, deleted and added files between the 2 versions... But why + // are we doing it, if we are doing virtually the same thing inside + // the initDatasetFilesDifferenceList(), below - but in a more efficient + // way (sorting both lists, then goint through them in parallel, at the + // cost of (N+M) max.? + // -- 4.6 Nov. 2016 + + for (FileMetadata fmdo : originalVersion.getFileMetadatas()) { + boolean deleted = true; + for (FileMetadata fmdn : newVersion.getFileMetadatas()) { + if (fmdo.getDataFile().equals(fmdn.getDataFile())) { + deleted = false; + if (!DatasetVersionDifference.compareFileMetadatas(fmdo, fmdn)) { + changedFileMetadata.add(fmdo); + changedFileMetadata.add(fmdn); + } + if (!VariableMetadataUtil.compareVariableMetadata(fmdo, fmdn) + || !DatasetVersionDifference.compareVarGroup(fmdo, fmdn)) { + changedVariableMetadata.add(fmdo); + changedVariableMetadata.add(fmdn); + } + break; + } + } + if (deleted) { + removedFiles.add(fmdo); + } + } + for (FileMetadata fmdn : newVersion.getFileMetadatas()) { + boolean added = true; + for (FileMetadata fmdo : originalVersion.getFileMetadatas()) { + if (fmdo.getDataFile().equals(fmdn.getDataFile())) { + added = false; + break; + } + } + if (added) { + addedFiles.add(fmdn); + } + } + + getReplacedFiles(); + logger.info("End org loops: " + System.currentTimeMillis()); + + } + + @Deprecated + // This is used only in the original algorithm and was removed from DatasetVersionDifference + private static void getReplacedFiles() { + if (addedFiles.isEmpty() || removedFiles.isEmpty()) { + return; + } + List addedToReplaced = new ArrayList<>(); + List removedToReplaced = new ArrayList<>(); + for (FileMetadata added : addedFiles) { + DataFile addedDF = added.getDataFile(); + Long replacedId = addedDF.getPreviousDataFileId(); + if (added.getDataFile().getPreviousDataFileId() != null) { + } + for (FileMetadata removed : removedFiles) { + DataFile test = removed.getDataFile(); + if (test.getId().equals(replacedId)) { + addedToReplaced.add(added); + removedToReplaced.add(removed); + FileMetadata[] replacedArray = new FileMetadata[2]; + replacedArray[0] = removed; + replacedArray[1] = added; + replacedFiles.add(replacedArray); + } + } + } + if (addedToReplaced.isEmpty()) { + } else { + addedToReplaced.stream().forEach((delete) -> { + addedFiles.remove(delete); + }); + removedToReplaced.stream().forEach((delete) -> { + removedFiles.remove(delete); + }); + } + } +} From 8437c63104b0c9c789bd4ab10727fc27f6db91ea Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 29 Aug 2024 16:37:16 -0400 Subject: [PATCH 06/54] tests for terms cleanup --- .../DatasetVersionDifferenceTest.java | 61 ++++++++++++++++--- 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java index 36c69605dda..afc708b86ad 100644 --- a/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java @@ -6,6 +6,8 @@ import edu.harvard.iq.dataverse.datavariable.VariableMetadata; import edu.harvard.iq.dataverse.datavariable.VariableMetadataUtil; import edu.harvard.iq.dataverse.license.License; +import edu.harvard.iq.dataverse.util.BundleUtil; + import java.net.URI; import java.sql.Timestamp; import java.text.ParseException; @@ -55,6 +57,7 @@ public void testDifferencing() { datasetVersion.setDataset(dataset); datasetVersion.setVersionState(DatasetVersion.VersionState.RELEASED); datasetVersion.setVersionNumber(1L); + datasetVersion.setTermsOfUseAndAccess(new TermsOfUseAndAccess()); DatasetVersion datasetVersion2 = new DatasetVersion(); datasetVersion2.setDataset(dataset); datasetVersion2.setVersionState(DatasetVersion.VersionState.DRAFT); @@ -103,19 +106,20 @@ public void testDifferencing() { List expectedChangedFileMetadata = Arrays.asList(fileMetadata2, fileMetadata4); List expectedChangedVariableMetadata = new ArrayList<>(); List expectedReplacedFiles = new ArrayList<>(); + List changedTerms = new ArrayList<>(); compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, - expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles); + expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles, changedTerms); // change label for first file as well fileMetadata3.setLabel("file1_updated.txt"); expectedChangedFileMetadata = Arrays.asList(fileMetadata1, fileMetadata3, fileMetadata2, fileMetadata4); compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, - expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles); + expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles, changedTerms); // Add one change to variable metadata fileMetadata3.setVariableMetadatas(Arrays.asList(new VariableMetadata())); expectedChangedVariableMetadata = Arrays.asList(fileMetadata1, fileMetadata3); compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, - expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles); + expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles, changedTerms); // Replaced File DataFile replacingFile = new DataFile(); replacingFile.setId(3L); @@ -130,7 +134,7 @@ public void testDifferencing() { expectedReplacedFiles = new ArrayList<>(); expectedReplacedFiles.add(filePair); compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, - expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles); + expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles, changedTerms); // Add a new file DataFile newFile = new DataFile(); @@ -139,14 +143,47 @@ public void testDifferencing() { datasetVersion2.getFileMetadatas().add(fileMetadata5); expectedAddedFiles = Arrays.asList(fileMetadata5); compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, - expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles); + expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles, changedTerms); // Remove a file datasetVersion2.getFileMetadatas().remove(fileMetadata4); expectedRemovedFiles = Arrays.asList(fileMetadata2); expectedChangedFileMetadata = new ArrayList<>(); compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, - expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles); + expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles, changedTerms); + + // Set the published version's TermsOfUseAndAccess to a non-null value + TermsOfUseAndAccess termsOfUseAndAccess = new TermsOfUseAndAccess(); + datasetVersion.setTermsOfUseAndAccess(termsOfUseAndAccess); + + compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, + expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles, changedTerms); + + // Set the draft version's TermsOfUseAndAccess to a non-null value + + datasetVersion2.setTermsOfUseAndAccess(new TermsOfUseAndAccess()); + + compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, + expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles, changedTerms); + + // Set a term field + + datasetVersion2.getTermsOfUseAndAccess().setTermsOfUse("Terms o' Use"); + String[] termField = new String[]{BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.header"), "", "Terms o' Use"}; + changedTerms.add(termField); + + compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, + expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles, changedTerms); + + // Set a term field in the original version + + datasetVersion.getTermsOfUseAndAccess().setDisclaimer("Not our fault"); + String[] termField2 = new String[]{BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.disclaimer"), "Not our fault", ""}; + changedTerms.add(termField2); + + compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, + expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles, changedTerms); + } private FileMetadata createFileMetadata(long id, DatasetVersion datasetVersion, DataFile dataFile, String label) { @@ -167,11 +204,12 @@ private FileMetadata createFileMetadata(long id, DatasetVersion datasetVersion, * correct (i.e. the manually created expected* parameters are set correctly for * each use case), we could drop running the originalCalculateDifference method * and just compare with the expected* results. + * @param changedTerms */ private void compareResults(DatasetVersion datasetVersion, DatasetVersion datasetVersion2, List expectedAddedFiles, List expectedRemovedFiles, List expectedChangedFileMetadata, List expectedChangedVariableMetadata, - List expectedReplacedFiles) { + List expectedReplacedFiles, List changedTerms) { DatasetVersionDifference diff = new DatasetVersionDifference(datasetVersion2, datasetVersion); // Run the original algorithm originalCalculateDifference(datasetVersion2, datasetVersion); @@ -197,7 +235,14 @@ private void compareResults(DatasetVersion datasetVersion, DatasetVersion datase assertEquals(expectedReplacedFiles.get(i)[0], diff.getReplacedFiles().get(i)[0]); assertEquals(expectedReplacedFiles.get(i)[1], diff.getReplacedFiles().get(i)[1]); } - + + assertEquals(changedTerms.size(), diff.getChangedTermsAccess().size()); + for (int i = 0; i < changedTerms.size(); i++) { + String[] diffArray = diff.getChangedTermsAccess().get(i); + assertEquals(changedTerms.get(i)[0], diffArray[0]); + assertEquals(changedTerms.get(i)[1], diffArray[1]); + assertEquals(changedTerms.get(i)[2], diffArray[2]); + } } @Deprecated From 4f4736efb4d695f1a6e553ebe352a09f4f27ca02 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 29 Aug 2024 16:55:02 -0400 Subject: [PATCH 07/54] Cleanup logging --- .../harvard/iq/dataverse/DatasetVersionDifference.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java index 888c920b279..63b207afe13 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java @@ -43,8 +43,6 @@ public final class DatasetVersionDifference { private List summaryDataForNote = new ArrayList<>(); private List blockDataForNote = new ArrayList<>(); - private VariableMetadataUtil variableMetadataUtil; - private List differenceSummaryGroups = new ArrayList<>(); public List getDifferenceSummaryGroups() { @@ -109,13 +107,13 @@ public DatasetVersionDifference(DatasetVersion newVersion, DatasetVersion origin addToSummary(null, dsfn); } } - logger.info("Start alt loop: " + System.currentTimeMillis()); + long startTime = System.currentTimeMillis(); Map originalFileMetadataMap = new HashMap<>(); Map previousIDtoFileMetadataMap = new HashMap<>(); for (FileMetadata fmdo : originalVersion.getFileMetadatas()) { originalFileMetadataMap.put(fmdo.getDataFile().getId(), fmdo); } - logger.info("End getorigids loop: " + System.currentTimeMillis()); + for (FileMetadata fmdn : newVersion.getFileMetadatas()) { DataFile ndf = fmdn.getDataFile(); Long id = ndf.getId(); @@ -124,8 +122,6 @@ public DatasetVersionDifference(DatasetVersion newVersion, DatasetVersion origin if(fmdo!= null) { //Check for differences if (!compareFileMetadatas(fmdo, fmdn)) { - logger.info("Adding file metadata diff: " + fmdo.getId()); - logger.info("Adding file metadata diff: " + fmdn.getId()); changedFileMetadata.add(fmdo); changedFileMetadata.add(fmdn); } @@ -171,7 +167,7 @@ public DatasetVersionDifference(DatasetVersion newVersion, DatasetVersion origin addedFiles.add(entry.getValue()); } - logger.info("End alt loop: " + System.currentTimeMillis()); + logger.fine("Main difference loop execution time: " + (System.currentTimeMillis() - startTime) + " ms"); initDatasetFilesDifferencesList(); //Sort within blocks by datasetfieldtype dispaly order then.... From 1fc6f51f99d8bf1e3c782a68ba1cd2439b3add73 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 29 Aug 2024 16:56:26 -0400 Subject: [PATCH 08/54] Show loop execution time in log --- .../DatasetVersionDifferenceTest.java | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java index afc708b86ad..44ac267abaf 100644 --- a/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java @@ -151,36 +151,39 @@ public void testDifferencing() { expectedChangedFileMetadata = new ArrayList<>(); compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles, changedTerms); - + // Set the published version's TermsOfUseAndAccess to a non-null value TermsOfUseAndAccess termsOfUseAndAccess = new TermsOfUseAndAccess(); datasetVersion.setTermsOfUseAndAccess(termsOfUseAndAccess); - + compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles, changedTerms); - + // Set the draft version's TermsOfUseAndAccess to a non-null value - + datasetVersion2.setTermsOfUseAndAccess(new TermsOfUseAndAccess()); - + compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles, changedTerms); - + // Set a term field - + datasetVersion2.getTermsOfUseAndAccess().setTermsOfUse("Terms o' Use"); - String[] termField = new String[]{BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.header"), "", "Terms o' Use"}; + String[] termField = new String[] { + BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.header"), "", "Terms o' Use" }; changedTerms.add(termField); - + compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles, changedTerms); - + // Set a term field in the original version - + datasetVersion.getTermsOfUseAndAccess().setDisclaimer("Not our fault"); - String[] termField2 = new String[]{BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.disclaimer"), "Not our fault", ""}; + String[] termField2 = new String[] { + BundleUtil.getStringFromBundle("file.dataFilesTab.terms.list.termsOfUse.addInfo.disclaimer"), + "Not our fault", "" }; changedTerms.add(termField2); - + compareResults(datasetVersion, datasetVersion2, expectedAddedFiles, expectedRemovedFiles, expectedChangedFileMetadata, expectedChangedVariableMetadata, expectedReplacedFiles, changedTerms); @@ -204,7 +207,8 @@ private FileMetadata createFileMetadata(long id, DatasetVersion datasetVersion, * correct (i.e. the manually created expected* parameters are set correctly for * each use case), we could drop running the originalCalculateDifference method * and just compare with the expected* results. - * @param changedTerms + * + * @param changedTerms */ private void compareResults(DatasetVersion datasetVersion, DatasetVersion datasetVersion2, List expectedAddedFiles, List expectedRemovedFiles, @@ -235,7 +239,7 @@ private void compareResults(DatasetVersion datasetVersion, DatasetVersion datase assertEquals(expectedReplacedFiles.get(i)[0], diff.getReplacedFiles().get(i)[0]); assertEquals(expectedReplacedFiles.get(i)[1], diff.getReplacedFiles().get(i)[1]); } - + assertEquals(changedTerms.size(), diff.getChangedTermsAccess().size()); for (int i = 0; i < changedTerms.size(); i++) { String[] diffArray = diff.getChangedTermsAccess().get(i); @@ -256,7 +260,7 @@ private static void originalCalculateDifference(DatasetVersion newVersion, Datas changedFileMetadata = new ArrayList<>(); changedVariableMetadata = new ArrayList<>(); replacedFiles = new ArrayList<>(); - logger.info("Start orig loops: " + System.currentTimeMillis()); + long startTime = System.currentTimeMillis(); // TODO: ? // It looks like we are going through the filemetadatas in both versions, // *sequentially* (i.e. at the cost of O(N*M)), to select the lists of @@ -302,12 +306,13 @@ private static void originalCalculateDifference(DatasetVersion newVersion, Datas } getReplacedFiles(); - logger.info("End org loops: " + System.currentTimeMillis()); + logger.info("Difference Loop Execution time: " + (System.currentTimeMillis() - startTime) + " ms"); } @Deprecated - // This is used only in the original algorithm and was removed from DatasetVersionDifference + // This is used only in the original algorithm and was removed from + // DatasetVersionDifference private static void getReplacedFiles() { if (addedFiles.isEmpty() || removedFiles.isEmpty()) { return; From b27739f236f78e62ab1b4780e2c92e6b18cca6b5 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 29 Aug 2024 17:00:24 -0400 Subject: [PATCH 09/54] release note --- doc/release-notes/10814-Differencing improvement.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/release-notes/10814-Differencing improvement.md diff --git a/doc/release-notes/10814-Differencing improvement.md b/doc/release-notes/10814-Differencing improvement.md new file mode 100644 index 00000000000..49bbdae3e1b --- /dev/null +++ b/doc/release-notes/10814-Differencing improvement.md @@ -0,0 +1,3 @@ +### More Scalable Dataset Version Differencing + +Differencing between dataset versions, which is done during dataset edit operations and to populate the dataset page versions table has been made signficantly more scalable. From e93b60769123e6e27104bbdec5c32e7a93d41033 Mon Sep 17 00:00:00 2001 From: Ludovic DANIEL Date: Mon, 22 Jul 2024 16:09:33 +0200 Subject: [PATCH 10/54] While harvesting OAI DC dc opening tag is not more required but other tags are ignored (cherry picked from commit 8514c7fca20b5ba0d9b889f5f72eb7c93d551075) --- .../api/imports/ImportGenericServiceBean.java | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java index d32a548c8bf..778d5a4167c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java @@ -175,8 +175,6 @@ public DatasetDTO processOAIDCxml(String DcXmlToParse) throws XMLStreamException //while (xmlr.next() == XMLStreamConstants.COMMENT); // skip pre root comments xmlr.nextTag(); - xmlr.require(XMLStreamConstants.START_ELEMENT, null, OAI_DC_OPENING_TAG); - processXMLElement(xmlr, ":", OAI_DC_OPENING_TAG, dublinCoreMapping, datasetDTO); } catch (XMLStreamException ex) { throw new EJBException("ERROR occurred while parsing XML fragment (" + DcXmlToParse.substring(0, 64) + "...); ", ex); @@ -205,10 +203,24 @@ public DatasetDTO processOAIDCxml(String DcXmlToParse) throws XMLStreamException private void processXMLElement(XMLStreamReader xmlr, String currentPath, String openingTag, ForeignMetadataFormatMapping foreignFormatMapping, DatasetDTO datasetDTO) throws XMLStreamException { logger.fine("entering processXMLElement; ("+currentPath+")"); - - for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { + + while (xmlr.hasNext()) { + + int event; + try { + event = xmlr.next(); + } catch (XMLStreamException ex) { + continue; // Skip Undeclared namespace prefix and Unexpected close tag related to com.ctc.wstx.exc.WstxParsingException + } + if (event == XMLStreamConstants.START_ELEMENT) { + String prefix = xmlr.getPrefix(); String currentElement = xmlr.getLocalName(); + + if (prefix != null && !prefix.equals(OAI_DC_OPENING_TAG)) { // Ignore non "dc:" prefix + logger.warning("Element " + prefix + ":" + currentElement + " is ignored"); + continue; + } ForeignMetadataFieldMapping mappingDefined = datasetfieldService.findFieldMapping(foreignFormatMapping.getName(), currentPath+currentElement); From 3925baff88d79a548a60e465fd8f4168ec63e82e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Thu, 12 Sep 2024 17:21:16 +0200 Subject: [PATCH 11/54] Remove unnecessary code --- .../dataverse/api/imports/ImportGenericServiceBean.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java index 778d5a4167c..661d99b3d52 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java @@ -175,6 +175,8 @@ public DatasetDTO processOAIDCxml(String DcXmlToParse) throws XMLStreamException //while (xmlr.next() == XMLStreamConstants.COMMENT); // skip pre root comments xmlr.nextTag(); + xmlr.require(XMLStreamConstants.START_ELEMENT, null, OAI_DC_OPENING_TAG); + processXMLElement(xmlr, ":", OAI_DC_OPENING_TAG, dublinCoreMapping, datasetDTO); } catch (XMLStreamException ex) { throw new EJBException("ERROR occurred while parsing XML fragment (" + DcXmlToParse.substring(0, 64) + "...); ", ex); @@ -210,18 +212,13 @@ private void processXMLElement(XMLStreamReader xmlr, String currentPath, String try { event = xmlr.next(); } catch (XMLStreamException ex) { + logger.warning("Error occurred in the OAI_DC XML parsing : " + ex.getMessage()); continue; // Skip Undeclared namespace prefix and Unexpected close tag related to com.ctc.wstx.exc.WstxParsingException } if (event == XMLStreamConstants.START_ELEMENT) { - String prefix = xmlr.getPrefix(); String currentElement = xmlr.getLocalName(); - if (prefix != null && !prefix.equals(OAI_DC_OPENING_TAG)) { // Ignore non "dc:" prefix - logger.warning("Element " + prefix + ":" + currentElement + " is ignored"); - continue; - } - ForeignMetadataFieldMapping mappingDefined = datasetfieldService.findFieldMapping(foreignFormatMapping.getName(), currentPath+currentElement); if (mappingDefined != null) { From 484e1d272188c8f46125ccf0a4b44cce3662a93d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Thu, 12 Sep 2024 17:21:16 +0200 Subject: [PATCH 12/54] Remove unnecessary code --- .../iq/dataverse/api/imports/ImportGenericServiceBean.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java index 661d99b3d52..41a57665010 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportGenericServiceBean.java @@ -212,13 +212,13 @@ private void processXMLElement(XMLStreamReader xmlr, String currentPath, String try { event = xmlr.next(); } catch (XMLStreamException ex) { - logger.warning("Error occurred in the OAI_DC XML parsing : " + ex.getMessage()); + logger.warning("Error occurred in the XML parsing : " + ex.getMessage()); continue; // Skip Undeclared namespace prefix and Unexpected close tag related to com.ctc.wstx.exc.WstxParsingException } if (event == XMLStreamConstants.START_ELEMENT) { String currentElement = xmlr.getLocalName(); - + ForeignMetadataFieldMapping mappingDefined = datasetfieldService.findFieldMapping(foreignFormatMapping.getName(), currentPath+currentElement); if (mappingDefined != null) { From 34cf77df9babadae038665fe87e6ccd9b6096308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=A9r=C3=B4me=20ROUCOU?= Date: Fri, 13 Sep 2024 14:58:31 +0200 Subject: [PATCH 13/54] release note --- doc/release-notes/10837-exclude-others-ns-harvesting-oai-dc.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/release-notes/10837-exclude-others-ns-harvesting-oai-dc.md diff --git a/doc/release-notes/10837-exclude-others-ns-harvesting-oai-dc.md b/doc/release-notes/10837-exclude-others-ns-harvesting-oai-dc.md new file mode 100644 index 00000000000..c1826bfaed5 --- /dev/null +++ b/doc/release-notes/10837-exclude-others-ns-harvesting-oai-dc.md @@ -0,0 +1,3 @@ +Some repository extend the "oai_dc" metadata prefix with specific namespaces. In this case, harvesting of these datasets is not possible, as an XML parsing error is raised. + +The PR [#10837](https://github.com/IQSS/dataverse/pull/10837) allows the harvesting of these datasets by excluding tags with namespaces that are not "dc:", and harvest only metadata with the "dc" namespace. From c094c8a6e45d02cac5c136e5b13bd5de113465bc Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 1 Nov 2024 16:37:39 -0400 Subject: [PATCH 14/54] github actions from v3 to v4 --- .github/workflows/container_app_pr.yml | 6 +++--- .github/workflows/container_app_push.yml | 2 +- .github/workflows/deploy_beta_testing.yml | 10 +++++----- .github/workflows/guides_build_sphinx.yml | 2 +- .github/workflows/maven_unit_test.yml | 22 +++++++++++----------- .github/workflows/reviewdog_checkstyle.yml | 2 +- .github/workflows/shellcheck.yml | 2 +- .github/workflows/shellspec.yml | 6 +++--- .github/workflows/spi_release.yml | 14 +++++++------- 9 files changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/container_app_pr.yml b/.github/workflows/container_app_pr.yml index c86d284e74b..a3468cbc220 100644 --- a/.github/workflows/container_app_pr.yml +++ b/.github/workflows/container_app_pr.yml @@ -20,14 +20,14 @@ jobs: if: ${{ github.repository_owner == 'IQSS' }} steps: # Checkout the pull request code as when merged - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge' - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4 with: java-version: "17" distribution: 'adopt' - - uses: actions/cache@v3 + - uses: actions/cache@v4 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 3b7ce066d73..184b69583a5 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -68,7 +68,7 @@ jobs: if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' && github.repository_owner == 'IQSS' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: peter-evans/dockerhub-description@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} diff --git a/.github/workflows/deploy_beta_testing.yml b/.github/workflows/deploy_beta_testing.yml index 028f0140cc9..87eb6e8c150 100644 --- a/.github/workflows/deploy_beta_testing.yml +++ b/.github/workflows/deploy_beta_testing.yml @@ -10,9 +10,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4.1.7 - - uses: actions/setup-java@v3 + - uses: actions/setup-java@v4.1.7 with: distribution: 'zulu' java-version: '17' @@ -32,7 +32,7 @@ jobs: run: echo "war_file=$(ls *.war | head -1)">> $GITHUB_ENV - name: Upload war artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4.1.7 with: name: built-app path: ./target/${{ env.war_file }} @@ -42,10 +42,10 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4.1.7 - name: Download war artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.7 with: name: built-app path: ./ diff --git a/.github/workflows/guides_build_sphinx.yml b/.github/workflows/guides_build_sphinx.yml index 86b59b11d35..fa3a876c418 100644 --- a/.github/workflows/guides_build_sphinx.yml +++ b/.github/workflows/guides_build_sphinx.yml @@ -10,7 +10,7 @@ jobs: docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - uses: uncch-rdmc/sphinx-action@master with: docs-folder: "doc/sphinx-guides/" diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index a94b17a67ba..5c7a154e2f5 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -32,9 +32,9 @@ jobs: steps: # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - - uses: actions/checkout@v3 + - uses: actions/checkout@v4.1.7 - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4.1.7 with: java-version: ${{ matrix.jdk }} distribution: temurin @@ -57,7 +57,7 @@ jobs: # Upload the built war file. For download, it will be wrapped in a ZIP by GitHub. # See also https://github.com/actions/upload-artifact#zipped-artifact-downloads - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4.1.7 with: name: dataverse-java${{ matrix.jdk }}.war path: target/dataverse*.war @@ -67,7 +67,7 @@ jobs: - run: | tar -cvf java-builddir.tar target tar -cvf java-m2-selection.tar ~/.m2/repository/io/gdcc/dataverse-* - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4.1.7 with: name: java-artifacts path: | @@ -98,16 +98,16 @@ jobs: steps: # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - - uses: actions/checkout@v3 + - uses: actions/checkout@v4.1.7 - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v4.1.7 with: java-version: ${{ matrix.jdk }} distribution: temurin cache: maven # Get the build output from the unit test job - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4.1.7 with: name: java-artifacts - run: | @@ -119,7 +119,7 @@ jobs: # Wrap up and send to coverage job - run: tar -cvf java-reportdir.tar target/site - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4.1.7 with: name: java-reportdir path: java-reportdir.tar @@ -132,15 +132,15 @@ jobs: steps: # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/checkout@v4.1.7 + - uses: actions/setup-java@v4.1.7 with: java-version: '17' distribution: temurin cache: maven # Get the build output from the integration test job - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4.1.7 with: name: java-reportdir - run: tar -xvf java-reportdir.tar diff --git a/.github/workflows/reviewdog_checkstyle.yml b/.github/workflows/reviewdog_checkstyle.yml index 90a0dd7d06b..804b04f696a 100644 --- a/.github/workflows/reviewdog_checkstyle.yml +++ b/.github/workflows/reviewdog_checkstyle.yml @@ -10,7 +10,7 @@ jobs: name: Checkstyle job steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Run check style uses: nikitasavinov/checkstyle-action@master with: diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index 56f7d648dc4..fb9cf5a0a1f 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -21,7 +21,7 @@ jobs: permissions: pull-requests: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: shellcheck uses: reviewdog/action-shellcheck@v1 with: diff --git a/.github/workflows/shellspec.yml b/.github/workflows/shellspec.yml index 3320d9d08a4..cc09992edac 100644 --- a/.github/workflows/shellspec.yml +++ b/.github/workflows/shellspec.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Install shellspec run: curl -fsSL https://git.io/shellspec | sh -s ${{ env.SHELLSPEC_VERSION }} --yes - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Run Shellspec run: | cd tests/shell @@ -30,7 +30,7 @@ jobs: container: image: rockylinux/rockylinux:9 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Install shellspec run: | curl -fsSL https://github.com/shellspec/shellspec/releases/download/${{ env.SHELLSPEC_VERSION }}/shellspec-dist.tar.gz | tar -xz -C /usr/share @@ -47,7 +47,7 @@ jobs: steps: - name: Install shellspec run: curl -fsSL https://git.io/shellspec | sh -s 0.28.1 --yes - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Run Shellspec run: | cd tests/shell diff --git a/.github/workflows/spi_release.yml b/.github/workflows/spi_release.yml index 8ad74b3e4bb..6398edca412 100644 --- a/.github/workflows/spi_release.yml +++ b/.github/workflows/spi_release.yml @@ -37,15 +37,15 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'pull_request' && needs.check-secrets.outputs.available == 'true' steps: - - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 with: java-version: '17' distribution: 'adopt' server-id: ossrh server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} @@ -63,12 +63,12 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'push' && needs.check-secrets.outputs.available == 'true' steps: - - uses: actions/checkout@v3 - - uses: actions/setup-java@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 with: java-version: '17' distribution: 'adopt' - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} @@ -76,7 +76,7 @@ jobs: # Running setup-java again overwrites the settings.xml - IT'S MANDATORY TO DO THIS SECOND SETUP!!! - name: Set up Maven Central Repository - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: java-version: '17' distribution: 'adopt' From 12c1597c5f788f6d23ee1035d9f996af4749a003 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 1 Nov 2024 16:45:26 -0400 Subject: [PATCH 15/54] Modify test file to force workflow to build and test DO NOT MERGEgit add .git add . --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 93f1024ae7a..db1e6eb5169 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -70,7 +70,8 @@ public class DatasetsIT { @BeforeAll public static void setUpClass() { - + // !!!!!!!! DO NOT CHECK THIS FILE IN + logger.warning(">>>>>>>>>>> Just a line change to force GitHub actions to build and test Dataverse !!!!!!!"); RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); From 570d1285ba5f4a39596cec88295d8a45acc2e08c Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 1 Nov 2024 16:49:32 -0400 Subject: [PATCH 16/54] Modify test file to force workflow to build and test DO NOT MERGE --- .github/workflows/deploy_beta_testing.yml | 6 +++--- .github/workflows/maven_unit_test.yml | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/deploy_beta_testing.yml b/.github/workflows/deploy_beta_testing.yml index 87eb6e8c150..c6a8bd909df 100644 --- a/.github/workflows/deploy_beta_testing.yml +++ b/.github/workflows/deploy_beta_testing.yml @@ -10,9 +10,9 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4 - - uses: actions/setup-java@v4.1.7 + - uses: actions/setup-java@v4 with: distribution: 'zulu' java-version: '17' @@ -42,7 +42,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4 - name: Download war artifact uses: actions/download-artifact@v4.1.7 diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index 5c7a154e2f5..67f20bbdc93 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -32,9 +32,9 @@ jobs: steps: # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v4.1.7 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.jdk }} distribution: temurin @@ -98,9 +98,9 @@ jobs: steps: # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - - uses: actions/checkout@v4.1.7 + - uses: actions/checkout@v4 - name: Set up JDK ${{ matrix.jdk }} - uses: actions/setup-java@v4.1.7 + uses: actions/setup-java@v4 with: java-version: ${{ matrix.jdk }} distribution: temurin @@ -132,8 +132,8 @@ jobs: steps: # TODO: As part of #10618 change to setup-maven custom action # Basic setup chores - - uses: actions/checkout@v4.1.7 - - uses: actions/setup-java@v4.1.7 + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 with: java-version: '17' distribution: temurin From 534ecdc0ab8834973d20c061fb84dc3a4e098de3 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 1 Nov 2024 16:55:09 -0400 Subject: [PATCH 17/54] Modify test file to force workflow to build and test DO NOT MERGE --- .github/workflows/deploy_beta_testing.yml | 2 +- .github/workflows/maven_unit_test.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy_beta_testing.yml b/.github/workflows/deploy_beta_testing.yml index c6a8bd909df..c36a4dfab43 100644 --- a/.github/workflows/deploy_beta_testing.yml +++ b/.github/workflows/deploy_beta_testing.yml @@ -32,7 +32,7 @@ jobs: run: echo "war_file=$(ls *.war | head -1)">> $GITHUB_ENV - name: Upload war artifact - uses: actions/upload-artifact@v4.1.7 + uses: actions/upload-artifact@v4 with: name: built-app path: ./target/${{ env.war_file }} diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index 67f20bbdc93..18a741e5ca5 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -57,7 +57,7 @@ jobs: # Upload the built war file. For download, it will be wrapped in a ZIP by GitHub. # See also https://github.com/actions/upload-artifact#zipped-artifact-downloads - - uses: actions/upload-artifact@v4.1.7 + - uses: actions/upload-artifact@v4 with: name: dataverse-java${{ matrix.jdk }}.war path: target/dataverse*.war @@ -67,7 +67,7 @@ jobs: - run: | tar -cvf java-builddir.tar target tar -cvf java-m2-selection.tar ~/.m2/repository/io/gdcc/dataverse-* - - uses: actions/upload-artifact@v4.1.7 + - uses: actions/upload-artifact@v4 with: name: java-artifacts path: | @@ -119,7 +119,7 @@ jobs: # Wrap up and send to coverage job - run: tar -cvf java-reportdir.tar target/site - - uses: actions/upload-artifact@v4.1.7 + - uses: actions/upload-artifact@v4 with: name: java-reportdir path: java-reportdir.tar From 5f19030e77861bf0ca92f9dc9ed8f91760ac21b8 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 1 Nov 2024 17:00:13 -0400 Subject: [PATCH 18/54] Modify test file to force workflow to build and test DO NOT MERGE --- conf/solr/update-fields.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/solr/update-fields.sh b/conf/solr/update-fields.sh index 386c1ee4e87..5c896a98a04 100755 --- a/conf/solr/update-fields.sh +++ b/conf/solr/update-fields.sh @@ -22,6 +22,7 @@ COPY_FIELDS="" TRIGGER_CHAIN=0 ED_DELETE_FIELDS="'a+,'b-d" ED_DELETE_COPYFIELDS="'a+,'b-d" +FAKEDONOTMERGE=0 SOLR_SCHEMA_FIELD_BEGIN_MARK="SCHEMA-FIELDS::BEGIN" SOLR_SCHEMA_FIELD_END_MARK="SCHEMA-FIELDS::END" From caa8bdef67fb137a593b0ce0a490c5077b811b60 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 1 Nov 2024 17:03:03 -0400 Subject: [PATCH 19/54] Modify test file to force workflow to build and test DO NOT MERGE --- .github/workflows/shellcheck.yml | 2 +- .github/workflows/shellspec.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index fb9cf5a0a1f..d83c0a5315b 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -21,7 +21,7 @@ jobs: permissions: pull-requests: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v2 - name: shellcheck uses: reviewdog/action-shellcheck@v1 with: diff --git a/.github/workflows/shellspec.yml b/.github/workflows/shellspec.yml index cc09992edac..3320d9d08a4 100644 --- a/.github/workflows/shellspec.yml +++ b/.github/workflows/shellspec.yml @@ -19,7 +19,7 @@ jobs: steps: - name: Install shellspec run: curl -fsSL https://git.io/shellspec | sh -s ${{ env.SHELLSPEC_VERSION }} --yes - - uses: actions/checkout@v4 + - uses: actions/checkout@v2 - name: Run Shellspec run: | cd tests/shell @@ -30,7 +30,7 @@ jobs: container: image: rockylinux/rockylinux:9 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v2 - name: Install shellspec run: | curl -fsSL https://github.com/shellspec/shellspec/releases/download/${{ env.SHELLSPEC_VERSION }}/shellspec-dist.tar.gz | tar -xz -C /usr/share @@ -47,7 +47,7 @@ jobs: steps: - name: Install shellspec run: curl -fsSL https://git.io/shellspec | sh -s 0.28.1 --yes - - uses: actions/checkout@v4 + - uses: actions/checkout@v2 - name: Run Shellspec run: | cd tests/shell From 25d0caca81999859ad4cad65f71113dbfdbb9951 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 1 Nov 2024 17:03:40 -0400 Subject: [PATCH 20/54] Modify test file to force workflow to build and test DO NOT MERGE --- .github/workflows/shellcheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/shellcheck.yml b/.github/workflows/shellcheck.yml index d83c0a5315b..56f7d648dc4 100644 --- a/.github/workflows/shellcheck.yml +++ b/.github/workflows/shellcheck.yml @@ -21,7 +21,7 @@ jobs: permissions: pull-requests: write steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: shellcheck uses: reviewdog/action-shellcheck@v1 with: From 74fe482e7dc6d4663e08fa9756b7e76ed58c93a4 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 1 Nov 2024 17:05:53 -0400 Subject: [PATCH 21/54] Modify test file to force workflow to build and test DO NOT MERGE --- .github/workflows/spi_release.yml | 14 +++++++------- conf/solr/update-fields.sh | 1 - 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/spi_release.yml b/.github/workflows/spi_release.yml index 6398edca412..8ad74b3e4bb 100644 --- a/.github/workflows/spi_release.yml +++ b/.github/workflows/spi_release.yml @@ -37,15 +37,15 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'pull_request' && needs.check-secrets.outputs.available == 'true' steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 with: java-version: '17' distribution: 'adopt' server-id: ossrh server-username: MAVEN_USERNAME server-password: MAVEN_PASSWORD - - uses: actions/cache@v4 + - uses: actions/cache@v2 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} @@ -63,12 +63,12 @@ jobs: runs-on: ubuntu-latest if: github.event_name == 'push' && needs.check-secrets.outputs.available == 'true' steps: - - uses: actions/checkout@v4 - - uses: actions/setup-java@v4 + - uses: actions/checkout@v3 + - uses: actions/setup-java@v3 with: java-version: '17' distribution: 'adopt' - - uses: actions/cache@v4 + - uses: actions/cache@v2 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} @@ -76,7 +76,7 @@ jobs: # Running setup-java again overwrites the settings.xml - IT'S MANDATORY TO DO THIS SECOND SETUP!!! - name: Set up Maven Central Repository - uses: actions/setup-java@v4 + uses: actions/setup-java@v3 with: java-version: '17' distribution: 'adopt' diff --git a/conf/solr/update-fields.sh b/conf/solr/update-fields.sh index 5c896a98a04..386c1ee4e87 100755 --- a/conf/solr/update-fields.sh +++ b/conf/solr/update-fields.sh @@ -22,7 +22,6 @@ COPY_FIELDS="" TRIGGER_CHAIN=0 ED_DELETE_FIELDS="'a+,'b-d" ED_DELETE_COPYFIELDS="'a+,'b-d" -FAKEDONOTMERGE=0 SOLR_SCHEMA_FIELD_BEGIN_MARK="SCHEMA-FIELDS::BEGIN" SOLR_SCHEMA_FIELD_END_MARK="SCHEMA-FIELDS::END" From ccee96af5021e31ff7eefaeeda655b93f3c26bab Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 1 Nov 2024 17:07:14 -0400 Subject: [PATCH 22/54] Modify test file to force workflow to build and test DO NOT MERGE --- .github/workflows/guides_build_sphinx.yml | 2 +- .github/workflows/reviewdog_checkstyle.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/guides_build_sphinx.yml b/.github/workflows/guides_build_sphinx.yml index fa3a876c418..86b59b11d35 100644 --- a/.github/workflows/guides_build_sphinx.yml +++ b/.github/workflows/guides_build_sphinx.yml @@ -10,7 +10,7 @@ jobs: docs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v2 - uses: uncch-rdmc/sphinx-action@master with: docs-folder: "doc/sphinx-guides/" diff --git a/.github/workflows/reviewdog_checkstyle.yml b/.github/workflows/reviewdog_checkstyle.yml index 804b04f696a..90a0dd7d06b 100644 --- a/.github/workflows/reviewdog_checkstyle.yml +++ b/.github/workflows/reviewdog_checkstyle.yml @@ -10,7 +10,7 @@ jobs: name: Checkstyle job steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v2 - name: Run check style uses: nikitasavinov/checkstyle-action@master with: From 9263b8a8aea13ad420a5b7a7db360e48ab1561c0 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 1 Nov 2024 17:08:45 -0400 Subject: [PATCH 23/54] Modify test file to force workflow to build and test DO NOT MERGE --- .github/workflows/container_app_pr.yml | 6 +++--- .github/workflows/container_app_push.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/container_app_pr.yml b/.github/workflows/container_app_pr.yml index a3468cbc220..c86d284e74b 100644 --- a/.github/workflows/container_app_pr.yml +++ b/.github/workflows/container_app_pr.yml @@ -20,14 +20,14 @@ jobs: if: ${{ github.repository_owner == 'IQSS' }} steps: # Checkout the pull request code as when merged - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 with: ref: 'refs/pull/${{ github.event.client_payload.pull_request.number }}/merge' - - uses: actions/setup-java@v4 + - uses: actions/setup-java@v3 with: java-version: "17" distribution: 'adopt' - - uses: actions/cache@v4 + - uses: actions/cache@v3 with: path: ~/.m2 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} diff --git a/.github/workflows/container_app_push.yml b/.github/workflows/container_app_push.yml index 184b69583a5..3b7ce066d73 100644 --- a/.github/workflows/container_app_push.yml +++ b/.github/workflows/container_app_push.yml @@ -68,7 +68,7 @@ jobs: if: ${{ github.event_name != 'pull_request' && github.ref_name == 'develop' && github.repository_owner == 'IQSS' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v3 - uses: peter-evans/dockerhub-description@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} From d719dde6b3a7274621bcafc2257607e012acfb6c Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 1 Nov 2024 17:20:05 -0400 Subject: [PATCH 24/54] remove test file --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index db1e6eb5169..ce123b60b13 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -70,8 +70,6 @@ public class DatasetsIT { @BeforeAll public static void setUpClass() { - // !!!!!!!! DO NOT CHECK THIS FILE IN - logger.warning(">>>>>>>>>>> Just a line change to force GitHub actions to build and test Dataverse !!!!!!!"); RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); From 37f44b52a5435141c4dd43b8155222f0746802d9 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 1 Nov 2024 17:21:42 -0400 Subject: [PATCH 25/54] remove test file --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index ce123b60b13..c7184f0c78b 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -70,7 +70,8 @@ public class DatasetsIT { @BeforeAll public static void setUpClass() { - + + RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); Response removeIdentifierGenerationStyle = UtilIT.deleteSetting(SettingsServiceBean.Key.IdentifierGenerationStyle); From a094d81f36c12cdd65f0f48656ed7948d3245b0d Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 1 Nov 2024 17:27:01 -0400 Subject: [PATCH 26/54] remove test file --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index c7184f0c78b..93f1024ae7a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -70,8 +70,8 @@ public class DatasetsIT { @BeforeAll public static void setUpClass() { - - + + RestAssured.baseURI = UtilIT.getRestAssuredBaseUri(); Response removeIdentifierGenerationStyle = UtilIT.deleteSetting(SettingsServiceBean.Key.IdentifierGenerationStyle); From 79c8cf38888563c8a7de40c99316a3ed8acbd192 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 5 Nov 2024 17:26:17 -0500 Subject: [PATCH 27/54] docs "Update a Dataverse Collection" vs. "Change Collection Attributes" #10904 --- doc/sphinx-guides/source/api/native-api.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 6254742eebb..b464b6df393 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -126,6 +126,8 @@ Same as in :ref:`create-dataverse-api`, the request JSON supports an optional `` To obtain an example of how these objects are included in the JSON file, download :download:`dataverse-complete-optional-params.json <../_static/api/dataverse-complete-optional-params.json>` file and modify it to suit your needs. +See also :ref:`collection-attributes-api`. + .. _view-dataverse: View a Dataverse Collection @@ -1058,6 +1060,8 @@ The following attributes are supported: * ``affiliation`` Affiliation * ``filePIDsEnabled`` ("true" or "false") Restricted to use by superusers and only when the :ref:`:AllowEnablingFilePIDsPerCollection <:AllowEnablingFilePIDsPerCollection>` setting is true. Enables or disables registration of file-level PIDs in datasets within the collection (overriding the instance-wide setting). +See also :ref:`update-dataverse-api`. + .. _collection-storage-quotas: Update Collection Input Levels From 9367eb1e8be85f91e0e0b015a084084671cb4b0e Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 12 Nov 2024 14:17:59 -0500 Subject: [PATCH 28/54] add comment to yml files as to reason for v4.1.7 --- .github/workflows/deploy_beta_testing.yml | 1 + .github/workflows/maven_unit_test.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/deploy_beta_testing.yml b/.github/workflows/deploy_beta_testing.yml index c36a4dfab43..2c684aa8ad1 100644 --- a/.github/workflows/deploy_beta_testing.yml +++ b/.github/workflows/deploy_beta_testing.yml @@ -45,6 +45,7 @@ jobs: - uses: actions/checkout@v4 - name: Download war artifact + # using v4.1.7 due to a bug in v4 uses: actions/download-artifact@v4.1.7 with: name: built-app diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index 18a741e5ca5..2ce872cd55c 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -107,6 +107,7 @@ jobs: cache: maven # Get the build output from the unit test job + # using v4.1.7 due to a bug in v4 - uses: actions/download-artifact@v4.1.7 with: name: java-artifacts @@ -140,6 +141,7 @@ jobs: cache: maven # Get the build output from the integration test job + # using v4.1.7 due to a bug in v4 - uses: actions/download-artifact@v4.1.7 with: name: java-reportdir From c9b685aa942c156670aaa78355bf57b45accd20d Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 13 Nov 2024 11:40:22 -0500 Subject: [PATCH 29/54] Update flyway script --- ..._10017-failure-with-long-custom-question.sql => V6.4.0.2.sql} | 1 + 1 file changed, 1 insertion(+) rename src/main/resources/db/migration/{V6.1.0.6__10017-failure-with-long-custom-question.sql => V6.4.0.2.sql} (90%) diff --git a/src/main/resources/db/migration/V6.1.0.6__10017-failure-with-long-custom-question.sql b/src/main/resources/db/migration/V6.4.0.2.sql similarity index 90% rename from src/main/resources/db/migration/V6.1.0.6__10017-failure-with-long-custom-question.sql rename to src/main/resources/db/migration/V6.4.0.2.sql index 9a3002378b3..414e98975db 100644 --- a/src/main/resources/db/migration/V6.1.0.6__10017-failure-with-long-custom-question.sql +++ b/src/main/resources/db/migration/V6.4.0.2.sql @@ -1 +1,2 @@ +#10118 ALTER TABLE customquestion ALTER COLUMN questionstring TYPE text; From 89892a4862b780d47a90699c729c798726a2c507 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 13 Nov 2024 12:52:45 -0500 Subject: [PATCH 30/54] getget typo --- .../edu/harvard/iq/dataverse/DatasetVersionDifference.java | 2 +- .../harvard/iq/dataverse/DatasetVersionDifferenceTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java index 19c3b02f4ee..27868e3c7ed 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetVersionDifference.java @@ -1616,7 +1616,7 @@ private static boolean fieldsAreDifferent(DatasetField originalField, DatasetFie return false; } - List getgetChangedVariableMetadata() { + List getChangedVariableMetadata() { return changedVariableMetadata; } diff --git a/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java b/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java index 44ac267abaf..8508c9ac34e 100644 --- a/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/DatasetVersionDifferenceTest.java @@ -221,7 +221,7 @@ private void compareResults(DatasetVersion datasetVersion, DatasetVersion datase assertEquals(addedFiles, diff.getAddedFiles()); assertEquals(removedFiles, diff.getRemovedFiles()); assertEquals(changedFileMetadata, diff.getChangedFileMetadata()); - assertEquals(changedVariableMetadata, diff.getgetChangedVariableMetadata()); + assertEquals(changedVariableMetadata, diff.getChangedVariableMetadata()); assertEquals(replacedFiles.size(), diff.getReplacedFiles().size()); for (int i = 0; i < replacedFiles.size(); i++) { assertEquals(replacedFiles.get(i)[0], diff.getReplacedFiles().get(i)[0]); @@ -233,7 +233,7 @@ private void compareResults(DatasetVersion datasetVersion, DatasetVersion datase assertEquals(expectedAddedFiles, diff.getAddedFiles()); assertEquals(expectedRemovedFiles, diff.getRemovedFiles()); assertEquals(expectedChangedFileMetadata, diff.getChangedFileMetadata()); - assertEquals(expectedChangedVariableMetadata, diff.getgetChangedVariableMetadata()); + assertEquals(expectedChangedVariableMetadata, diff.getChangedVariableMetadata()); assertEquals(expectedReplacedFiles.size(), diff.getReplacedFiles().size()); for (int i = 0; i < expectedReplacedFiles.size(); i++) { assertEquals(expectedReplacedFiles.get(i)[0], diff.getReplacedFiles().get(i)[0]); From a663ad443e44ae2b01c8fde66f4234763e73c3c1 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 13 Nov 2024 15:29:08 -0500 Subject: [PATCH 31/54] fix comment --- src/main/resources/db/migration/V6.4.0.2.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/resources/db/migration/V6.4.0.2.sql b/src/main/resources/db/migration/V6.4.0.2.sql index 414e98975db..bc4a85b278f 100644 --- a/src/main/resources/db/migration/V6.4.0.2.sql +++ b/src/main/resources/db/migration/V6.4.0.2.sql @@ -1,2 +1,2 @@ -#10118 +-- #10118 ALTER TABLE customquestion ALTER COLUMN questionstring TYPE text; From c645af208d3df7e805b7ec08545320c7fdf21bbb Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 15 Nov 2024 09:49:59 -0500 Subject: [PATCH 32/54] validate guestbook email field --- src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java index 1ea7d02791d..1eea1f3ac7d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java +++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java @@ -15,6 +15,8 @@ import java.util.ArrayList; import java.util.Date; import java.util.List; + +import edu.harvard.iq.dataverse.validation.ValidateEmail; import jakarta.persistence.*; import jakarta.validation.constraints.Size; import java.util.Collections; @@ -80,8 +82,8 @@ public class GuestbookResponse implements Serializable { @Size(max = 255, message = "{guestbook.response.nameLength}") private String name; - // TODO: Consider using EMailValidator as well. @Size(max = 255, message = "{guestbook.response.nameLength}") + @ValidateEmail(message = "{user.invalidEmail}") private String email; @Size(max = 255, message = "{guestbook.response.nameLength}") From d324db8dae6a4476cd1e707c9c47200c8522451c Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 15 Nov 2024 10:02:16 -0500 Subject: [PATCH 33/54] add release note --- doc/release-notes/10661-guestbook-email-bug-fix.md | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 doc/release-notes/10661-guestbook-email-bug-fix.md diff --git a/doc/release-notes/10661-guestbook-email-bug-fix.md b/doc/release-notes/10661-guestbook-email-bug-fix.md new file mode 100644 index 00000000000..7b317c4dd61 --- /dev/null +++ b/doc/release-notes/10661-guestbook-email-bug-fix.md @@ -0,0 +1,4 @@ + +### Guestbook Email Validation Bug fixe + +Guestbook UI Form: Email address is now checked for valid email From 99ea2499bdbf7a6dbdd3ac8c2059abf3f1a49535 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 15 Nov 2024 10:02:43 -0500 Subject: [PATCH 34/54] add release note --- doc/release-notes/10661-guestbook-email-bug-fix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/10661-guestbook-email-bug-fix.md b/doc/release-notes/10661-guestbook-email-bug-fix.md index 7b317c4dd61..887b3ff86c7 100644 --- a/doc/release-notes/10661-guestbook-email-bug-fix.md +++ b/doc/release-notes/10661-guestbook-email-bug-fix.md @@ -1,4 +1,4 @@ -### Guestbook Email Validation Bug fixe +### Guestbook Email Validation Bug fix Guestbook UI Form: Email address is now checked for valid email From b2d57f53b28c516a1017ccc83626ec2ae4746723 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 15 Nov 2024 10:16:52 -0500 Subject: [PATCH 35/54] add release note --- doc/release-notes/10661-guestbook-email-bug-fix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/10661-guestbook-email-bug-fix.md b/doc/release-notes/10661-guestbook-email-bug-fix.md index 887b3ff86c7..05e70c9762a 100644 --- a/doc/release-notes/10661-guestbook-email-bug-fix.md +++ b/doc/release-notes/10661-guestbook-email-bug-fix.md @@ -1,4 +1,4 @@ ### Guestbook Email Validation Bug fix -Guestbook UI Form: Email address is now checked for valid email +Guestbook UI Form: Email address is now checked for valid email format From a6d835c32b09593555614e79a9610bc1acec15c9 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Fri, 15 Nov 2024 14:20:03 -0500 Subject: [PATCH 36/54] fix failing email validation for api calls --- src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java index 1eea1f3ac7d..830c7740e34 100644 --- a/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java +++ b/src/main/java/edu/harvard/iq/dataverse/GuestbookResponse.java @@ -200,7 +200,8 @@ public String getEmail() { } public void setEmail(String email) { - this.email = email; + // ValidateEmail requires NULL or valid email. Empty String will fail validation + this.email = (email == null || email.trim().isEmpty()) ? null : email; } public Guestbook getGuestbook() { From 5ef2d15986cbc862a52c7a3464d6b56af4a0804c Mon Sep 17 00:00:00 2001 From: Guido Schmutz Date: Fri, 15 Nov 2024 20:44:12 +0100 Subject: [PATCH 37/54] add previewers-provider and register-previewers container --- docker/compose/demo/compose.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index a0a85ef5217..0da3bcaae6e 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -74,6 +74,34 @@ services: volumes: - ./data/app/data:/dv + dataverse-previewers-provider: + image: trivadis/dataverse-previewers-provider:latest + container_name: dataverse-previewers-provider + hostname: dataverse-previewers-provider + ports: + - 9000:80 + environment: + - NGINX_HTTP_PORT=80 + - PREVIEWERS_PROVIDER_URL=http://${PUBLIC_IP:-localhost}:9000 + - VERSIONS="v1.4,betatest" + restart: unless-stopped + + dataverse-register-previewers: + container_name: dataverse-register-previewers + hostname: dataverse-register-previewers + image: trivadis/dataverse-deploy-previewers:latest + environment: + - API_TOKEN=${API_TOKEN} + - DATAVERSE_URL=http://${PUBLIC_IP:-localhost}:8080 + - TIMEOUT=10m + - PREVIEWER_PROVIDER_URL=http://${PUBLIC_IP:-localhost}:9000 + - INCLUDE_PREVIEWERS=text,html,pdf,csv,comma-separated-values,tsv,tab-separated-values,jpeg,png,gif,markdown,x-markdown + - EXCLUDE_PREVIEWERS= + - REMOVE_EXISTING=True + command: + - deploy + restart: no + postgres: container_name: "postgres" hostname: postgres From 5b4d931c111e203b379841eea8ab8dcd5ac7d890 Mon Sep 17 00:00:00 2001 From: Guido Schmutz Date: Fri, 15 Nov 2024 20:53:25 +0100 Subject: [PATCH 38/54] Update compose.yml --- docker/compose/demo/compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index 0da3bcaae6e..ba413ff1d98 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -94,7 +94,7 @@ services: - API_TOKEN=${API_TOKEN} - DATAVERSE_URL=http://${PUBLIC_IP:-localhost}:8080 - TIMEOUT=10m - - PREVIEWER_PROVIDER_URL=http://${PUBLIC_IP:-localhost}:9000 + - PREVIEWERS_PROVIDER_URL=http://${PUBLIC_IP:-localhost}:9000 - INCLUDE_PREVIEWERS=text,html,pdf,csv,comma-separated-values,tsv,tab-separated-values,jpeg,png,gif,markdown,x-markdown - EXCLUDE_PREVIEWERS= - REMOVE_EXISTING=True From 22184aedbf940fff2b7e8e03490016a3dc2153a6 Mon Sep 17 00:00:00 2001 From: Guido Schmutz Date: Fri, 15 Nov 2024 20:57:12 +0100 Subject: [PATCH 39/54] Update compose.yml --- docker/compose/demo/compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index ba413ff1d98..979ad0aba2a 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -82,7 +82,7 @@ services: - 9000:80 environment: - NGINX_HTTP_PORT=80 - - PREVIEWERS_PROVIDER_URL=http://${PUBLIC_IP:-localhost}:9000 + - PREVIEWERS_PROVIDER_URL=http://${IP:-localhost}:9000 - VERSIONS="v1.4,betatest" restart: unless-stopped @@ -92,9 +92,9 @@ services: image: trivadis/dataverse-deploy-previewers:latest environment: - API_TOKEN=${API_TOKEN} - - DATAVERSE_URL=http://${PUBLIC_IP:-localhost}:8080 + - DATAVERSE_URL=http://${IP:-localhost}:8080 - TIMEOUT=10m - - PREVIEWERS_PROVIDER_URL=http://${PUBLIC_IP:-localhost}:9000 + - PREVIEWERS_PROVIDER_URL=http://${IP:-localhost}:9000 - INCLUDE_PREVIEWERS=text,html,pdf,csv,comma-separated-values,tsv,tab-separated-values,jpeg,png,gif,markdown,x-markdown - EXCLUDE_PREVIEWERS= - REMOVE_EXISTING=True From 799ba6c57bbe21823cca75d40ea4363aa2fa82b5 Mon Sep 17 00:00:00 2001 From: Guido Schmutz Date: Fri, 15 Nov 2024 20:59:25 +0100 Subject: [PATCH 40/54] Update compose.yml --- docker/compose/demo/compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index 979ad0aba2a..ba413ff1d98 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -82,7 +82,7 @@ services: - 9000:80 environment: - NGINX_HTTP_PORT=80 - - PREVIEWERS_PROVIDER_URL=http://${IP:-localhost}:9000 + - PREVIEWERS_PROVIDER_URL=http://${PUBLIC_IP:-localhost}:9000 - VERSIONS="v1.4,betatest" restart: unless-stopped @@ -92,9 +92,9 @@ services: image: trivadis/dataverse-deploy-previewers:latest environment: - API_TOKEN=${API_TOKEN} - - DATAVERSE_URL=http://${IP:-localhost}:8080 + - DATAVERSE_URL=http://${PUBLIC_IP:-localhost}:8080 - TIMEOUT=10m - - PREVIEWERS_PROVIDER_URL=http://${IP:-localhost}:9000 + - PREVIEWERS_PROVIDER_URL=http://${PUBLIC_IP:-localhost}:9000 - INCLUDE_PREVIEWERS=text,html,pdf,csv,comma-separated-values,tsv,tab-separated-values,jpeg,png,gif,markdown,x-markdown - EXCLUDE_PREVIEWERS= - REMOVE_EXISTING=True From b14a5eb469efbc48d98ce0a183a65ef8decf90b0 Mon Sep 17 00:00:00 2001 From: Guido Schmutz Date: Fri, 15 Nov 2024 21:01:18 +0100 Subject: [PATCH 41/54] Update compose.yml --- docker/compose/demo/compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index ba413ff1d98..19bd23f5bfb 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -82,7 +82,7 @@ services: - 9000:80 environment: - NGINX_HTTP_PORT=80 - - PREVIEWERS_PROVIDER_URL=http://${PUBLIC_IP:-localhost}:9000 + - PREVIEWERS_PROVIDER_URL=http://${IP:-localhost}:9000 - VERSIONS="v1.4,betatest" restart: unless-stopped @@ -92,9 +92,9 @@ services: image: trivadis/dataverse-deploy-previewers:latest environment: - API_TOKEN=${API_TOKEN} - - DATAVERSE_URL=http://${PUBLIC_IP:-localhost}:8080 + - DATAVERSE_URL=http://dataverse:8080 - TIMEOUT=10m - - PREVIEWERS_PROVIDER_URL=http://${PUBLIC_IP:-localhost}:9000 + - PREVIEWERS_PROVIDER_URL=http://${IP:-localhost}:9000 - INCLUDE_PREVIEWERS=text,html,pdf,csv,comma-separated-values,tsv,tab-separated-values,jpeg,png,gif,markdown,x-markdown - EXCLUDE_PREVIEWERS= - REMOVE_EXISTING=True From 0fdfd80fe6513f76c981cbcd6a04101efc7226c8 Mon Sep 17 00:00:00 2001 From: Guido Schmutz Date: Fri, 15 Nov 2024 21:11:45 +0100 Subject: [PATCH 42/54] Update compose.yml --- docker/compose/demo/compose.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index 19bd23f5bfb..1e89eb8d021 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -80,6 +80,8 @@ services: hostname: dataverse-previewers-provider ports: - 9000:80 + networks: + - dataverse environment: - NGINX_HTTP_PORT=80 - PREVIEWERS_PROVIDER_URL=http://${IP:-localhost}:9000 @@ -90,8 +92,9 @@ services: container_name: dataverse-register-previewers hostname: dataverse-register-previewers image: trivadis/dataverse-deploy-previewers:latest + networks: + - dataverse environment: - - API_TOKEN=${API_TOKEN} - DATAVERSE_URL=http://dataverse:8080 - TIMEOUT=10m - PREVIEWERS_PROVIDER_URL=http://${IP:-localhost}:9000 From 8401c779a51f36ee33f8c7acd1aaf68c6d388b61 Mon Sep 17 00:00:00 2001 From: Guido Schmutz Date: Fri, 15 Nov 2024 21:31:29 +0100 Subject: [PATCH 43/54] Update compose.yml --- docker/compose/demo/compose.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index 1e89eb8d021..e2565545b0a 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -74,30 +74,29 @@ services: volumes: - ./data/app/data:/dv - dataverse-previewers-provider: + previewers-provider: image: trivadis/dataverse-previewers-provider:latest - container_name: dataverse-previewers-provider - hostname: dataverse-previewers-provider + container_name: previewers-provider + hostname: previewers-provider ports: - 9000:80 networks: - dataverse environment: - NGINX_HTTP_PORT=80 - - PREVIEWERS_PROVIDER_URL=http://${IP:-localhost}:9000 + - PREVIEWERS_PROVIDER_URL=http://${MACHINE_IP:-localhost}:9000 - VERSIONS="v1.4,betatest" - restart: unless-stopped - dataverse-register-previewers: - container_name: dataverse-register-previewers - hostname: dataverse-register-previewers + register-previewers: + container_name: register-previewers + hostname: register-previewers image: trivadis/dataverse-deploy-previewers:latest networks: - dataverse environment: - DATAVERSE_URL=http://dataverse:8080 - TIMEOUT=10m - - PREVIEWERS_PROVIDER_URL=http://${IP:-localhost}:9000 + - PREVIEWERS_PROVIDER_URL=http://${MACHINE_IP:-localhost}:9000 - INCLUDE_PREVIEWERS=text,html,pdf,csv,comma-separated-values,tsv,tab-separated-values,jpeg,png,gif,markdown,x-markdown - EXCLUDE_PREVIEWERS= - REMOVE_EXISTING=True From f54e51af476c895c8425907f3eaf96055b3aff5f Mon Sep 17 00:00:00 2001 From: Guido Schmutz Date: Fri, 15 Nov 2024 21:45:56 +0100 Subject: [PATCH 44/54] Update compose.yml --- docker/compose/demo/compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index e2565545b0a..c940fde1786 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -79,11 +79,11 @@ services: container_name: previewers-provider hostname: previewers-provider ports: - - 9000:80 + - 9000:9000 networks: - dataverse environment: - - NGINX_HTTP_PORT=80 + - NGINX_HTTP_PORT=9000 - PREVIEWERS_PROVIDER_URL=http://${MACHINE_IP:-localhost}:9000 - VERSIONS="v1.4,betatest" From d84ac89d166052a774e6a697b4d978aa86810a65 Mon Sep 17 00:00:00 2001 From: Guido Schmutz Date: Fri, 15 Nov 2024 22:21:31 +0100 Subject: [PATCH 45/54] Update compose.yml --- docker/compose/demo/compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index c940fde1786..f03d81f5957 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -9,7 +9,7 @@ services: restart: on-failure user: payara environment: - DATAVERSE_SITEURL: "https://demo.example.org" + DATAVERSE_SITEURL: "http://${MACHINE_IP:-localhost}:8080" DATAVERSE_DB_HOST: postgres DATAVERSE_DB_PASSWORD: secret DATAVERSE_DB_USER: dataverse From 9a580f8d683b60da0f4dad1f4f157db926620594 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 18 Nov 2024 14:29:31 -0500 Subject: [PATCH 46/54] add wait to async export call --- .../edu/harvard/iq/dataverse/api/DatasetsIT.java | 4 ++-- .../java/edu/harvard/iq/dataverse/api/UtilIT.java | 14 +++++++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 93f1024ae7a..10584f2df71 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -4242,7 +4242,7 @@ public void testCitationDate() throws IOException { .statusCode(OK.getStatusCode()) .body("data.message", is(expectedCitation)); - Response exportDatasetAsDublinCore = UtilIT.exportDataset(datasetPid, "oai_dc", apiToken); + Response exportDatasetAsDublinCore = UtilIT.exportDataset(datasetPid, "oai_dc", apiToken, true); exportDatasetAsDublinCore.prettyPrint(); exportDatasetAsDublinCore.then().assertThat() .body("oai_dc.type", equalTo("Dataset")) @@ -4259,7 +4259,7 @@ public void testCitationDate() throws IOException { rexport.then().assertThat().statusCode(OK.getStatusCode()); String todayDate = LocalDate.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd")); - Response exportPostClear = UtilIT.exportDataset(datasetPid, "oai_dc", apiToken); + Response exportPostClear = UtilIT.exportDataset(datasetPid, "oai_dc", apiToken, true); exportPostClear.prettyPrint(); exportPostClear.then().assertThat() .body("oai_dc.type", equalTo("Dataset")) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 502f1ecb0a8..56a680966ba 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2161,20 +2161,28 @@ static Response uploadProvFreeForm(String idOrPersistentId, JsonObject jsonObjec // return requestSpecification.delete("/api/files/" + idInPath + "/prov-freeform" + optionalQueryParam); // } static Response exportDataset(String datasetPersistentId, String exporter) { - return exportDataset(datasetPersistentId, exporter, null); + return exportDataset(datasetPersistentId, exporter, null, false); } - static Response exportDataset(String datasetPersistentId, String exporter, String apiToken) { + return exportDataset(datasetPersistentId, exporter, apiToken, false); + } + static Response exportDataset(String datasetPersistentId, String exporter, String apiToken, boolean wait) { // http://localhost:8080/api/datasets/export?exporter=dataverse_json&persistentId=doi%3A10.5072/FK2/W6WIMQ RequestSpecification requestSpecification = given(); if (apiToken != null) { requestSpecification = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken); } - return requestSpecification + Response resp = requestSpecification // .header(API_TOKEN_HTTP_HEADER, apiToken) // .get("/api/datasets/:persistentId/export" + "?persistentId=" + datasetPersistentId + "&exporter=" + exporter); .get("/api/datasets/export" + "?persistentId=" + datasetPersistentId + "&exporter=" + exporter); + // Wait for the Async call to finish to get the updated data + if (wait) { + sleepForReexport(datasetPersistentId, apiToken, 10); + resp = requestSpecification.get("/api/datasets/export" + "?persistentId=" + datasetPersistentId + "&exporter=" + exporter); + } + return resp; } static Response reexportDatasetAllFormats(String idOrPersistentId) { From a04cdabe224843cadabe4cbdd62ac4c4ec46cc42 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Mon, 18 Nov 2024 14:59:11 -0500 Subject: [PATCH 47/54] add wait to async export call --- .../java/edu/harvard/iq/dataverse/api/UtilIT.java | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 56a680966ba..232e8132c5c 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2167,22 +2167,17 @@ static Response exportDataset(String datasetPersistentId, String exporter, Strin return exportDataset(datasetPersistentId, exporter, apiToken, false); } static Response exportDataset(String datasetPersistentId, String exporter, String apiToken, boolean wait) { -// http://localhost:8080/api/datasets/export?exporter=dataverse_json&persistentId=doi%3A10.5072/FK2/W6WIMQ + // Wait for the Async call to finish to get the updated data + if (wait) { + sleepForReexport(datasetPersistentId, apiToken, 10); + } RequestSpecification requestSpecification = given(); if (apiToken != null) { requestSpecification = given() .header(UtilIT.API_TOKEN_HTTP_HEADER, apiToken); } - Response resp = requestSpecification - // .header(API_TOKEN_HTTP_HEADER, apiToken) - // .get("/api/datasets/:persistentId/export" + "?persistentId=" + datasetPersistentId + "&exporter=" + exporter); + return requestSpecification .get("/api/datasets/export" + "?persistentId=" + datasetPersistentId + "&exporter=" + exporter); - // Wait for the Async call to finish to get the updated data - if (wait) { - sleepForReexport(datasetPersistentId, apiToken, 10); - resp = requestSpecification.get("/api/datasets/export" + "?persistentId=" + datasetPersistentId + "&exporter=" + exporter); - } - return resp; } static Response reexportDatasetAllFormats(String idOrPersistentId) { From 54677ce2a5411e46aafeb8d81a7daac5753f5e13 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 19 Nov 2024 08:49:54 -0500 Subject: [PATCH 48/54] file detection using first few bytes disabled on direct upload --- doc/sphinx-guides/source/api/native-api.rst | 1 + doc/sphinx-guides/source/developers/big-data-support.rst | 1 + 2 files changed, 2 insertions(+) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index b464b6df393..4d6d6c28a49 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3617,6 +3617,7 @@ The fully expanded example above (without environment variables) looks like this Currently the following methods are used to detect file types: - The file type detected by the browser (or sent via API). +- Custom code that reads the first few bytes. As explained at :ref:`s3-direct-upload-features-disabled`, this code is disabled during direct upload to S3. However, this code is active when the "redetect" API is used. - JHOVE: https://jhove.openpreservation.org - The file extension (e.g. ".ipybn") is used, defined in a file called ``MimeTypeDetectionByFileExtension.properties``. - The file name (e.g. "Dockerfile") is used, defined in a file called ``MimeTypeDetectionByFileName.properties``. diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 759dd40413b..dad0af5ae87 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -44,6 +44,7 @@ Features that are Disabled if S3 Direct Upload is Enabled The following features are disabled when S3 direct upload is enabled. - Unzipping of zip files. (See :ref:`compressed-files`.) +- Detection of file type based on custom code that reads the first few bytes. (See :ref:`redetect-file-type`.) - Extraction of metadata from FITS files. (See :ref:`fits`.) - Creation of NcML auxiliary files (See :ref:`netcdf-and-hdf5`.) - Extraction of a geospatial bounding box from NetCDF and HDF5 files (see :ref:`netcdf-and-hdf5`) unless :ref:`dataverse.netcdf.geo-extract-s3-direct-upload` is set to true. From a4261014177e2306c7000b24d934dd140ebae2d1 Mon Sep 17 00:00:00 2001 From: Steven Winship <39765413+stevenwinship@users.noreply.github.com> Date: Tue, 19 Nov 2024 09:55:33 -0500 Subject: [PATCH 49/54] revert v4.1.7 to v4 (latest) --- .github/workflows/deploy_beta_testing.yml | 3 +-- .github/workflows/maven_unit_test.yml | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy_beta_testing.yml b/.github/workflows/deploy_beta_testing.yml index 2c684aa8ad1..4cec08564a4 100644 --- a/.github/workflows/deploy_beta_testing.yml +++ b/.github/workflows/deploy_beta_testing.yml @@ -45,8 +45,7 @@ jobs: - uses: actions/checkout@v4 - name: Download war artifact - # using v4.1.7 due to a bug in v4 - uses: actions/download-artifact@v4.1.7 + uses: actions/download-artifact@v4 with: name: built-app path: ./ diff --git a/.github/workflows/maven_unit_test.yml b/.github/workflows/maven_unit_test.yml index 2ce872cd55c..45180ea7aec 100644 --- a/.github/workflows/maven_unit_test.yml +++ b/.github/workflows/maven_unit_test.yml @@ -107,8 +107,7 @@ jobs: cache: maven # Get the build output from the unit test job - # using v4.1.7 due to a bug in v4 - - uses: actions/download-artifact@v4.1.7 + - uses: actions/download-artifact@v4 with: name: java-artifacts - run: | @@ -141,8 +140,7 @@ jobs: cache: maven # Get the build output from the integration test job - # using v4.1.7 due to a bug in v4 - - uses: actions/download-artifact@v4.1.7 + - uses: actions/download-artifact@v4 with: name: java-reportdir - run: tar -xvf java-reportdir.tar From 89b8a1ccb46d553d0b31af55e6fb3e03c58cbabb Mon Sep 17 00:00:00 2001 From: landreev Date: Tue, 19 Nov 2024 10:02:27 -0500 Subject: [PATCH 50/54] Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 4d6d6c28a49..929dac592e0 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3617,7 +3617,7 @@ The fully expanded example above (without environment variables) looks like this Currently the following methods are used to detect file types: - The file type detected by the browser (or sent via API). -- Custom code that reads the first few bytes. As explained at :ref:`s3-direct-upload-features-disabled`, this code is disabled during direct upload to S3. However, this code is active when the "redetect" API is used. +- Custom code that reads the first few bytes. As explained at :ref:`s3-direct-upload-features-disabled`, this method of file type detection is not utilized during direct upload to S3, since by nature of direct upload Dataverse never sees the contents of the file. However, this code is utilized when the "redetect" API is used. - JHOVE: https://jhove.openpreservation.org - The file extension (e.g. ".ipybn") is used, defined in a file called ``MimeTypeDetectionByFileExtension.properties``. - The file name (e.g. "Dockerfile") is used, defined in a file called ``MimeTypeDetectionByFileName.properties``. From be0979d8047d5e1aaa2191830b332233a42d91fd Mon Sep 17 00:00:00 2001 From: landreev Date: Tue, 19 Nov 2024 10:02:33 -0500 Subject: [PATCH 51/54] Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 929dac592e0..442e4862dab 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3618,7 +3618,7 @@ Currently the following methods are used to detect file types: - The file type detected by the browser (or sent via API). - Custom code that reads the first few bytes. As explained at :ref:`s3-direct-upload-features-disabled`, this method of file type detection is not utilized during direct upload to S3, since by nature of direct upload Dataverse never sees the contents of the file. However, this code is utilized when the "redetect" API is used. -- JHOVE: https://jhove.openpreservation.org +- JHOVE: https://jhove.openpreservation.org . Note that the same applies about direct upload to S3 and the "redirect" API. - The file extension (e.g. ".ipybn") is used, defined in a file called ``MimeTypeDetectionByFileExtension.properties``. - The file name (e.g. "Dockerfile") is used, defined in a file called ``MimeTypeDetectionByFileName.properties``. From 813063e5454623c0cc7b5a32d76b57346ca01d5d Mon Sep 17 00:00:00 2001 From: landreev Date: Tue, 19 Nov 2024 10:02:45 -0500 Subject: [PATCH 52/54] Update doc/sphinx-guides/source/developers/big-data-support.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/developers/big-data-support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index dad0af5ae87..f3d98fae0bf 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -44,7 +44,7 @@ Features that are Disabled if S3 Direct Upload is Enabled The following features are disabled when S3 direct upload is enabled. - Unzipping of zip files. (See :ref:`compressed-files`.) -- Detection of file type based on custom code that reads the first few bytes. (See :ref:`redetect-file-type`.) +- Detection of file type based on JHOVE and custom code that reads the first few bytes. (See :ref:`redetect-file-type`.) - Extraction of metadata from FITS files. (See :ref:`fits`.) - Creation of NcML auxiliary files (See :ref:`netcdf-and-hdf5`.) - Extraction of a geospatial bounding box from NetCDF and HDF5 files (see :ref:`netcdf-and-hdf5`) unless :ref:`dataverse.netcdf.geo-extract-s3-direct-upload` is set to true. From e5f841597dcd4be4150c969da40d70777a3ed773 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 19 Nov 2024 10:04:48 -0500 Subject: [PATCH 53/54] Update doc/sphinx-guides/source/api/native-api.rst --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 442e4862dab..691aa94c834 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -3618,7 +3618,7 @@ Currently the following methods are used to detect file types: - The file type detected by the browser (or sent via API). - Custom code that reads the first few bytes. As explained at :ref:`s3-direct-upload-features-disabled`, this method of file type detection is not utilized during direct upload to S3, since by nature of direct upload Dataverse never sees the contents of the file. However, this code is utilized when the "redetect" API is used. -- JHOVE: https://jhove.openpreservation.org . Note that the same applies about direct upload to S3 and the "redirect" API. +- JHOVE: https://jhove.openpreservation.org . Note that the same applies about direct upload to S3 and the "redetect" API. - The file extension (e.g. ".ipybn") is used, defined in a file called ``MimeTypeDetectionByFileExtension.properties``. - The file name (e.g. "Dockerfile") is used, defined in a file called ``MimeTypeDetectionByFileName.properties``. From dcb2f7daa2ee05cd88ce01fd7bd0a7de9daba234 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 19 Nov 2024 10:55:06 -0500 Subject: [PATCH 54/54] we no longer show the shibboleth.te file on the Shib page #10193 --- doc/sphinx-guides/source/developers/selinux.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/selinux.rst b/doc/sphinx-guides/source/developers/selinux.rst index ca41ab82d25..1d3d01610fe 100644 --- a/doc/sphinx-guides/source/developers/selinux.rst +++ b/doc/sphinx-guides/source/developers/selinux.rst @@ -8,7 +8,7 @@ SELinux Introduction ------------ -The ``shibboleth.te`` file below that is mentioned in the :doc:`/installation/shibboleth` section of the Installation Guide was created on CentOS 6 as part of https://github.com/IQSS/dataverse/issues/3406 but may need to be revised for future versions of RHEL/CentOS (pull requests welcome!). The file is versioned with the docs and can be found in the following location: +The ``shibboleth.te`` file below that was mentioned in the :doc:`/installation/shibboleth` section of the Installation Guide was created on CentOS 6 as part of https://github.com/IQSS/dataverse/issues/3406 but may need to be revised for future versions of RHEL/CentOS (pull requests welcome!). The file is versioned with the docs and can be found in the following location: ``doc/sphinx-guides/source/_static/installation/files/etc/selinux/targeted/src/policy/domains/misc/shibboleth.te``