Merge tag 'v5.12' into overwrite_mimetype

.github/workflows/guides_build_sphinx.yml

@@ -2,7 +2,9 @@ name: "Guides Build Status"
 on: 
   pull_request:
     paths:
-      - 'doc/sphinx-guides/**/*.rst'
+      - "doc/sphinx-guides/**/*.rst"
+      - "doc/sphinx-guides/**/requirements.txt"
+      - "doc/sphinx-guides/**/conf.py"
 
 jobs:
   docs:

.github/workflows/maven_unit_test.yml

@@ -4,22 +4,34 @@ on:
     push:
         paths:
             - "**.java"
+            - "pom.xml"
+            - "modules/**/pom.xml"
     pull_request:
         paths:
             - "**.java"
+            - "pom.xml"
+            - "modules/**/pom.xml"
 
 jobs:
     unittest:
-        name: (JDK ${{ matrix.jdk }} / ${{ matrix.os }}) Unit Tests
+        name: (${{ matrix.status}} / JDK ${{ matrix.jdk }}) Unit Tests
         strategy:
             fail-fast: false
             matrix:
-                os: [ ubuntu-latest ]
                 jdk: [ '11' ]
+                experimental: [false]
+                status:  ["Stable"]
+                #
+                # JDK 17 builds disabled due to non-essential fails marking CI jobs as completely failed within
+                # Github Projects, PR lists etc. This was consensus on Slack #dv-tech. See issue #8094
+                # (This is a limitation of how Github is currently handling these things.)
+                #
                 #include:
-                #    -   os: ubuntu-latest
-                #        jdk: '16'
-        runs-on: ${{ matrix.os }}
+                #    - jdk: '17'
+                #      experimental: true
+                #      status: "Experimental"
+        continue-on-error: ${{ matrix.experimental }}
+        runs-on: ubuntu-latest
         steps:
           - uses: actions/checkout@v2
           - name: Set up JDK ${{ matrix.jdk }}
@@ -34,7 +46,7 @@ jobs:
                 key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
                 restore-keys: ${{ runner.os }}-m2
           - name: Build with Maven
-            run: mvn -DcompilerArgument=-Xlint:unchecked -P all-unit-tests clean test
+            run: mvn -DcompilerArgument=-Xlint:unchecked -Dtarget.java.version=${{ matrix.jdk }} -P all-unit-tests clean test
           - name: Maven Code Coverage
             env:
                 CI_NAME: github

.gitignore

@@ -70,3 +70,6 @@ venv
 scripts/search/data/binary/trees.png.thumb140
 src/main/webapp/resources/images/cc0.png.thumb140
 src/main/webapp/resources/images/dataverseproject.png.thumb140
+
+# apache-maven is downloaded by docker-aio
+apache-maven*

README.md

@@ -1,4 +1,4 @@
-Dataverse® 
+Dataverse®  
 ===============
 
 Dataverse is an [open source][] software platform for sharing, finding, citing, and preserving research data (developed by the [Data Science and Products team](http://www.iq.harvard.edu/people/people/data-science-products) at the [Institute for Quantitative Social Science](http://iq.harvard.edu/) and the [Dataverse community][]).

checkstyle.xml

@@ -98,7 +98,7 @@
         </module>
         -->
         <module name="IllegalImport">
-            <property name="illegalPkgs" value="org.apache.commons.lang"/>
+            <property name="illegalPkgs" value="org.apache.commons.lang, org.apache.log4j"/>
         </module>
         <!-- <module name="RedundantImport"/> -->
         <!-- <module name="UnusedImports">

conf/docker-aio/0prep_deps.sh

@@ -4,16 +4,16 @@ if [ ! -d dv/deps ]; then
 fi
 wdir=`pwd`
 
-if [ ! -e dv/deps/payara-5.2021.5.zip ]; then
+if [ ! -e dv/deps/payara-5.2022.3.zip ]; then
 	echo "payara dependency prep"
-	wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.5/payara-5.2021.5.zip  -O dv/deps/payara-5.2021.5.zip
+	wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2022.3/payara-5.2022.3.zip  -O dv/deps/payara-5.2022.3.zip
 fi
 
-if [ ! -e dv/deps/solr-8.8.1dv.tgz ]; then
+if [ ! -e dv/deps/solr-8.11.1dv.tgz ]; then
 	echo "solr dependency prep"
 	# schema changes *should* be the only ones...
 	cd dv/deps/	
-	wget https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz -O solr-8.8.1dv.tgz
+	wget https://archive.apache.org/dist/lucene/solr/8.11.1/solr-8.11.1.tgz -O solr-8.11.1dv.tgz
 	cd ../../
 fi
 

conf/docker-aio/1prep.sh

@@ -4,18 +4,18 @@
 # this was based off the phoenix deployment; and is likely uglier and bulkier than necessary in a perfect world
 
 mkdir -p testdata/doc/sphinx-guides/source/_static/util/
-cp ../solr/8.8.1/schema*.xml testdata/
-cp ../solr/8.8.1/solrconfig.xml testdata/
+cp ../solr/8.11.1/schema*.xml testdata/
+cp ../solr/8.11.1/solrconfig.xml testdata/
 cp ../jhove/jhove.conf testdata/
 cp ../jhove/jhoveConfig.xsd testdata/
 cd ../../
 cp -r scripts conf/docker-aio/testdata/
 cp doc/sphinx-guides/source/_static/util/createsequence.sql conf/docker-aio/testdata/doc/sphinx-guides/source/_static/util/
 
-wget -q https://downloads.apache.org/maven/maven-3/3.8.4/binaries/apache-maven-3.8.4-bin.tar.gz
-tar xfz apache-maven-3.8.4-bin.tar.gz
+wget -q https://downloads.apache.org/maven/maven-3/3.8.5/binaries/apache-maven-3.8.5-bin.tar.gz
+tar xfz apache-maven-3.8.5-bin.tar.gz
 mkdir maven
-mv apache-maven-3.8.4/* maven/
+mv apache-maven-3.8.5/* maven/
 echo "export JAVA_HOME=/usr/lib/jvm/jre-openjdk" > maven/maven.sh
 echo "export M2_HOME=../maven" >> maven/maven.sh
 echo "export MAVEN_HOME=../maven" >> maven/maven.sh

conf/docker-aio/c8.dockerfile

@@ -23,8 +23,8 @@ COPY testdata/sushi_sample_logs.json /tmp/
 COPY disableipv6.conf /etc/sysctl.d/
 RUN rm /etc/httpd/conf/*
 COPY httpd.conf /etc/httpd/conf 
-RUN cd /opt ; tar zxf /tmp/dv/deps/solr-8.8.1dv.tgz 
-RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2021.5.zip ; ln -s /opt/payara5 /opt/glassfish4
+RUN cd /opt ; tar zxf /tmp/dv/deps/solr-8.11.1dv.tgz
+RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2022.3.zip ; ln -s /opt/payara5 /opt/glassfish4
 
 # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm`
 # this dies under Java 11, do we keep it?
@@ -34,9 +34,9 @@ RUN sudo -u postgres /usr/pgsql-13/bin/initdb -D /var/lib/pgsql/13/data -E 'UTF-
 
 # copy configuration related files
 RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/13/data/
-RUN cp -r /opt/solr-8.8.1/server/solr/configsets/_default /opt/solr-8.8.1/server/solr/collection1
-RUN cp /tmp/dv/schema*.xml /opt/solr-8.8.1/server/solr/collection1/conf/
-RUN cp /tmp/dv/solrconfig.xml /opt/solr-8.8.1/server/solr/collection1/conf/solrconfig.xml
+RUN cp -r /opt/solr-8.11.1/server/solr/configsets/_default /opt/solr-8.11.1/server/solr/collection1
+RUN cp /tmp/dv/schema*.xml /opt/solr-8.11.1/server/solr/collection1/conf/
+RUN cp /tmp/dv/solrconfig.xml /opt/solr-8.11.1/server/solr/collection1/conf/solrconfig.xml
 
 # skipping payara user and solr user (run both as root)
 

conf/docker-aio/dv/install/default.config

@@ -9,7 +9,6 @@ POSTGRES_DATABASE	dvndb
 POSTGRES_USER	dvnapp
 POSTGRES_PASSWORD	secret
 SOLR_LOCATION	idx
-TWORAVENS_LOCATION	NOT INSTALLED
 RSERVE_HOST	localhost
 RSERVE_PORT	6311
 RSERVE_USER	rserve

conf/docker-aio/entrypoint.bash

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 export LANG=en_US.UTF-8
 sudo -u postgres /usr/pgsql-13/bin/pg_ctl start -D /var/lib/pgsql/13/data &
-cd /opt/solr-8.8.1/
+cd /opt/solr-8.11.1/
 # TODO: Run Solr as non-root and remove "-force".
 bin/solr start -force
 bin/solr create_core -c collection1 -d server/solr/collection1/conf -force

conf/docker-aio/httpd.conf

@@ -17,9 +17,6 @@ Group apache
  CustomLog logs/access_log combined
 
   # proxy config (aka - what to send to glassfish or not)
-  ProxyPassMatch ^/RApacheInfo$ !
-  ProxyPassMatch ^/custom !
-  ProxyPassMatch ^/dataexplore !
   ProxyPassMatch ^/Shibboleth.sso !
   ProxyPassMatch ^/shibboleth-ds !
   # pass everything else to Glassfish

conf/docker-aio/testdata/httpd.conf

@@ -17,9 +17,6 @@ Group apache
  CustomLog logs/access_log combined
 
   # proxy config (aka - what to send to glassfish or not)
-  ProxyPassMatch ^/RApacheInfo$ !
-  ProxyPassMatch ^/custom !
-  ProxyPassMatch ^/dataexplore !
   ProxyPassMatch ^/Shibboleth.sso !
   ProxyPassMatch ^/shibboleth-ds !
   # pass everything else to Glassfish

conf/httpd/conf.d/dataverse.conf

@@ -1,8 +1,3 @@
-# don't pass paths used by rApache and TwoRavens to Glassfish
-ProxyPassMatch ^/RApacheInfo$ !
-ProxyPassMatch ^/dataexplore !
-ProxyPassMatch ^/custom !
-ProxyPassMatch ^/rookzelig !
 # don't pass paths used by Shibboleth to Glassfish
 ProxyPassMatch ^/Shibboleth.sso !
 ProxyPassMatch ^/shibboleth-ds !

conf/solr/8.8.1/schema.xml → conf/solr/8.11.1/schema.xml

@@ -259,8 +259,11 @@
     <field name="characteristicOfSources" type="text_en" multiValued="false" stored="true" indexed="true"/>
     <field name="city" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="cleaningOperations" type="text_en" multiValued="false" stored="true" indexed="true"/>
-    <field name="collectionMode" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="collectionMode" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="collectorTraining" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="workflowType" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="workflowCodeRepository" type="text_en" multiValued="true" stored="true" indexed="true"/>
+    <field name="workflowDocumentation" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="contributor" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="contributorName" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="contributorType" type="text_en" multiValued="true" stored="true" indexed="true"/>
@@ -315,7 +318,7 @@
     <field name="grantNumber" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="grantNumberAgency" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="grantNumberValue" type="text_en" multiValued="true" stored="true" indexed="true"/>
-    <field name="journalArticleType" type="text_en" multiValued="false" stored="true" indexed="true"/>
+    <field name="journalArticleType" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="journalIssue" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="journalPubDate" type="text_en" multiValued="true" stored="true" indexed="true"/>
     <field name="journalVolume" type="text_en" multiValued="true" stored="true" indexed="true"/>
@@ -498,6 +501,9 @@
     <copyField source="cleaningOperations" dest="_text_" maxChars="3000"/>
     <copyField source="collectionMode" dest="_text_" maxChars="3000"/>
     <copyField source="collectorTraining" dest="_text_" maxChars="3000"/>
+    <copyField source="workflowType" dest="_text_" maxChars="3000"/>
+    <copyField source="workflowCodeRepository" dest="_text_" maxChars="3000"/>
+    <copyField source="workflowDocumentation" dest="_text_" maxChars="3000"/>
     <copyField source="contributor" dest="_text_" maxChars="3000"/>
     <copyField source="contributorName" dest="_text_" maxChars="3000"/>
     <copyField source="contributorType" dest="_text_" maxChars="3000"/>

conf/solr/8.8.1/solrconfig.xml → conf/solr/8.11.1/solrconfig.xml

@@ -408,7 +408,7 @@
                       to occupy. Note that when this option is specified, the size
                       and initialSize parameters are ignored.
       -->
-    <filterCache class="solr.FastLRUCache"
+    <filterCache class="solr.search.CaffeineCache"
                  size="512"
                  initialSize="512"
                  autowarmCount="0"/>
@@ -421,7 +421,7 @@
             maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
                        to occupy
       -->
-    <queryResultCache class="solr.LRUCache"
+    <queryResultCache class="solr.search.CaffeineCache"
                       size="512"
                       initialSize="512"
                       autowarmCount="0"/>
@@ -432,14 +432,14 @@
          document).  Since Lucene internal document ids are transient,
          this cache will not be autowarmed.
       -->
-    <documentCache class="solr.LRUCache"
+    <documentCache class="solr.search.CaffeineCache"
                    size="512"
                    initialSize="512"
                    autowarmCount="0"/>
 
     <!-- custom cache currently used by block join -->
     <cache name="perSegFilter"
-           class="solr.search.LRUCache"
+           class="solr.search.CaffeineCache"
            size="10"
            initialSize="0"
            autowarmCount="10"

conf/vagrant/etc/yum.repos.d/shibboleth.repo

@@ -1,7 +1,9 @@
-[security_shibboleth]
-name=Shibboleth (CentOS_8)
+[shibboleth]
+name=Shibboleth (rockylinux8)
+# Please report any problems to https://shibboleth.atlassian.net/jira
 type=rpm-md
-baseurl=http://download.opensuse.org/repositories/security:/shibboleth/CentOS_8/
+mirrorlist=https://shibboleth.net/cgi-bin/mirrorlist.cgi/rockylinux8
 gpgcheck=1
-gpgkey=http://download.opensuse.org/repositories/security:/shibboleth/CentOS_8/repodata/repomd.xml.key
+gpgkey=https://shibboleth.net/downloads/service-provider/RPMS/repomd.xml.key
+        https://shibboleth.net/downloads/service-provider/RPMS/cantor.repomd.xml.key
 enabled=1

doc/Architecture/components.uml

@@ -40,14 +40,11 @@ node "DatabaseServer2" {
 }
 
 node "RserveServer1" {
-  component "rApache" {
-  }
   database "Rserve" {
   }
 } 
 
 Clients --> LoadBalancer
-Clients --> rApache
 
 LoadBalancer --> Apache1
 LoadBalancer --> Apache2