From 6cca160c9d91679bb85ae597f40e66746723b906 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Sun, 21 Jul 2024 13:16:04 -0400 Subject: [PATCH 1/5] solr 9.6 and other lib updates --- conf/solr/solrconfig.xml | 2 +- modules/dataverse-parent/pom.xml | 2 +- pom.xml | 26 ++++++++++++-------------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/conf/solr/solrconfig.xml b/conf/solr/solrconfig.xml index 34386375fe1..2115cb1f822 100644 --- a/conf/solr/solrconfig.xml +++ b/conf/solr/solrconfig.xml @@ -35,7 +35,7 @@ that you fully re-index after changing this setting as it can affect both how text is indexed and queried. --> - 9.7 + 9.10 6.2024.6 42.7.2 - 9.4.1 + 9.6.1 1.12.748 26.30.0 diff --git a/pom.xml b/pom.xml index 76a8f61444f..9f0eaaa11ce 100644 --- a/pom.xml +++ b/pom.xml @@ -29,8 +29,8 @@ 1.2.18.4 9.22.1 1.20.1 - 5.2.1 - 2.4.1 + 5.2.5 + 2.9.2 5.5.3 Dataverse API @@ -136,12 +136,12 @@ com.apicatalog titanium-json-ld - 1.3.2 + 1.4.0 com.google.code.gson gson - 2.8.9 + 2.9.1 compile @@ -157,11 +157,9 @@ provided - - org.everit.json - org.everit.json.schema - 1.5.1 + com.github.erosb + everit-json-schema + 1.14.1 org.mindrot @@ -319,7 +317,7 @@ org.apache.solr solr-solrj - 9.4.1 + 9.6.1 colt @@ -390,7 +388,7 @@ com.github.jai-imageio jai-imageio-core - 1.3.1 + 1.4.0 org.ocpsoft.rewrite @@ -474,7 +472,7 @@ com.google.auto.service auto-service - 1.0-rc2 + 1.1.1 true jar @@ -553,7 +551,7 @@ org.xmlunit xmlunit-core - 2.9.1 + 2.10.0 com.google.cloud @@ -615,7 +613,7 @@ org.xmlunit xmlunit-assertj3 - 2.8.2 + 2.10.0 test From 2046950e4d93ce988aaee2e1e178a9d2ba5b5c13 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 10 Oct 2024 17:04:58 -0400 Subject: [PATCH 2/5] more solr version changes --- .../10713-Solr9.6.1 and lib updates.md | 10 ++++++++++ .../_static/installation/files/etc/init.d/solr | 2 +- .../installation/files/etc/systemd/solr.service | 6 +++--- .../source/developers/classic-dev-env.rst | 2 +- .../source/installation/prerequisites.rst | 16 ++++++++-------- docker/compose/demo/compose.yml | 2 +- 6 files changed, 24 insertions(+), 14 deletions(-) create mode 100644 doc/release-notes/10713-Solr9.6.1 and lib updates.md diff --git a/doc/release-notes/10713-Solr9.6.1 and lib updates.md b/doc/release-notes/10713-Solr9.6.1 and lib updates.md new file mode 100644 index 00000000000..84807f6c176 --- /dev/null +++ b/doc/release-notes/10713-Solr9.6.1 and lib updates.md @@ -0,0 +1,10 @@ +Solr 9.6.1 is now the version recommended in our installation guides and used with automated testing. Other libraries Dataverse uses have been updated as well. + +For the upgrade instructions section: + +[note that 6.5 may contain other solr-related changes, so the instructions may need to contain information merged from multiple release notes!] + +If you are upgrading Solr: + - Install solr-9.6.1 following the instructions from the Installation guide. + - Run a full reindex to populate the search catalog. + - Note that it may be possible to skip the reindexing step by simply moving the existing `.../server/solr/collection1/` under the new `solr-9.6.1` installation directory. This however has not been thoroughly tested and is not officially supported. \ No newline at end of file diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr index 14df734cca7..4326ca70aaf 100755 --- a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr +++ b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr @@ -5,7 +5,7 @@ # chkconfig: 35 92 08 # description: Starts and stops Apache Solr -SOLR_DIR="/usr/local/solr/solr-9.4.1" +SOLR_DIR="/usr/local/solr/solr-9.6.1" SOLR_COMMAND="bin/solr" SOLR_ARGS="-m 1g" SOLR_USER=solr diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service index 8ccf7652a49..8aee5db3d4d 100644 --- a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service +++ b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service @@ -5,9 +5,9 @@ After = syslog.target network.target remote-fs.target nss-lookup.target [Service] User = solr Type = forking -WorkingDirectory = /usr/local/solr/solr-9.4.1 -ExecStart = /usr/local/solr/solr-9.4.1/bin/solr start -m 1g -ExecStop = /usr/local/solr/solr-9.4.1/bin/solr stop +WorkingDirectory = /usr/local/solr/solr-9.6.1 +ExecStart = /usr/local/solr/solr-9.6.1/bin/solr start -m 1g +ExecStop = /usr/local/solr/solr-9.6.1/bin/solr stop LimitNOFILE=65000 LimitNPROC=65000 Restart=on-failure diff --git a/doc/sphinx-guides/source/developers/classic-dev-env.rst b/doc/sphinx-guides/source/developers/classic-dev-env.rst index d305019004e..3806a791fff 100755 --- a/doc/sphinx-guides/source/developers/classic-dev-env.rst +++ b/doc/sphinx-guides/source/developers/classic-dev-env.rst @@ -136,7 +136,7 @@ On Linux, you should just install PostgreSQL using your favorite package manager Install Solr ^^^^^^^^^^^^ -`Solr `_ 9.4.1 is required. +`Solr `_ 9.6.1 is required. Follow the instructions in the "Installing Solr" section of :doc:`/installation/prerequisites` in the main Installation guide. diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index f61321ef245..48d0158e740 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -163,7 +163,7 @@ The Dataverse software search index is powered by Solr. Supported Versions ================== -The Dataverse software has been tested with Solr version 9.4.1. Future releases in the 9.x series are likely to be compatible. Please get in touch (:ref:`support`) if you are having trouble with a newer version. +The Dataverse software has been tested with Solr version 9.6.1. Future releases in the 9.x series are likely to be compatible. Please get in touch (:ref:`support`) if you are having trouble with a newer version. Installing Solr =============== @@ -178,19 +178,19 @@ Become the ``solr`` user and then download and configure Solr:: su - solr cd /usr/local/solr - wget https://archive.apache.org/dist/solr/solr/9.4.1/solr-9.4.1.tgz - tar xvzf solr-9.4.1.tgz - cd solr-9.4.1 + wget https://archive.apache.org/dist/solr/solr/9.6.1/solr-9.6.1.tgz + tar xvzf solr-9.6.1.tgz + cd solr-9.6.1 cp -r server/solr/configsets/_default server/solr/collection1 You should already have a "dvinstall.zip" file that you downloaded from https://github.com/IQSS/dataverse/releases . Unzip it into ``/tmp``. Then copy the files into place:: - cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf - cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-9.4.1/server/solr/collection1/conf + cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-9.6.1/server/solr/collection1/conf + cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-9.6.1/server/solr/collection1/conf Note: The Dataverse Project team has customized Solr to boost results that come from certain indexed elements inside the Dataverse installation, for example prioritizing results from Dataverse collections over Datasets. If you would like to remove this, edit your ``solrconfig.xml`` and remove the ```` element and its contents. If you have ideas about how this boosting could be improved, feel free to contact us through our Google Group https://groups.google.com/forum/#!forum/dataverse-dev . -A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-9.4.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400`` +A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-9.6.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400`` Solr will warn about needing to increase the number of file descriptors and max processes in a production environment but will still run with defaults. We have increased these values to the recommended levels by adding ulimit -n 65000 to the init script, and the following to ``/etc/security/limits.conf``:: @@ -209,7 +209,7 @@ Solr launches asynchronously and attempts to use the ``lsof`` binary to watch fo Finally, you need to tell Solr to create the core "collection1" on startup:: - echo "name=collection1" > /usr/local/solr/solr-9.4.1/server/solr/collection1/core.properties + echo "name=collection1" > /usr/local/solr/solr-9.6.1/server/solr/collection1/core.properties Dataverse collection ("dataverse") page uses Solr very heavily. On a busy instance this may cause the search engine to become the performance bottleneck, making these pages take increasingly longer to load, potentially affecting the overall performance of the application and/or causing Solr itself to crash. If this is observed on your instance, we recommend uncommenting the following lines in the ```` section of the ``solrconfig.xml`` file:: diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index 33e7b52004b..19f4440081a 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -103,7 +103,7 @@ services: solr: container_name: "solr" hostname: "solr" - image: solr:9.4.1 + image: solr:9.6.1 depends_on: - solr_initializer restart: on-failure From cd63c39b0513dc8bb85d9b452cbaf4a6dc5e7500 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 10 Oct 2024 17:25:49 -0400 Subject: [PATCH 3/5] up solr version --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index d5cffcec0aa..ccabcca2b02 100644 --- a/.env +++ b/.env @@ -1,5 +1,5 @@ APP_IMAGE=gdcc/dataverse:unstable POSTGRES_VERSION=16 DATAVERSE_DB_USER=dataverse -SOLR_VERSION=9.3.0 +SOLR_VERSION=9.6.1 SKIP_DEPLOY=0 \ No newline at end of file From 2826b11d1f81d462bfa285613c103d8243f6f976 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Thu, 12 Dec 2024 16:03:41 -0500 Subject: [PATCH 4/5] 9.7.0 changes --- conf/solr/solrconfig.xml | 21 ++++++++++++++++++- ....md => 10713-Solr9.7.0 and lib updates.md} | 6 +++--- .../files/etc/systemd/solr.service | 6 +++--- .../source/developers/classic-dev-env.rst | 2 +- .../source/installation/prerequisites.rst | 16 +++++++------- docker/compose/demo/compose.yml | 2 +- 6 files changed, 36 insertions(+), 17 deletions(-) rename doc/release-notes/{10713-Solr9.6.1 and lib updates.md => 10713-Solr9.7.0 and lib updates.md} (74%) diff --git a/conf/solr/solrconfig.xml b/conf/solr/solrconfig.xml index 2115cb1f822..17a2ea9b987 100644 --- a/conf/solr/solrconfig.xml +++ b/conf/solr/solrconfig.xml @@ -35,7 +35,7 @@ that you fully re-index after changing this setting as it can affect both how text is indexed and queried. --> - 9.10 + 9.11 ${solr.max.booleanClauses:1024} + + ${solr.query.minPrefixLength:-1} + @@ -1015,6 +1030,10 @@ [^\w-\.] _ + + 1000 + true + diff --git a/doc/release-notes/10713-Solr9.6.1 and lib updates.md b/doc/release-notes/10713-Solr9.7.0 and lib updates.md similarity index 74% rename from doc/release-notes/10713-Solr9.6.1 and lib updates.md rename to doc/release-notes/10713-Solr9.7.0 and lib updates.md index 84807f6c176..6700f541dc4 100644 --- a/doc/release-notes/10713-Solr9.6.1 and lib updates.md +++ b/doc/release-notes/10713-Solr9.7.0 and lib updates.md @@ -1,10 +1,10 @@ -Solr 9.6.1 is now the version recommended in our installation guides and used with automated testing. Other libraries Dataverse uses have been updated as well. +Solr 9.7.0 is now the version recommended in our installation guides and used with automated testing. Other libraries Dataverse uses have been updated as well. For the upgrade instructions section: [note that 6.5 may contain other solr-related changes, so the instructions may need to contain information merged from multiple release notes!] If you are upgrading Solr: - - Install solr-9.6.1 following the instructions from the Installation guide. + - Install solr-9.7.0 following the instructions from the Installation guide. - Run a full reindex to populate the search catalog. - - Note that it may be possible to skip the reindexing step by simply moving the existing `.../server/solr/collection1/` under the new `solr-9.6.1` installation directory. This however has not been thoroughly tested and is not officially supported. \ No newline at end of file + - Note that it may be possible to skip the reindexing step by simply moving the existing `.../server/solr/collection1/` under the new `solr-9.7.0` installation directory. This however has not been thoroughly tested and is not officially supported. \ No newline at end of file diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service index 8aee5db3d4d..72915ac7673 100644 --- a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service +++ b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service @@ -5,9 +5,9 @@ After = syslog.target network.target remote-fs.target nss-lookup.target [Service] User = solr Type = forking -WorkingDirectory = /usr/local/solr/solr-9.6.1 -ExecStart = /usr/local/solr/solr-9.6.1/bin/solr start -m 1g -ExecStop = /usr/local/solr/solr-9.6.1/bin/solr stop +WorkingDirectory = /usr/local/solr/solr-9.7.0 +ExecStart = /usr/local/solr/solr-9.7.0/bin/solr start -m 1g +ExecStop = /usr/local/solr/solr-9.7.0/bin/solr stop LimitNOFILE=65000 LimitNPROC=65000 Restart=on-failure diff --git a/doc/sphinx-guides/source/developers/classic-dev-env.rst b/doc/sphinx-guides/source/developers/classic-dev-env.rst index 3806a791fff..ae94ebcc553 100755 --- a/doc/sphinx-guides/source/developers/classic-dev-env.rst +++ b/doc/sphinx-guides/source/developers/classic-dev-env.rst @@ -136,7 +136,7 @@ On Linux, you should just install PostgreSQL using your favorite package manager Install Solr ^^^^^^^^^^^^ -`Solr `_ 9.6.1 is required. +`Solr `_ 9.7.0 is required. Follow the instructions in the "Installing Solr" section of :doc:`/installation/prerequisites` in the main Installation guide. diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index 48d0158e740..a7163074d3e 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -163,7 +163,7 @@ The Dataverse software search index is powered by Solr. Supported Versions ================== -The Dataverse software has been tested with Solr version 9.6.1. Future releases in the 9.x series are likely to be compatible. Please get in touch (:ref:`support`) if you are having trouble with a newer version. +The Dataverse software has been tested with Solr version 9.7.0. Future releases in the 9.x series are likely to be compatible. Please get in touch (:ref:`support`) if you are having trouble with a newer version. Installing Solr =============== @@ -178,19 +178,19 @@ Become the ``solr`` user and then download and configure Solr:: su - solr cd /usr/local/solr - wget https://archive.apache.org/dist/solr/solr/9.6.1/solr-9.6.1.tgz - tar xvzf solr-9.6.1.tgz - cd solr-9.6.1 + wget https://archive.apache.org/dist/solr/solr/9.7.0/solr-9.7.0.tgz + tar xvzf solr-9.7.0.tgz + cd solr-9.7.0 cp -r server/solr/configsets/_default server/solr/collection1 You should already have a "dvinstall.zip" file that you downloaded from https://github.com/IQSS/dataverse/releases . Unzip it into ``/tmp``. Then copy the files into place:: - cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-9.6.1/server/solr/collection1/conf - cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-9.6.1/server/solr/collection1/conf + cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-9.7.0/server/solr/collection1/conf + cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-9.7.0/server/solr/collection1/conf Note: The Dataverse Project team has customized Solr to boost results that come from certain indexed elements inside the Dataverse installation, for example prioritizing results from Dataverse collections over Datasets. If you would like to remove this, edit your ``solrconfig.xml`` and remove the ```` element and its contents. If you have ideas about how this boosting could be improved, feel free to contact us through our Google Group https://groups.google.com/forum/#!forum/dataverse-dev . -A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-9.6.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400`` +A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-9.7.0/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400`` Solr will warn about needing to increase the number of file descriptors and max processes in a production environment but will still run with defaults. We have increased these values to the recommended levels by adding ulimit -n 65000 to the init script, and the following to ``/etc/security/limits.conf``:: @@ -209,7 +209,7 @@ Solr launches asynchronously and attempts to use the ``lsof`` binary to watch fo Finally, you need to tell Solr to create the core "collection1" on startup:: - echo "name=collection1" > /usr/local/solr/solr-9.6.1/server/solr/collection1/core.properties + echo "name=collection1" > /usr/local/solr/solr-9.7.0/server/solr/collection1/core.properties Dataverse collection ("dataverse") page uses Solr very heavily. On a busy instance this may cause the search engine to become the performance bottleneck, making these pages take increasingly longer to load, potentially affecting the overall performance of the application and/or causing Solr itself to crash. If this is observed on your instance, we recommend uncommenting the following lines in the ```` section of the ``solrconfig.xml`` file:: diff --git a/docker/compose/demo/compose.yml b/docker/compose/demo/compose.yml index 4f993bc7905..eb9d28acb6a 100644 --- a/docker/compose/demo/compose.yml +++ b/docker/compose/demo/compose.yml @@ -134,7 +134,7 @@ services: solr: container_name: "solr" hostname: "solr" - image: solr:9.6.1 + image: solr:9.7.0 depends_on: - solr_initializer restart: on-failure From da03d52d82515c535af978e0e97e4a8821287b45 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 13 Dec 2024 14:48:32 -0500 Subject: [PATCH 5/5] missed 9.6.1 --- .../source/_static/installation/files/etc/init.d/solr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr index 4326ca70aaf..ffd1e414d44 100755 --- a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr +++ b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr @@ -5,7 +5,7 @@ # chkconfig: 35 92 08 # description: Starts and stops Apache Solr -SOLR_DIR="/usr/local/solr/solr-9.6.1" +SOLR_DIR="/usr/local/solr/solr-9.7.0" SOLR_COMMAND="bin/solr" SOLR_ARGS="-m 1g" SOLR_USER=solr