Merge pull request #8415 from IQSS/47-upgrade-solr-8-11

47 upgrade solr 8 11

.gitignore

@@ -70,3 +70,6 @@ venv
 scripts/search/data/binary/trees.png.thumb140
 src/main/webapp/resources/images/cc0.png.thumb140
 src/main/webapp/resources/images/dataverseproject.png.thumb140
+
+# apache-maven is downloaded by docker-aio
+apache-maven*

conf/docker-aio/0prep_deps.sh

@@ -9,11 +9,11 @@ if [ ! -e dv/deps/payara-5.2021.5.zip ]; then
 	wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.5/payara-5.2021.5.zip  -O dv/deps/payara-5.2021.5.zip
 fi
 
-if [ ! -e dv/deps/solr-8.8.1dv.tgz ]; then
+if [ ! -e dv/deps/solr-8.11.1dv.tgz ]; then
 	echo "solr dependency prep"
 	# schema changes *should* be the only ones...
 	cd dv/deps/	
-	wget https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz -O solr-8.8.1dv.tgz
+	wget https://archive.apache.org/dist/lucene/solr/8.11.1/solr-8.11.1.tgz -O solr-8.11.1dv.tgz
 	cd ../../
 fi
 

conf/docker-aio/1prep.sh

@@ -4,8 +4,8 @@
 # this was based off the phoenix deployment; and is likely uglier and bulkier than necessary in a perfect world
 
 mkdir -p testdata/doc/sphinx-guides/source/_static/util/
-cp ../solr/8.8.1/schema*.xml testdata/
-cp ../solr/8.8.1/solrconfig.xml testdata/
+cp ../solr/8.11.1/schema*.xml testdata/
+cp ../solr/8.11.1/solrconfig.xml testdata/
 cp ../jhove/jhove.conf testdata/
 cp ../jhove/jhoveConfig.xsd testdata/
 cd ../../

conf/docker-aio/c8.dockerfile

@@ -23,7 +23,7 @@ COPY testdata/sushi_sample_logs.json /tmp/
 COPY disableipv6.conf /etc/sysctl.d/
 RUN rm /etc/httpd/conf/*
 COPY httpd.conf /etc/httpd/conf 
-RUN cd /opt ; tar zxf /tmp/dv/deps/solr-8.8.1dv.tgz 
+RUN cd /opt ; tar zxf /tmp/dv/deps/solr-8.11.1dv.tgz
 RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2021.5.zip ; ln -s /opt/payara5 /opt/glassfish4
 
 # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm`
@@ -34,9 +34,9 @@ RUN sudo -u postgres /usr/pgsql-13/bin/initdb -D /var/lib/pgsql/13/data -E 'UTF-
 
 # copy configuration related files
 RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/13/data/
-RUN cp -r /opt/solr-8.8.1/server/solr/configsets/_default /opt/solr-8.8.1/server/solr/collection1
-RUN cp /tmp/dv/schema*.xml /opt/solr-8.8.1/server/solr/collection1/conf/
-RUN cp /tmp/dv/solrconfig.xml /opt/solr-8.8.1/server/solr/collection1/conf/solrconfig.xml
+RUN cp -r /opt/solr-8.11.1/server/solr/configsets/_default /opt/solr-8.11.1/server/solr/collection1
+RUN cp /tmp/dv/schema*.xml /opt/solr-8.11.1/server/solr/collection1/conf/
+RUN cp /tmp/dv/solrconfig.xml /opt/solr-8.11.1/server/solr/collection1/conf/solrconfig.xml
 
 # skipping payara user and solr user (run both as root)
 

conf/docker-aio/entrypoint.bash

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 export LANG=en_US.UTF-8
 sudo -u postgres /usr/pgsql-13/bin/pg_ctl start -D /var/lib/pgsql/13/data &
-cd /opt/solr-8.8.1/
+cd /opt/solr-8.11.1/
 # TODO: Run Solr as non-root and remove "-force".
 bin/solr start -force
 bin/solr create_core -c collection1 -d server/solr/collection1/conf -force

conf/solr/8.8.1/solrconfig.xml → conf/solr/8.11.1/solrconfig.xml

@@ -408,7 +408,7 @@
                       to occupy. Note that when this option is specified, the size
                       and initialSize parameters are ignored.
       -->
-    <filterCache class="solr.FastLRUCache"
+    <filterCache class="solr.search.CaffeineCache"
                  size="512"
                  initialSize="512"
                  autowarmCount="0"/>
@@ -421,7 +421,7 @@
             maxRamMB - the maximum amount of RAM (in MB) that this cache is allowed
                        to occupy
       -->
-    <queryResultCache class="solr.LRUCache"
+    <queryResultCache class="solr.search.CaffeineCache"
                       size="512"
                       initialSize="512"
                       autowarmCount="0"/>
@@ -432,14 +432,14 @@
          document).  Since Lucene internal document ids are transient,
          this cache will not be autowarmed.
       -->
-    <documentCache class="solr.LRUCache"
+    <documentCache class="solr.search.CaffeineCache"
                    size="512"
                    initialSize="512"
                    autowarmCount="0"/>
 
     <!-- custom cache currently used by block join -->
     <cache name="perSegFilter"
-           class="solr.search.LRUCache"
+           class="solr.search.CaffeineCache"
            size="10"
            initialSize="0"
            autowarmCount="10"

doc/release-notes/47-solr-upgrade.md

@@ -0,0 +1,13 @@
+### Solr 8 Support/Upgrade
+
+The Dataverse Software now runs on Solr 8.11.1, the latest available stable release in the Solr 8.x series.
+
+### Solr Upgrade
+
+With this release we upgrade to the latest available stable release in the Solr 8.x branch. We recommend a fresh installation of Solr (the index will be empty) followed by an "index all".
+
+Before you start the "index all", the Dataverse installation will appear to be empty because the search results come from Solr. As indexing progresses, partial results will appear until indexing is complete.
+
+See <http://guides.dataverse.org/en/5.10/installation/prerequisites.html#installing-solr> for more information.
+
+Also please refer to <https://guides.dataverse.org/en/5.10/admin/metadatacustomization.html#updating-the-solr-schema> for information on updating your solr schema for any custom metadata blocks in your installation.

doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr

@@ -5,7 +5,7 @@
 # chkconfig: 35 92 08
 # description: Starts and stops Apache Solr
 
-SOLR_DIR="/usr/local/solr/solr-8.8.1"
+SOLR_DIR="/usr/local/solr/solr-8.11.1"
 SOLR_COMMAND="bin/solr"
 SOLR_ARGS="-m 1g -j jetty.host=127.0.0.1"
 SOLR_USER=solr

doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service

@@ -5,9 +5,9 @@ After = syslog.target network.target remote-fs.target nss-lookup.target
 [Service]
 User = solr
 Type = forking
-WorkingDirectory = /usr/local/solr/solr-8.8.1
-ExecStart = /usr/local/solr/solr-8.8.1/bin/solr start -m 1g -j "jetty.host=127.0.0.1"
-ExecStop = /usr/local/solr/solr-8.8.1/bin/solr stop
+WorkingDirectory = /usr/local/solr/solr-8.11.1
+ExecStart = /usr/local/solr/solr-8.11.1/bin/solr start -m 1g -j "jetty.host=127.0.0.1"
+ExecStop = /usr/local/solr/solr-8.11.1/bin/solr stop
 LimitNOFILE=65000
 LimitNPROC=65000
 Restart=on-failure

doc/sphinx-guides/source/admin/metadatacustomization.rst

@@ -644,7 +644,7 @@ the Solr schema configuration, including any enabled metadata schemas:
 
 ``curl "http://localhost:8080/api/admin/index/solr/schema"``
 
-You can use :download:`update-fields.sh <../../../../conf/solr/8.8.1/update-fields.sh>` to easily add these to the
+You can use :download:`update-fields.sh <../../../../conf/solr/8.11.1/update-fields.sh>` to easily add these to the
 Solr schema you installed for your Dataverse installation.
 
 The script needs a target XML file containing your Solr schema. (See the :doc:`/installation/prerequisites/` section of
@@ -668,7 +668,7 @@ from some place else than your Dataverse installation).
 Please note that reconfigurations of your Solr index might require a re-index. Usually release notes indicate
 a necessary re-index, but for your custom metadata you will need to keep track on your own.
 
-Please note also that if you are going to make a pull request updating ``conf/solr/8.8.1/schema.xml`` with fields you have
+Please note also that if you are going to make a pull request updating ``conf/solr/8.11.1/schema.xml`` with fields you have
 added, you should first load all the custom metadata blocks in ``scripts/api/data/metadatablocks`` (including ones you
 don't care about) to create a complete list of fields. (This might change in the future.)
 

doc/sphinx-guides/source/developers/dev-environment.rst

@@ -117,7 +117,7 @@ On Linux, you should just install PostgreSQL using your favorite package manager
 Install Solr
 ~~~~~~~~~~~~
 
-`Solr <http://lucene.apache.org/solr/>`_ 8.8.1 is required.
+`Solr <http://lucene.apache.org/solr/>`_ 8.11.1 is required.
 
 To install Solr, execute the following commands:
 
@@ -127,25 +127,25 @@ To install Solr, execute the following commands:
 
 ``cd /usr/local/solr``
 
-``curl -O http://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz``
+``curl -O http://archive.apache.org/dist/lucene/solr/8.11.1/solr-8.11.1.tgz``
 
-``tar xvfz solr-8.8.1.tgz``
+``tar xvfz solr-8.11.1.tgz``
 
-``cd solr-8.8.1/server/solr``
+``cd solr-8.11.1/server/solr``
 
 ``cp -r configsets/_default collection1``
 
-``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.8.1/schema.xml``
+``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.11.1/schema.xml``
 
-``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.8.1/schema_dv_mdb_fields.xml``
+``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.11.1/schema_dv_mdb_fields.xml``
 
 ``mv schema*.xml collection1/conf``
 
-``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.8.1/solrconfig.xml``
+``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.11.1/solrconfig.xml``
 
 ``mv solrconfig.xml collection1/conf/solrconfig.xml``
 
-``cd /usr/local/solr/solr-8.8.1``
+``cd /usr/local/solr/solr-8.11.1``
 
 (Please note that the extra jetty argument below is a security measure to limit connections to Solr to only your computer. For extra security, run a firewall.)
 

doc/sphinx-guides/source/installation/prerequisites.rst

@@ -157,7 +157,7 @@ The Dataverse Software search index is powered by Solr.
 Supported Versions
 ==================
 
-The Dataverse Software has been tested with Solr version 8.8.1. Future releases in the 8.x series are likely to be compatible; however, this cannot be confirmed until they are officially tested. Major releases above 8.x (e.g. 9.x) are not supported.
+The Dataverse Software has been tested with Solr version 8.11.1. Future releases in the 8.x series are likely to be compatible; however, this cannot be confirmed until they are officially tested. Major releases above 8.x (e.g. 9.x) are not supported.
 
 Installing Solr
 ===============
@@ -172,19 +172,19 @@ Become the ``solr`` user and then download and configure Solr::
 
         su - solr
         cd /usr/local/solr
-        wget https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz
-        tar xvzf solr-8.8.1.tgz
-        cd solr-8.8.1
+        wget https://archive.apache.org/dist/lucene/solr/8.11.1/solr-8.11.1.tgz
+        tar xvzf solr-8.11.1.tgz
+        cd solr-8.11.1
         cp -r server/solr/configsets/_default server/solr/collection1
 
 You should already have a "dvinstall.zip" file that you downloaded from https://github.com/IQSS/dataverse/releases . Unzip it into ``/tmp``. Then copy the files into place::
 
-        cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-8.8.1/server/solr/collection1/conf
-        cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-8.8.1/server/solr/collection1/conf
+        cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-8.11.1/server/solr/collection1/conf
+        cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-8.11.1/server/solr/collection1/conf
 
 Note: The Dataverse Project team has customized Solr to boost results that come from certain indexed elements inside the Dataverse installation, for example prioritizing results from Dataverse collections over Datasets. If you would like to remove this, edit your ``solrconfig.xml`` and remove the ``<str name="qf">`` element and its contents. If you have ideas about how this boosting could be improved, feel free to contact us through our Google Group https://groups.google.com/forum/#!forum/dataverse-dev .
 
-A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-8.8.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400``
+A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-8.11.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400``
 
 Solr will warn about needing to increase the number of file descriptors and max processes in a production environment but will still run with defaults. We have increased these values to the recommended levels by adding ulimit -n 65000 to the init script, and the following to ``/etc/security/limits.conf``::
 
@@ -203,7 +203,7 @@ Solr launches asynchronously and attempts to use the ``lsof`` binary to watch fo
 
 Finally, you need to tell Solr to create the core "collection1" on startup::
 
-        echo "name=collection1" > /usr/local/solr/solr-8.8.1/server/solr/collection1/core.properties
+        echo "name=collection1" > /usr/local/solr/solr-8.11.1/server/solr/collection1/core.properties
 
 Solr Init Script
 ================

downloads/download.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
 curl -L -O https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.5/payara-5.2021.5.zip
-curl -L -O https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz
+curl -L -O https://archive.apache.org/dist/lucene/solr/8.11.1/solr-8.11.1.tgz
 curl -L -O https://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar
 curl -s -L http://sourceforge.net/projects/schemaspy/files/schemaspy/SchemaSpy%205.0.0/schemaSpy_5.0.0.jar/download > schemaSpy_5.0.0.jar

pom.xml

@@ -434,7 +434,7 @@
         <dependency>
             <groupId>org.apache.solr</groupId>
             <artifactId>solr-solrj</artifactId>
-            <version>8.8.1</version>
+            <version>8.11.1</version>
         </dependency>
         <dependency>
             <groupId>colt</groupId>

scripts/installer/Makefile

@@ -56,13 +56,13 @@ ${JHOVE_SCHEMA}:	../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR}
 	@echo copying jhove schema file
 	/bin/cp ../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR}
 
-${SOLR_SCHEMA}:		../../conf/solr/8.8.1/schema.xml ../../conf/solr/8.8.1/update-fields.sh ${INSTALLER_ZIP_DIR}
+${SOLR_SCHEMA}:		../../conf/solr/8.11.1/schema.xml ../../conf/solr/8.11.1/update-fields.sh ${INSTALLER_ZIP_DIR}
 	@echo copying Solr schema file
-	/bin/cp ../../conf/solr/8.8.1/schema.xml ../../conf/solr/8.8.1/update-fields.sh ${INSTALLER_ZIP_DIR}
+	/bin/cp ../../conf/solr/8.11.1/schema.xml ../../conf/solr/8.11.1/update-fields.sh ${INSTALLER_ZIP_DIR}
 
-${SOLR_CONFIG}:		../../conf/solr/8.8.1/solrconfig.xml ${INSTALLER_ZIP_DIR}
+${SOLR_CONFIG}:		../../conf/solr/8.11.1/solrconfig.xml ${INSTALLER_ZIP_DIR}
 	@echo copying Solr config file
-	/bin/cp ../../conf/solr/8.8.1/solrconfig.xml ${INSTALLER_ZIP_DIR}
+	/bin/cp ../../conf/solr/8.11.1/solrconfig.xml ${INSTALLER_ZIP_DIR}
 
 ${PYTHON_FILES}: README_python.txt install.py installConfig.py installAppServer.py installUtils.py requirements.txt default.config interactive.config ${INSTALLER_ZIP_DIR}
 	@echo copying Python installer files

scripts/installer/README.txt

@@ -35,7 +35,7 @@ from conf/jhove:
 
 jhove.conf
 
-SOLR schema and config files, from conf/solr/8.8.1: 
+SOLR schema and config files, from conf/solr/8.11.1:
 
 schema.xml
 schema_dv_mdb_fields.xml

scripts/vagrant/setup-solr.sh

@@ -5,12 +5,12 @@ SOLR_USER=solr
 SOLR_HOME=/usr/local/solr
 mkdir $SOLR_HOME
 chown $SOLR_USER:$SOLR_USER $SOLR_HOME
-su $SOLR_USER -s /bin/sh -c "cp /dataverse/downloads/solr-8.8.1.tgz $SOLR_HOME"
-su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-8.8.1.tgz"
-su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-8.8.1/server/solr && cp -r configsets/_default . && mv _default collection1"
-su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.8.1/schema*.xml $SOLR_HOME/solr-8.8.1/server/solr/collection1/conf/"
-su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.8.1/solrconfig.xml $SOLR_HOME/solr-8.8.1/server/solr/collection1/conf/solrconfig.xml"
-su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-8.8.1 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/"
+su $SOLR_USER -s /bin/sh -c "cp /dataverse/downloads/solr-8.11.1.tgz $SOLR_HOME"
+su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-8.11.1.tgz"
+su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-8.11.1/server/solr && cp -r configsets/_default . && mv _default collection1"
+su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.11.1/schema*.xml $SOLR_HOME/solr-8.11.1/server/solr/collection1/conf/"
+su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.11.1/solrconfig.xml $SOLR_HOME/solr-8.11.1/server/solr/collection1/conf/solrconfig.xml"
+su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-8.11.1 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/"
 cp /dataverse/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr /etc/init.d/solr
 chmod 755 /etc/init.d/solr
 /etc/init.d/solr stop

scripts/vagrant/setup.sh

@@ -52,7 +52,7 @@ echo "Ensuring Unix user '$SOLR_USER' exists"
 useradd $SOLR_USER || :
 DOWNLOAD_DIR='/dataverse/downloads'
 PAYARA_ZIP="$DOWNLOAD_DIR/payara-5.2021.5.zip"
-SOLR_TGZ="$DOWNLOAD_DIR/solr-8.8.1.tgz"
+SOLR_TGZ="$DOWNLOAD_DIR/solr-8.11.1.tgz"
 if [ ! -f $PAYARA_ZIP ] || [ ! -f $SOLR_TGZ ]; then
     echo "Couldn't find $PAYARA_ZIP or $SOLR_TGZ! Running download script...."
     cd $DOWNLOAD_DIR && ./download.sh && cd

tests/shell/spec/update_fields_spec.sh

@@ -1,16 +1,16 @@
 #shellcheck shell=sh
 
 update_fields() {
-  ../../conf/solr/8.8.1/update-fields.sh "$@"
+  ../../conf/solr/8.11.1/update-fields.sh "$@"
 }
 
 Describe "Update fields command"
 
   Describe "can operate on upstream data"
-    copyUpstreamSchema() { cp ../../conf/solr/8.8.1/schema.xml data/solr/upstream-schema.xml; }
+    copyUpstreamSchema() { cp ../../conf/solr/8.11.1/schema.xml data/solr/upstream-schema.xml; }
     AfterAll 'copyUpstreamSchema'
 
-    Path schema-xml="../../conf/solr/8.8.1/schema.xml"
+    Path schema-xml="../../conf/solr/8.11.1/schema.xml"
     It "needs upstream schema.xml"
       The path schema-xml should be exist
     End
@@ -166,4 +166,4 @@ Describe "Update fields command"
       The output should satisfy match_content data/solr/chain-output.xml
     End
   End
-End
+End