From 339042c6c8e51de55cbb246a7dc7514847eba158 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 27 Jan 2022 10:56:02 -0500 Subject: [PATCH 01/21] #47 update solr scripts --- scripts/vagrant/setup-solr.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/vagrant/setup-solr.sh b/scripts/vagrant/setup-solr.sh index 41635994bc5..70d3fc632a7 100755 --- a/scripts/vagrant/setup-solr.sh +++ b/scripts/vagrant/setup-solr.sh @@ -5,12 +5,12 @@ SOLR_USER=solr SOLR_HOME=/usr/local/solr mkdir $SOLR_HOME chown $SOLR_USER:$SOLR_USER $SOLR_HOME -su $SOLR_USER -s /bin/sh -c "cp /dataverse/downloads/solr-8.8.1.tgz $SOLR_HOME" -su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-8.8.1.tgz" -su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-8.8.1/server/solr && cp -r configsets/_default . && mv _default collection1" -su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.8.1/schema*.xml $SOLR_HOME/solr-8.8.1/server/solr/collection1/conf/" -su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.8.1/solrconfig.xml $SOLR_HOME/solr-8.8.1/server/solr/collection1/conf/solrconfig.xml" -su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-8.8.1 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/" +su $SOLR_USER -s /bin/sh -c "cp /dataverse/downloads/solr-8.11.1.tgz $SOLR_HOME" +su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME && tar xfz solr-8.11.1.tgz" +su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-8.11.1/server/solr && cp -r configsets/_default . && mv _default collection1" +su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.11.1/schema*.xml $SOLR_HOME/solr-8.11.1/server/solr/collection1/conf/" +su $SOLR_USER -s /bin/sh -c "cp /dataverse/conf/solr/8.11.1/solrconfig.xml $SOLR_HOME/solr-8.11.1/server/solr/collection1/conf/solrconfig.xml" +su $SOLR_USER -s /bin/sh -c "cd $SOLR_HOME/solr-8.11.1 && bin/solr start && bin/solr create_core -c collection1 -d server/solr/collection1/conf/" cp /dataverse/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr /etc/init.d/solr chmod 755 /etc/init.d/solr /etc/init.d/solr stop From 9c9be885f4cd40808c2fb1065aa4f1a8476f6e7b Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Mon, 31 Jan 2022 09:48:02 -0500 Subject: [PATCH 02/21] #47 update download script --- downloads/download.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/downloads/download.sh b/downloads/download.sh index 33476c24b76..5362a0a2b7c 100755 --- a/downloads/download.sh +++ b/downloads/download.sh @@ -1,5 +1,5 @@ #!/bin/sh curl -L -O https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.5/payara-5.2021.5.zip -curl -L -O https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz +curl -L -O https://archive.apache.org/dist/lucene/solr/8.11.1/solr-8.11.1.tgz curl -L -O https://search.maven.org/remotecontent?filepath=org/jboss/weld/weld-osgi-bundle/2.2.10.Final/weld-osgi-bundle-2.2.10.Final-glassfish4.jar curl -s -L http://sourceforge.net/projects/schemaspy/files/schemaspy/SchemaSpy%205.0.0/schemaSpy_5.0.0.jar/download > schemaSpy_5.0.0.jar From 1ba337d2a5933b474caba794fe69b5529d786289 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 2 Feb 2022 10:42:44 -0500 Subject: [PATCH 03/21] #47 update solr.service --- .../_static/installation/files/etc/systemd/solr.service | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service index 96960793938..d89ee108377 100644 --- a/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service +++ b/doc/sphinx-guides/source/_static/installation/files/etc/systemd/solr.service @@ -5,9 +5,9 @@ After = syslog.target network.target remote-fs.target nss-lookup.target [Service] User = solr Type = forking -WorkingDirectory = /usr/local/solr/solr-8.8.1 -ExecStart = /usr/local/solr/solr-8.8.1/bin/solr start -m 1g -j "jetty.host=127.0.0.1" -ExecStop = /usr/local/solr/solr-8.8.1/bin/solr stop +WorkingDirectory = /usr/local/solr/solr-8.11.1 +ExecStart = /usr/local/solr/solr-8.11.1/bin/solr start -m 1g -j "jetty.host=127.0.0.1" +ExecStop = /usr/local/solr/solr-8.11.1/bin/solr stop LimitNOFILE=65000 LimitNPROC=65000 Restart=on-failure From 375ea9c110d817d7b78ae15cb205b7555fd152eb Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Wed, 9 Feb 2022 16:15:57 -0500 Subject: [PATCH 04/21] #47 add 8.11 directory --- conf/solr/8.11.1/readme.md | 1 + conf/solr/8.11.1/schema.xml | 1546 +++++++++++++++++++++++++++++ conf/solr/8.11.1/solrconfig.xml | 1410 ++++++++++++++++++++++++++ conf/solr/8.11.1/update-fields.sh | 218 ++++ 4 files changed, 3175 insertions(+) create mode 100644 conf/solr/8.11.1/readme.md create mode 100644 conf/solr/8.11.1/schema.xml create mode 100644 conf/solr/8.11.1/solrconfig.xml create mode 100644 conf/solr/8.11.1/update-fields.sh diff --git a/conf/solr/8.11.1/readme.md b/conf/solr/8.11.1/readme.md new file mode 100644 index 00000000000..4457cf9a7df --- /dev/null +++ b/conf/solr/8.11.1/readme.md @@ -0,0 +1 @@ +Please see the dev guide for what to do with Solr config files. \ No newline at end of file diff --git a/conf/solr/8.11.1/schema.xml b/conf/solr/8.11.1/schema.xml new file mode 100644 index 00000000000..1718962bfb8 --- /dev/null +++ b/conf/solr/8.11.1/schema.xml @@ -0,0 +1,1546 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/conf/solr/8.11.1/solrconfig.xml b/conf/solr/8.11.1/solrconfig.xml new file mode 100644 index 00000000000..3e4e5adc7b6 --- /dev/null +++ b/conf/solr/8.11.1/solrconfig.xml @@ -0,0 +1,1410 @@ + + + + + + + + + 7.3.0 + + + + + + + + + + + + + + + + + + + + ${solr.data.dir:} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.lock.type:native} + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${solr.ulog.dir:} + ${solr.ulog.numVersionBuckets:65536} + + + + + ${solr.autoCommit.maxTime:15000} + false + + + + + + ${solr.autoSoftCommit.maxTime:-1} + + + + + + + + + + + + + + 1024 + + + + + + + + + + + + + + + + + + + + + + + + true + + + + + + 20 + + + 200 + + + + + + + + + + + + + + + + false + + + + + + + + + + + + + + + + + + + + + + explicit + 10 + edismax + 0.075 + + dvName^400 + authorName^180 + dvSubject^190 + dvDescription^180 + dvAffiliation^170 + title^130 + subject^120 + keyword^110 + topicClassValue^100 + dsDescriptionValue^90 + authorAffiliation^80 + publicationCitation^60 + producerName^50 + fileName^30 + fileDescription^30 + variableLabel^20 + variableName^10 + _text_^1.0 + + + dvName^200 + authorName^100 + dvSubject^100 + dvDescription^100 + dvAffiliation^100 + title^75 + subject^75 + keyword^75 + topicClassValue^75 + dsDescriptionValue^75 + authorAffiliation^75 + publicationCitation^75 + producerName^75 + + + + isHarvested:false^25000 + + + + + + + + + + + + + + + + + + explicit + json + true + + + + + + + + explicit + + + + + + _text_ + + + + + + + true + ignored_ + _text_ + + + + + + + + + text_general + + + + + + default + _text_ + solr.DirectSolrSpellChecker + + internal + + 0.5 + + 2 + + 1 + + 5 + + 4 + + 0.01 + + + + + + + + + + + + default + on + true + 10 + 5 + 5 + true + true + 10 + 5 + + + spellcheck + + + + + + + + + + true + + + tvComponent + + + + + + + + + + + + true + false + + + terms + + + + + + + + string + + + + + + explicit + + + elevator + + + + + + + + + + + 100 + + + + + + + + 70 + + 0.5 + + [-\w ,/\n\"']{20,200} + + + + + + + ]]> + ]]> + + + + + + + + + + + + + + + + + + + + + + + + ,, + ,, + ,, + ,, + ,]]> + ]]> + + + + + + 10 + .,!? + + + + + + + WORD + + + en + US + + + + + + + + + + + + + + [^\w-\.] + _ + + + + + + + yyyy-MM-dd'T'HH:mm:ss.SSSZ + yyyy-MM-dd'T'HH:mm:ss,SSSZ + yyyy-MM-dd'T'HH:mm:ss.SSS + yyyy-MM-dd'T'HH:mm:ss,SSS + yyyy-MM-dd'T'HH:mm:ssZ + yyyy-MM-dd'T'HH:mm:ss + yyyy-MM-dd'T'HH:mmZ + yyyy-MM-dd'T'HH:mm + yyyy-MM-dd HH:mm:ss.SSSZ + yyyy-MM-dd HH:mm:ss,SSSZ + yyyy-MM-dd HH:mm:ss.SSS + yyyy-MM-dd HH:mm:ss,SSS + yyyy-MM-dd HH:mm:ssZ + yyyy-MM-dd HH:mm:ss + yyyy-MM-dd HH:mmZ + yyyy-MM-dd HH:mm + yyyy-MM-dd + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/plain; charset=UTF-8 + + + + + ${velocity.template.base.dir:} + ${velocity.solr.resource.loader.enabled:true} + ${velocity.params.resource.loader.enabled:false} + + + + + 5 + + + + + + + + + + + + + + diff --git a/conf/solr/8.11.1/update-fields.sh b/conf/solr/8.11.1/update-fields.sh new file mode 100644 index 00000000000..49ea8151c77 --- /dev/null +++ b/conf/solr/8.11.1/update-fields.sh @@ -0,0 +1,218 @@ +#!/usr/bin/env bash + +set -euo pipefail + +#### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### +# This script will +# 1. take a file (or read it from STDIN) with all and definitions +# 2. and replace the sections between the include guards with those in a given +# schema.xml file +# The script validates the presence, uniqueness and order of the include guards. +#### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### + + +### Variables +# Internal use only (fork to change) +VERSION="0.1" +INPUT="" +FIELDS="" +COPY_FIELDS="" +TRIGGER_CHAIN=0 +ED_DELETE_FIELDS="'a+,'b-d" +ED_DELETE_COPYFIELDS="'a+,'b-d" + +SOLR_SCHEMA_FIELD_BEGIN_MARK="SCHEMA-FIELDS::BEGIN" +SOLR_SCHEMA_FIELD_END_MARK="SCHEMA-FIELDS::END" +SOLR_SCHEMA_COPYFIELD_BEGIN_MARK="SCHEMA-COPY-FIELDS::BEGIN" +SOLR_SCHEMA_COPYFIELD_END_MARK="SCHEMA-COPY-FIELDS::END" +MARKS_ORDERED="${SOLR_SCHEMA_FIELD_BEGIN_MARK} ${SOLR_SCHEMA_FIELD_END_MARK} ${SOLR_SCHEMA_COPYFIELD_BEGIN_MARK} ${SOLR_SCHEMA_COPYFIELD_END_MARK}" + +### Common functions +function error { + echo "ERROR:" "$@" >&2 + exit 2 +} + +function exists { + type "$1" >/dev/null 2>&1 && return 0 + ( IFS=:; for p in $PATH; do [ -x "${p%/}/$1" ] && return 0; done; return 1 ) +} + +function usage { + cat << EOF +$(basename "$0") ${VERSION} +Usage: $(basename "$0") [-hp] [ schema file ] [ source file ] + +-h Print usage (this text) +-p Chained printing: write all metadata schema related + and present in Solr XML to stdout + +Provide target Solr Schema XML via argument or \$SCHEMA env var. + +Provide source file via argument, \$SOURCE env var or piped input +(wget/curl, chained). Source file = "-" means read STDIN. +EOF + exit 0 +} + +### Options +while getopts ":hp" opt; do + case $opt in + h) usage ;; + p) TRIGGER_CHAIN=1 ;; + \?) echo "Invalid option: -$OPTARG" >&2; exit 1 ;; + :) echo "Option -$OPTARG requires an argument." >&2; exit 1 ;; + esac +done + +# Check for ed and bc being present +exists ed || error "Please ensure ed, bc, sed + awk are installed" +exists bc || error "Please ensure ed, bc, sed + awk are installed" +exists awk || error "Please ensure ed, bc, sed + awk are installed" +exists sed || error "Please ensure ed, bc, sed + awk are installed" + +# remove all the parsed options +shift $((OPTIND-1)) + +# User overrideable locations +SCHEMA=${SCHEMA:-${1:-schema.xml}} +SOURCE=${SOURCE:-${2:-"-"}} + + +### VERIFY SCHEMA FILE EXISTS AND CONTAINS INCLUDE GUARDS ### +# Check for schema file & writeable +if [ ! -w "${SCHEMA}" ]; then + error "Cannot find or write to a XML schema at ${SCHEMA}" +else + # Check schema file for include guards + CHECKS=$( + for MARK in ${MARKS_ORDERED} + do + grep -c "${MARK}" "${SCHEMA}" || error "Missing ${MARK} from ${SCHEMA}" + done + ) + + # Check guards are unique (count occurrences and sum calc via bc) + # Note: fancy workaround to re-add closing \n on Linux & MacOS or no calculation + [ "$( (echo -n "${CHECKS}" | tr '\n' '+' ; echo ) | bc)" -eq 4 ] || \ + error "Some include guards are not unique in ${SCHEMA}" + + # Check guards are in order (line number comparison via bc tricks) + CHECKS=$( + for MARK in ${MARKS_ORDERED} + do + grep -n "${MARK}" "${SCHEMA}" | cut -f 1 -d ":" + done + ) + # Actual comparison of line numbers + echo "${CHECKS}" | tr '\n' '<' | awk -F'<' '{ if ($1 < $2 && $2 < $3 && $3 < $4) {exit 0} else {exit 1} }' || \ + error "Include guards are not in correct order in ${SCHEMA}" + + # Check guards are exclusively in their lines + # (no field or copyField on same line) + for MARK in ${MARKS_ORDERED} + do + grep "${MARK}" "${SCHEMA}" | grep -q -v -e '\(" "${SOURCE}" | sed -e 's#^\s\+##' -e 's#\s\+$##' || true) + + +### DATA HANDLING ### +# Split input into different types +if [ -z "${INPUT}" ]; then + error "No or in input" +else + # Check for definitions (if nomatch, avoid failing pipe) + FIELDS=$(mktemp) + echo "${INPUT}" | grep -e "" | sed -e 's#^# #' > "${FIELDS}" || true + # If file actually contains output, write to schema + if [ -s "${FIELDS}" ]; then + # Use an ed script to replace all + cat << EOF | grep -v -e "^#" | ed -s "${SCHEMA}" +H +# Mark field begin as 'a' +/${SOLR_SCHEMA_FIELD_BEGIN_MARK}/ka +# Mark field end as 'b' +/${SOLR_SCHEMA_FIELD_END_MARK}/kb +# Delete all between lines a and b +${ED_DELETE_FIELDS} +# Read fields file and paste after line a +'ar ${FIELDS} +# Write fields to schema +w +q +EOF + fi + rm "${FIELDS}" + + # Check for definitions (if nomatch, avoid failing pipe) + COPY_FIELDS=$(mktemp) + echo "${INPUT}" | grep -e "" | sed -e 's#^# #' > "${COPY_FIELDS}" || true + # If file actually contains output, write to schema + if [ -s "${COPY_FIELDS}" ]; then + # Use an ed script to replace all , filter comments (BSD ed does not support comments) + cat << EOF | grep -v -e "^#" | ed -s "${SCHEMA}" +H +# Mark copyField begin as 'a' +/${SOLR_SCHEMA_COPYFIELD_BEGIN_MARK}/ka +# Mark copyField end as 'b' +/${SOLR_SCHEMA_COPYFIELD_END_MARK}/kb +# Delete all between lines a and b +${ED_DELETE_COPYFIELDS} +# Read fields file and paste after line a +'ar ${COPY_FIELDS} +# Write copyFields to schema +w +q +EOF + fi + rm "${COPY_FIELDS}" +fi + + +### CHAINING OUTPUT +# Scripts following this one might want to use the field definitions now present +if [ "${TRIGGER_CHAIN}" -eq 1 ]; then + grep -A1000 "${SOLR_SCHEMA_FIELD_BEGIN_MARK}" "${SCHEMA}" | grep -B1000 "${SOLR_SCHEMA_FIELD_END_MARK}" + grep -A1000 "${SOLR_SCHEMA_COPYFIELD_BEGIN_MARK}" "${SCHEMA}" | grep -B1000 "${SOLR_SCHEMA_COPYFIELD_END_MARK}" +fi From ed9f719b7d10f35e93070540e2689a9ef5307ab1 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 10 Feb 2022 10:31:33 -0500 Subject: [PATCH 05/21] #47 add release notes --- doc/release-notes/47-solr-upgrade.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 doc/release-notes/47-solr-upgrade.md diff --git a/doc/release-notes/47-solr-upgrade.md b/doc/release-notes/47-solr-upgrade.md new file mode 100644 index 00000000000..077f5230fc8 --- /dev/null +++ b/doc/release-notes/47-solr-upgrade.md @@ -0,0 +1,11 @@ +### Solr 8 Support/Upgrade + +The Dataverse Software now runs on Solr 8.11.1, the latest available stable release in the Solr 8.x series. + +### Solr Upgrade + +With this release we upgrade to the latest available stable release in the Solr 8.x branch. We recommend a fresh installation of Solr 8.8.1 (the index will be empty) followed by an "index all". + +Before you start the "index all", the Dataverse installation will appear to be empty because the search results come from Solr. As indexing progresses, partial results will appear until indexing is complete. + +See for more information. From 559a84d4e4fce2de119b4dca29864dde889a7b49 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 10 Feb 2022 10:45:14 -0500 Subject: [PATCH 06/21] Update prerequisites.rst --- .../source/installation/prerequisites.rst | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index d25076aab89..0eaf811c8d8 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -157,7 +157,7 @@ The Dataverse Software search index is powered by Solr. Supported Versions ================== -The Dataverse Software has been tested with Solr version 8.8.1. Future releases in the 8.x series are likely to be compatible; however, this cannot be confirmed until they are officially tested. Major releases above 8.x (e.g. 9.x) are not supported. +The Dataverse Software has been tested with Solr version 8.11.1. Future releases in the 8.x series are likely to be compatible; however, this cannot be confirmed until they are officially tested. Major releases above 8.x (e.g. 9.x) are not supported. Installing Solr =============== @@ -172,15 +172,15 @@ Become the ``solr`` user and then download and configure Solr:: su - solr cd /usr/local/solr - wget https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz - tar xvzf solr-8.8.1.tgz - cd solr-8.8.1 + wget https://archive.apache.org/dist/lucene/solr/8.11.1/solr-8.11.1.tgz + tar xvzf solr-8.11.1.tgz + cd solr-8.11.1 cp -r server/solr/configsets/_default server/solr/collection1 You should already have a "dvinstall.zip" file that you downloaded from https://github.com/IQSS/dataverse/releases . Unzip it into ``/tmp``. Then copy the files into place:: - cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-8.8.1/server/solr/collection1/conf - cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-8.8.1/server/solr/collection1/conf + cp /tmp/dvinstall/schema*.xml /usr/local/solr/solr-8.11.1/server/solr/collection1/conf + cp /tmp/dvinstall/solrconfig.xml /usr/local/solr/solr-8.11.1/server/solr/collection1/conf Note: The Dataverse Project team has customized Solr to boost results that come from certain indexed elements inside the Dataverse installation, for example prioritizing results from Dataverse collections over Datasets. If you would like to remove this, edit your ``solrconfig.xml`` and remove the ```` element and its contents. If you have ideas about how this boosting could be improved, feel free to contact us through our Google Group https://groups.google.com/forum/#!forum/dataverse-dev . @@ -203,7 +203,7 @@ Solr launches asynchronously and attempts to use the ``lsof`` binary to watch fo Finally, you need to tell Solr to create the core "collection1" on startup:: - echo "name=collection1" > /usr/local/solr/solr-8.8.1/server/solr/collection1/core.properties + echo "name=collection1" > /usr/local/solr/solr-8.11.1/server/solr/collection1/core.properties Solr Init Script ================ From 21bb3c402ae991806b63b620b1b829276a583708 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 10 Feb 2022 11:06:39 -0500 Subject: [PATCH 07/21] Update prerequisites.rst --- doc/sphinx-guides/source/installation/prerequisites.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/installation/prerequisites.rst b/doc/sphinx-guides/source/installation/prerequisites.rst index 0eaf811c8d8..ac8beb0f284 100644 --- a/doc/sphinx-guides/source/installation/prerequisites.rst +++ b/doc/sphinx-guides/source/installation/prerequisites.rst @@ -184,7 +184,7 @@ You should already have a "dvinstall.zip" file that you downloaded from https:// Note: The Dataverse Project team has customized Solr to boost results that come from certain indexed elements inside the Dataverse installation, for example prioritizing results from Dataverse collections over Datasets. If you would like to remove this, edit your ``solrconfig.xml`` and remove the ```` element and its contents. If you have ideas about how this boosting could be improved, feel free to contact us through our Google Group https://groups.google.com/forum/#!forum/dataverse-dev . -A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-8.8.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400`` +A Dataverse installation requires a change to the ``jetty.xml`` file that ships with Solr. Edit ``/usr/local/solr/solr-8.11.1/server/etc/jetty.xml`` , increasing ``requestHeaderSize`` from ``8192`` to ``102400`` Solr will warn about needing to increase the number of file descriptors and max processes in a production environment but will still run with defaults. We have increased these values to the recommended levels by adding ulimit -n 65000 to the init script, and the following to ``/etc/security/limits.conf``:: From 30d0d925cc6b41d377b9f03cac60ca86e610584d Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 16 Feb 2022 14:55:10 -0500 Subject: [PATCH 08/21] upgrade Docker to Solr 8.11.1 --- conf/docker-aio/0prep_deps.sh | 4 ++-- conf/docker-aio/c8.dockerfile | 8 ++++---- conf/docker-aio/entrypoint.bash | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/conf/docker-aio/0prep_deps.sh b/conf/docker-aio/0prep_deps.sh index 335c644f0ab..02534b51d6a 100755 --- a/conf/docker-aio/0prep_deps.sh +++ b/conf/docker-aio/0prep_deps.sh @@ -9,11 +9,11 @@ if [ ! -e dv/deps/payara-5.2021.5.zip ]; then wget https://s3-eu-west-1.amazonaws.com/payara.fish/Payara+Downloads/5.2021.5/payara-5.2021.5.zip -O dv/deps/payara-5.2021.5.zip fi -if [ ! -e dv/deps/solr-8.8.1dv.tgz ]; then +if [ ! -e dv/deps/solr-8.11.1dv.tgz ]; then echo "solr dependency prep" # schema changes *should* be the only ones... cd dv/deps/ - wget https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz -O solr-8.8.1dv.tgz + wget https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.11.1.tgz -O solr-8.11.1dv.tgz cd ../../ fi diff --git a/conf/docker-aio/c8.dockerfile b/conf/docker-aio/c8.dockerfile index 2216c669f46..dacb46d6924 100644 --- a/conf/docker-aio/c8.dockerfile +++ b/conf/docker-aio/c8.dockerfile @@ -23,7 +23,7 @@ COPY testdata/sushi_sample_logs.json /tmp/ COPY disableipv6.conf /etc/sysctl.d/ RUN rm /etc/httpd/conf/* COPY httpd.conf /etc/httpd/conf -RUN cd /opt ; tar zxf /tmp/dv/deps/solr-8.8.1dv.tgz +RUN cd /opt ; tar zxf /tmp/dv/deps/solr-8.11.1dv.tgz RUN cd /opt ; unzip /tmp/dv/deps/payara-5.2021.5.zip ; ln -s /opt/payara5 /opt/glassfish4 # this copy of domain.xml is the result of running `asadmin set server.monitoring-service.module-monitoring-levels.jvm=LOW` on a default glassfish installation (aka - enable the glassfish REST monitir endpoint for the jvm` @@ -34,9 +34,9 @@ RUN sudo -u postgres /usr/pgsql-13/bin/initdb -D /var/lib/pgsql/13/data -E 'UTF- # copy configuration related files RUN cp /tmp/dv/pg_hba.conf /var/lib/pgsql/13/data/ -RUN cp -r /opt/solr-8.8.1/server/solr/configsets/_default /opt/solr-8.8.1/server/solr/collection1 -RUN cp /tmp/dv/schema*.xml /opt/solr-8.8.1/server/solr/collection1/conf/ -RUN cp /tmp/dv/solrconfig.xml /opt/solr-8.8.1/server/solr/collection1/conf/solrconfig.xml +RUN cp -r /opt/solr-8.11.1/server/solr/configsets/_default /opt/solr-8.11.1/server/solr/collection1 +RUN cp /tmp/dv/schema*.xml /opt/solr-8.11.1/server/solr/collection1/conf/ +RUN cp /tmp/dv/solrconfig.xml /opt/solr-8.11.1/server/solr/collection1/conf/solrconfig.xml # skipping payara user and solr user (run both as root) diff --git a/conf/docker-aio/entrypoint.bash b/conf/docker-aio/entrypoint.bash index 90f60da20f2..236bb30f67a 100755 --- a/conf/docker-aio/entrypoint.bash +++ b/conf/docker-aio/entrypoint.bash @@ -1,7 +1,7 @@ #!/usr/bin/env bash export LANG=en_US.UTF-8 sudo -u postgres /usr/pgsql-13/bin/pg_ctl start -D /var/lib/pgsql/13/data & -cd /opt/solr-8.8.1/ +cd /opt/solr-8.11.1/ # TODO: Run Solr as non-root and remove "-force". bin/solr start -force bin/solr create_core -c collection1 -d server/solr/collection1/conf -force From 40ebccff3f42fe7ad0bcd390612b1e5afd2c331a Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 16 Feb 2022 14:55:45 -0500 Subject: [PATCH 09/21] upgrade Vagrant to Solr 8.11.1 --- scripts/vagrant/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/vagrant/setup.sh b/scripts/vagrant/setup.sh index c151a9addf3..a424a99975f 100644 --- a/scripts/vagrant/setup.sh +++ b/scripts/vagrant/setup.sh @@ -52,7 +52,7 @@ echo "Ensuring Unix user '$SOLR_USER' exists" useradd $SOLR_USER || : DOWNLOAD_DIR='/dataverse/downloads' PAYARA_ZIP="$DOWNLOAD_DIR/payara-5.2021.5.zip" -SOLR_TGZ="$DOWNLOAD_DIR/solr-8.8.1.tgz" +SOLR_TGZ="$DOWNLOAD_DIR/solr-8.11.1.tgz" if [ ! -f $PAYARA_ZIP ] || [ ! -f $SOLR_TGZ ]; then echo "Couldn't find $PAYARA_ZIP or $SOLR_TGZ! Running download script...." cd $DOWNLOAD_DIR && ./download.sh && cd From 70856a30843b9aea8991740e001e5eb683d9440f Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 16 Feb 2022 14:56:17 -0500 Subject: [PATCH 10/21] upgrade init script to Solr 8.11.1 --- .../source/_static/installation/files/etc/init.d/solr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr index 16d364fc9cf..7ca04cdff3f 100755 --- a/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr +++ b/doc/sphinx-guides/source/_static/installation/files/etc/init.d/solr @@ -5,7 +5,7 @@ # chkconfig: 35 92 08 # description: Starts and stops Apache Solr -SOLR_DIR="/usr/local/solr/solr-8.8.1" +SOLR_DIR="/usr/local/solr/solr-8.11.1" SOLR_COMMAND="bin/solr" SOLR_ARGS="-m 1g -j jetty.host=127.0.0.1" SOLR_USER=solr From 1614384703714dbcb23d4a974100dca74bf1cefa Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 16 Feb 2022 14:56:43 -0500 Subject: [PATCH 11/21] upgrade to Solr 8.11.1 in dev guide --- .../source/developers/dev-environment.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/sphinx-guides/source/developers/dev-environment.rst b/doc/sphinx-guides/source/developers/dev-environment.rst index 828a94954c0..4b76e475da7 100755 --- a/doc/sphinx-guides/source/developers/dev-environment.rst +++ b/doc/sphinx-guides/source/developers/dev-environment.rst @@ -117,7 +117,7 @@ On Linux, you should just install PostgreSQL using your favorite package manager Install Solr ~~~~~~~~~~~~ -`Solr `_ 8.8.1 is required. +`Solr `_ 8.11.1 is required. To install Solr, execute the following commands: @@ -127,25 +127,25 @@ To install Solr, execute the following commands: ``cd /usr/local/solr`` -``curl -O http://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.8.1.tgz`` +``curl -O http://archive.apache.org/dist/lucene/solr/8.11.1/solr-8.11.1.tgz`` -``tar xvfz solr-8.8.1.tgz`` +``tar xvfz solr-8.11.1.tgz`` -``cd solr-8.8.1/server/solr`` +``cd solr-8.11.1/server/solr`` ``cp -r configsets/_default collection1`` -``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.8.1/schema.xml`` +``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.11.1/schema.xml`` -``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.8.1/schema_dv_mdb_fields.xml`` +``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.11.1/schema_dv_mdb_fields.xml`` ``mv schema*.xml collection1/conf`` -``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.8.1/solrconfig.xml`` +``curl -O https://raw.githubusercontent.com/IQSS/dataverse/develop/conf/solr/8.11.1/solrconfig.xml`` ``mv solrconfig.xml collection1/conf/solrconfig.xml`` -``cd /usr/local/solr/solr-8.8.1`` +``cd /usr/local/solr/solr-8.11.1`` (Please note that the extra jetty argument below is a security measure to limit connections to Solr to only your computer. For extra security, run a firewall.) From 5538d6f816e75da0464ae4604abd74b24a9e9078 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 16 Feb 2022 15:10:31 -0500 Subject: [PATCH 12/21] fix link to Solr in Docker --- conf/docker-aio/0prep_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/docker-aio/0prep_deps.sh b/conf/docker-aio/0prep_deps.sh index 02534b51d6a..09eaefdf27f 100755 --- a/conf/docker-aio/0prep_deps.sh +++ b/conf/docker-aio/0prep_deps.sh @@ -13,7 +13,7 @@ if [ ! -e dv/deps/solr-8.11.1dv.tgz ]; then echo "solr dependency prep" # schema changes *should* be the only ones... cd dv/deps/ - wget https://archive.apache.org/dist/lucene/solr/8.8.1/solr-8.11.1.tgz -O solr-8.11.1dv.tgz + wget https://archive.apache.org/dist/lucene/solr/8.11.1/solr-8.11.1.tgz -O solr-8.11.1dv.tgz cd ../../ fi From 7f2370d084db2c07c8b8cebf082db1773d1b52a7 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 16 Feb 2022 15:33:13 -0500 Subject: [PATCH 13/21] remove old, unused Solr 8.8.1 files (cruft) --- conf/solr/8.8.1/readme.md | 1 - conf/solr/8.8.1/schema.xml | 1546 ------------------------------ conf/solr/8.8.1/solrconfig.xml | 1410 --------------------------- conf/solr/8.8.1/update-fields.sh | 218 ----- 4 files changed, 3175 deletions(-) delete mode 100644 conf/solr/8.8.1/readme.md delete mode 100644 conf/solr/8.8.1/schema.xml delete mode 100644 conf/solr/8.8.1/solrconfig.xml delete mode 100755 conf/solr/8.8.1/update-fields.sh diff --git a/conf/solr/8.8.1/readme.md b/conf/solr/8.8.1/readme.md deleted file mode 100644 index 4457cf9a7df..00000000000 --- a/conf/solr/8.8.1/readme.md +++ /dev/null @@ -1 +0,0 @@ -Please see the dev guide for what to do with Solr config files. \ No newline at end of file diff --git a/conf/solr/8.8.1/schema.xml b/conf/solr/8.8.1/schema.xml deleted file mode 100644 index 1718962bfb8..00000000000 --- a/conf/solr/8.8.1/schema.xml +++ /dev/null @@ -1,1546 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - id - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/conf/solr/8.8.1/solrconfig.xml b/conf/solr/8.8.1/solrconfig.xml deleted file mode 100644 index 51100af7d6e..00000000000 --- a/conf/solr/8.8.1/solrconfig.xml +++ /dev/null @@ -1,1410 +0,0 @@ - - - - - - - - - 7.3.0 - - - - - - - - - - - - - - - - - - - - ${solr.data.dir:} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.lock.type:native} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - - - - ${solr.autoCommit.maxTime:15000} - false - - - - - - ${solr.autoSoftCommit.maxTime:-1} - - - - - - - - - - - - - - 1024 - - - - - - - - - - - - - - - - - - - - - - - - true - - - - - - 20 - - - 200 - - - - - - - - - - - - - - - - false - - - - - - - - - - - - - - - - - - - - - - explicit - 10 - edismax - 0.075 - - dvName^400 - authorName^180 - dvSubject^190 - dvDescription^180 - dvAffiliation^170 - title^130 - subject^120 - keyword^110 - topicClassValue^100 - dsDescriptionValue^90 - authorAffiliation^80 - publicationCitation^60 - producerName^50 - fileName^30 - fileDescription^30 - variableLabel^20 - variableName^10 - _text_^1.0 - - - dvName^200 - authorName^100 - dvSubject^100 - dvDescription^100 - dvAffiliation^100 - title^75 - subject^75 - keyword^75 - topicClassValue^75 - dsDescriptionValue^75 - authorAffiliation^75 - publicationCitation^75 - producerName^75 - - - - isHarvested:false^25000 - - - - - - - - - - - - - - - - - - explicit - json - true - - - - - - - - explicit - - - - - - _text_ - - - - - - - true - ignored_ - _text_ - - - - - - - - - text_general - - - - - - default - _text_ - solr.DirectSolrSpellChecker - - internal - - 0.5 - - 2 - - 1 - - 5 - - 4 - - 0.01 - - - - - - - - - - - - default - on - true - 10 - 5 - 5 - true - true - 10 - 5 - - - spellcheck - - - - - - - - - - true - - - tvComponent - - - - - - - - - - - - true - false - - - terms - - - - - - - - string - - - - - - explicit - - - elevator - - - - - - - - - - - 100 - - - - - - - - 70 - - 0.5 - - [-\w ,/\n\"']{20,200} - - - - - - - ]]> - ]]> - - - - - - - - - - - - - - - - - - - - - - - - ,, - ,, - ,, - ,, - ,]]> - ]]> - - - - - - 10 - .,!? - - - - - - - WORD - - - en - US - - - - - - - - - - - - - - [^\w-\.] - _ - - - - - - - yyyy-MM-dd'T'HH:mm:ss.SSSZ - yyyy-MM-dd'T'HH:mm:ss,SSSZ - yyyy-MM-dd'T'HH:mm:ss.SSS - yyyy-MM-dd'T'HH:mm:ss,SSS - yyyy-MM-dd'T'HH:mm:ssZ - yyyy-MM-dd'T'HH:mm:ss - yyyy-MM-dd'T'HH:mmZ - yyyy-MM-dd'T'HH:mm - yyyy-MM-dd HH:mm:ss.SSSZ - yyyy-MM-dd HH:mm:ss,SSSZ - yyyy-MM-dd HH:mm:ss.SSS - yyyy-MM-dd HH:mm:ss,SSS - yyyy-MM-dd HH:mm:ssZ - yyyy-MM-dd HH:mm:ss - yyyy-MM-dd HH:mmZ - yyyy-MM-dd HH:mm - yyyy-MM-dd - - - - - - - - - - - - - - - - - - - - - - - - - - - - - text/plain; charset=UTF-8 - - - - - ${velocity.template.base.dir:} - ${velocity.solr.resource.loader.enabled:true} - ${velocity.params.resource.loader.enabled:false} - - - - - 5 - - - - - - - - - - - - - - diff --git a/conf/solr/8.8.1/update-fields.sh b/conf/solr/8.8.1/update-fields.sh deleted file mode 100755 index 49ea8151c77..00000000000 --- a/conf/solr/8.8.1/update-fields.sh +++ /dev/null @@ -1,218 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -#### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### -# This script will -# 1. take a file (or read it from STDIN) with all and definitions -# 2. and replace the sections between the include guards with those in a given -# schema.xml file -# The script validates the presence, uniqueness and order of the include guards. -#### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### #### - - -### Variables -# Internal use only (fork to change) -VERSION="0.1" -INPUT="" -FIELDS="" -COPY_FIELDS="" -TRIGGER_CHAIN=0 -ED_DELETE_FIELDS="'a+,'b-d" -ED_DELETE_COPYFIELDS="'a+,'b-d" - -SOLR_SCHEMA_FIELD_BEGIN_MARK="SCHEMA-FIELDS::BEGIN" -SOLR_SCHEMA_FIELD_END_MARK="SCHEMA-FIELDS::END" -SOLR_SCHEMA_COPYFIELD_BEGIN_MARK="SCHEMA-COPY-FIELDS::BEGIN" -SOLR_SCHEMA_COPYFIELD_END_MARK="SCHEMA-COPY-FIELDS::END" -MARKS_ORDERED="${SOLR_SCHEMA_FIELD_BEGIN_MARK} ${SOLR_SCHEMA_FIELD_END_MARK} ${SOLR_SCHEMA_COPYFIELD_BEGIN_MARK} ${SOLR_SCHEMA_COPYFIELD_END_MARK}" - -### Common functions -function error { - echo "ERROR:" "$@" >&2 - exit 2 -} - -function exists { - type "$1" >/dev/null 2>&1 && return 0 - ( IFS=:; for p in $PATH; do [ -x "${p%/}/$1" ] && return 0; done; return 1 ) -} - -function usage { - cat << EOF -$(basename "$0") ${VERSION} -Usage: $(basename "$0") [-hp] [ schema file ] [ source file ] - --h Print usage (this text) --p Chained printing: write all metadata schema related - and present in Solr XML to stdout - -Provide target Solr Schema XML via argument or \$SCHEMA env var. - -Provide source file via argument, \$SOURCE env var or piped input -(wget/curl, chained). Source file = "-" means read STDIN. -EOF - exit 0 -} - -### Options -while getopts ":hp" opt; do - case $opt in - h) usage ;; - p) TRIGGER_CHAIN=1 ;; - \?) echo "Invalid option: -$OPTARG" >&2; exit 1 ;; - :) echo "Option -$OPTARG requires an argument." >&2; exit 1 ;; - esac -done - -# Check for ed and bc being present -exists ed || error "Please ensure ed, bc, sed + awk are installed" -exists bc || error "Please ensure ed, bc, sed + awk are installed" -exists awk || error "Please ensure ed, bc, sed + awk are installed" -exists sed || error "Please ensure ed, bc, sed + awk are installed" - -# remove all the parsed options -shift $((OPTIND-1)) - -# User overrideable locations -SCHEMA=${SCHEMA:-${1:-schema.xml}} -SOURCE=${SOURCE:-${2:-"-"}} - - -### VERIFY SCHEMA FILE EXISTS AND CONTAINS INCLUDE GUARDS ### -# Check for schema file & writeable -if [ ! -w "${SCHEMA}" ]; then - error "Cannot find or write to a XML schema at ${SCHEMA}" -else - # Check schema file for include guards - CHECKS=$( - for MARK in ${MARKS_ORDERED} - do - grep -c "${MARK}" "${SCHEMA}" || error "Missing ${MARK} from ${SCHEMA}" - done - ) - - # Check guards are unique (count occurrences and sum calc via bc) - # Note: fancy workaround to re-add closing \n on Linux & MacOS or no calculation - [ "$( (echo -n "${CHECKS}" | tr '\n' '+' ; echo ) | bc)" -eq 4 ] || \ - error "Some include guards are not unique in ${SCHEMA}" - - # Check guards are in order (line number comparison via bc tricks) - CHECKS=$( - for MARK in ${MARKS_ORDERED} - do - grep -n "${MARK}" "${SCHEMA}" | cut -f 1 -d ":" - done - ) - # Actual comparison of line numbers - echo "${CHECKS}" | tr '\n' '<' | awk -F'<' '{ if ($1 < $2 && $2 < $3 && $3 < $4) {exit 0} else {exit 1} }' || \ - error "Include guards are not in correct order in ${SCHEMA}" - - # Check guards are exclusively in their lines - # (no field or copyField on same line) - for MARK in ${MARKS_ORDERED} - do - grep "${MARK}" "${SCHEMA}" | grep -q -v -e '\(" "${SOURCE}" | sed -e 's#^\s\+##' -e 's#\s\+$##' || true) - - -### DATA HANDLING ### -# Split input into different types -if [ -z "${INPUT}" ]; then - error "No or in input" -else - # Check for definitions (if nomatch, avoid failing pipe) - FIELDS=$(mktemp) - echo "${INPUT}" | grep -e "" | sed -e 's#^# #' > "${FIELDS}" || true - # If file actually contains output, write to schema - if [ -s "${FIELDS}" ]; then - # Use an ed script to replace all - cat << EOF | grep -v -e "^#" | ed -s "${SCHEMA}" -H -# Mark field begin as 'a' -/${SOLR_SCHEMA_FIELD_BEGIN_MARK}/ka -# Mark field end as 'b' -/${SOLR_SCHEMA_FIELD_END_MARK}/kb -# Delete all between lines a and b -${ED_DELETE_FIELDS} -# Read fields file and paste after line a -'ar ${FIELDS} -# Write fields to schema -w -q -EOF - fi - rm "${FIELDS}" - - # Check for definitions (if nomatch, avoid failing pipe) - COPY_FIELDS=$(mktemp) - echo "${INPUT}" | grep -e "" | sed -e 's#^# #' > "${COPY_FIELDS}" || true - # If file actually contains output, write to schema - if [ -s "${COPY_FIELDS}" ]; then - # Use an ed script to replace all , filter comments (BSD ed does not support comments) - cat << EOF | grep -v -e "^#" | ed -s "${SCHEMA}" -H -# Mark copyField begin as 'a' -/${SOLR_SCHEMA_COPYFIELD_BEGIN_MARK}/ka -# Mark copyField end as 'b' -/${SOLR_SCHEMA_COPYFIELD_END_MARK}/kb -# Delete all between lines a and b -${ED_DELETE_COPYFIELDS} -# Read fields file and paste after line a -'ar ${COPY_FIELDS} -# Write copyFields to schema -w -q -EOF - fi - rm "${COPY_FIELDS}" -fi - - -### CHAINING OUTPUT -# Scripts following this one might want to use the field definitions now present -if [ "${TRIGGER_CHAIN}" -eq 1 ]; then - grep -A1000 "${SOLR_SCHEMA_FIELD_BEGIN_MARK}" "${SCHEMA}" | grep -B1000 "${SOLR_SCHEMA_FIELD_END_MARK}" - grep -A1000 "${SOLR_SCHEMA_COPYFIELD_BEGIN_MARK}" "${SCHEMA}" | grep -B1000 "${SOLR_SCHEMA_COPYFIELD_END_MARK}" -fi From 1bedb40c44ed8af61c0bc1e547696c4e62ca80e9 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 16 Feb 2022 15:35:24 -0500 Subject: [PATCH 14/21] .gitignore apache-maven downloaded by docker-aio --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 7be8263f483..83671abf43e 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,6 @@ venv scripts/search/data/binary/trees.png.thumb140 src/main/webapp/resources/images/cc0.png.thumb140 src/main/webapp/resources/images/dataverseproject.png.thumb140 + +# apache-maven is downloaded by docker-aio +apache-maven* From ef0ef27868c79f8046159f8b24dfd6388fce099e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 16 Feb 2022 15:48:47 -0500 Subject: [PATCH 15/21] update Makefile to Solr 8.11.1 --- scripts/installer/Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/installer/Makefile b/scripts/installer/Makefile index 006b5c818cd..1722f208175 100644 --- a/scripts/installer/Makefile +++ b/scripts/installer/Makefile @@ -56,13 +56,13 @@ ${JHOVE_SCHEMA}: ../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR} @echo copying jhove schema file /bin/cp ../../conf/jhove/jhoveConfig.xsd ${INSTALLER_ZIP_DIR} -${SOLR_SCHEMA}: ../../conf/solr/8.8.1/schema.xml ../../conf/solr/8.8.1/update-fields.sh ${INSTALLER_ZIP_DIR} +${SOLR_SCHEMA}: ../../conf/solr/8.11.1/schema.xml ../../conf/solr/8.11.1/update-fields.sh ${INSTALLER_ZIP_DIR} @echo copying Solr schema file - /bin/cp ../../conf/solr/8.8.1/schema.xml ../../conf/solr/8.8.1/update-fields.sh ${INSTALLER_ZIP_DIR} + /bin/cp ../../conf/solr/8.11.1/schema.xml ../../conf/solr/8.11.1/update-fields.sh ${INSTALLER_ZIP_DIR} -${SOLR_CONFIG}: ../../conf/solr/8.8.1/solrconfig.xml ${INSTALLER_ZIP_DIR} +${SOLR_CONFIG}: ../../conf/solr/8.11.1/solrconfig.xml ${INSTALLER_ZIP_DIR} @echo copying Solr config file - /bin/cp ../../conf/solr/8.8.1/solrconfig.xml ${INSTALLER_ZIP_DIR} + /bin/cp ../../conf/solr/8.11.1/solrconfig.xml ${INSTALLER_ZIP_DIR} ${PYTHON_FILES}: README_python.txt install.py installConfig.py installAppServer.py installUtils.py requirements.txt default.config interactive.config ${INSTALLER_ZIP_DIR} @echo copying Python installer files From 7633a001159b7ff67159a074135ab938904aaec2 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 16 Feb 2022 16:12:10 -0500 Subject: [PATCH 16/21] bump solrj to 8.11.1 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index d4b4fb7f2d6..235d1ec0317 100644 --- a/pom.xml +++ b/pom.xml @@ -434,7 +434,7 @@ org.apache.solr solr-solrj - 8.8.1 + 8.11.1 colt From c11d2811f7c2cbd22f0ec68743a014cfd2f7aa58 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 16 Feb 2022 16:13:06 -0500 Subject: [PATCH 17/21] make script executable again --- conf/solr/8.11.1/update-fields.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 conf/solr/8.11.1/update-fields.sh diff --git a/conf/solr/8.11.1/update-fields.sh b/conf/solr/8.11.1/update-fields.sh old mode 100644 new mode 100755 From 0b5b895877dec2f44356e0d207d88a32bede2bb3 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 16 Feb 2022 16:14:02 -0500 Subject: [PATCH 18/21] git docs building again following solr upgrade --- doc/sphinx-guides/source/admin/metadatacustomization.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/admin/metadatacustomization.rst b/doc/sphinx-guides/source/admin/metadatacustomization.rst index 1321968c758..9fb10099f22 100644 --- a/doc/sphinx-guides/source/admin/metadatacustomization.rst +++ b/doc/sphinx-guides/source/admin/metadatacustomization.rst @@ -644,7 +644,7 @@ the Solr schema configuration, including any enabled metadata schemas: ``curl "http://localhost:8080/api/admin/index/solr/schema"`` -You can use :download:`update-fields.sh <../../../../conf/solr/8.8.1/update-fields.sh>` to easily add these to the +You can use :download:`update-fields.sh <../../../../conf/solr/8.11.1/update-fields.sh>` to easily add these to the Solr schema you installed for your Dataverse installation. The script needs a target XML file containing your Solr schema. (See the :doc:`/installation/prerequisites/` section of @@ -668,7 +668,7 @@ from some place else than your Dataverse installation). Please note that reconfigurations of your Solr index might require a re-index. Usually release notes indicate a necessary re-index, but for your custom metadata you will need to keep track on your own. -Please note also that if you are going to make a pull request updating ``conf/solr/8.8.1/schema.xml`` with fields you have +Please note also that if you are going to make a pull request updating ``conf/solr/8.11.1/schema.xml`` with fields you have added, you should first load all the custom metadata blocks in ``scripts/api/data/metadatablocks`` (including ones you don't care about) to create a complete list of fields. (This might change in the future.) From 7e55ea43d3bf10ea932e7e4b588bc729cf21b512 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 16 Feb 2022 16:15:40 -0500 Subject: [PATCH 19/21] Solr 8.11.1 in docker-aio, installer readme, shell test --- conf/docker-aio/1prep.sh | 4 ++-- scripts/installer/README.txt | 2 +- tests/shell/spec/update_fields_spec.sh | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/conf/docker-aio/1prep.sh b/conf/docker-aio/1prep.sh index 8986772b12f..0cdcf668374 100755 --- a/conf/docker-aio/1prep.sh +++ b/conf/docker-aio/1prep.sh @@ -4,8 +4,8 @@ # this was based off the phoenix deployment; and is likely uglier and bulkier than necessary in a perfect world mkdir -p testdata/doc/sphinx-guides/source/_static/util/ -cp ../solr/8.8.1/schema*.xml testdata/ -cp ../solr/8.8.1/solrconfig.xml testdata/ +cp ../solr/8.11.1/schema*.xml testdata/ +cp ../solr/8.11.1/solrconfig.xml testdata/ cp ../jhove/jhove.conf testdata/ cp ../jhove/jhoveConfig.xsd testdata/ cd ../../ diff --git a/scripts/installer/README.txt b/scripts/installer/README.txt index f4f316dc958..350a17fc00c 100644 --- a/scripts/installer/README.txt +++ b/scripts/installer/README.txt @@ -35,7 +35,7 @@ from conf/jhove: jhove.conf -SOLR schema and config files, from conf/solr/8.8.1: +SOLR schema and config files, from conf/solr/8.11.1: schema.xml schema_dv_mdb_fields.xml diff --git a/tests/shell/spec/update_fields_spec.sh b/tests/shell/spec/update_fields_spec.sh index 829b67fdcc4..e77121672dd 100644 --- a/tests/shell/spec/update_fields_spec.sh +++ b/tests/shell/spec/update_fields_spec.sh @@ -1,16 +1,16 @@ #shellcheck shell=sh update_fields() { - ../../conf/solr/8.8.1/update-fields.sh "$@" + ../../conf/solr/8.11.1/update-fields.sh "$@" } Describe "Update fields command" Describe "can operate on upstream data" - copyUpstreamSchema() { cp ../../conf/solr/8.8.1/schema.xml data/solr/upstream-schema.xml; } + copyUpstreamSchema() { cp ../../conf/solr/8.11.1/schema.xml data/solr/upstream-schema.xml; } AfterAll 'copyUpstreamSchema' - Path schema-xml="../../conf/solr/8.8.1/schema.xml" + Path schema-xml="../../conf/solr/8.11.1/schema.xml" It "needs upstream schema.xml" The path schema-xml should be exist End @@ -166,4 +166,4 @@ Describe "Update fields command" The output should satisfy match_content data/solr/chain-output.xml End End -End \ No newline at end of file +End From f5a4622b1b4b635c76d51714defc09d40020d6c3 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 16 Feb 2022 16:20:19 -0500 Subject: [PATCH 20/21] remove old Solr version from release note --- doc/release-notes/47-solr-upgrade.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/47-solr-upgrade.md b/doc/release-notes/47-solr-upgrade.md index 077f5230fc8..15729bc4b84 100644 --- a/doc/release-notes/47-solr-upgrade.md +++ b/doc/release-notes/47-solr-upgrade.md @@ -4,7 +4,7 @@ The Dataverse Software now runs on Solr 8.11.1, the latest available stable rele ### Solr Upgrade -With this release we upgrade to the latest available stable release in the Solr 8.x branch. We recommend a fresh installation of Solr 8.8.1 (the index will be empty) followed by an "index all". +With this release we upgrade to the latest available stable release in the Solr 8.x branch. We recommend a fresh installation of Solr (the index will be empty) followed by an "index all". Before you start the "index all", the Dataverse installation will appear to be empty because the search results come from Solr. As indexing progresses, partial results will appear until indexing is complete. From b02fb94113d810dfa35d63f6c303b717e048be86 Mon Sep 17 00:00:00 2001 From: Stephen Kraffmiller Date: Thu, 17 Feb 2022 16:05:16 -0500 Subject: [PATCH 21/21] #47 add line about custom metadata blocks --- doc/release-notes/47-solr-upgrade.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/release-notes/47-solr-upgrade.md b/doc/release-notes/47-solr-upgrade.md index 15729bc4b84..d9f52d3dd9b 100644 --- a/doc/release-notes/47-solr-upgrade.md +++ b/doc/release-notes/47-solr-upgrade.md @@ -9,3 +9,5 @@ With this release we upgrade to the latest available stable release in the Solr Before you start the "index all", the Dataverse installation will appear to be empty because the search results come from Solr. As indexing progresses, partial results will appear until indexing is complete. See for more information. + +Also please refer to for information on updating your solr schema for any custom metadata blocks in your installation.