Merge branch 'apache:main' into feature/SOLR-17334

otto-de · Sep 16, 2024 · b6343e6 · b6343e6
2 parents 4a2bf64 + e2a0402
commit b6343e6
Show file tree

Hide file tree

Showing 148 changed files with 1,354 additions and 999 deletions.
diff --git a/README.md b/README.md
@@ -18,8 +18,8 @@
 # Welcome to the Apache Solr project!
 -----------------------------------
 
-Solr is the popular, blazing fast open source search platform for all your
-enterprise, e-commerce, and analytics needs, built on [Apache Lucene](https://lucene.apache.org/).
+Solr is the blazing-fast, open source, multi-modal search platform built on [Apache Lucene](https://lucene.apache.org/).
+It powers full-text, vector, and geospatial search at many of the world's largest organizations.
 
 [![Build Status](https://ci-builds.apache.org/job/Solr/job/Solr-Artifacts-main/badge/icon?subject=Solr%20Artifacts)](https://ci-builds.apache.org/job/Solr/job/Solr-Artifacts-main/)
 [![Build Status](https://ci-builds.apache.org/job/Solr/job/Solr-Check-main/badge/icon?subject=Solr%20Check)](https://ci-builds.apache.org/job/Solr/job/Solr-Check-main/)

diff --git a/build.gradle b/build.gradle
@@ -132,6 +132,7 @@ apply from: file('gradle/testing/fail-on-no-tests.gradle')
 apply from: file('gradle/testing/fail-on-unsupported-jdk.gradle')
 apply from: file('gradle/testing/alternative-jdk-support.gradle')
 apply from: file('gradle/java/jar-manifest.gradle')
+apply from: file('gradle/testing/retry-test.gradle')
 
 // Publishing and releasing
 apply from: file('gradle/maven/defaults-maven.gradle')

diff --git a/dev-tools/doap/solr.rdf b/dev-tools/doap/solr.rdf
@@ -68,6 +68,13 @@
     </maintainer>
 
     <!-- NOTE: please insert releases in numeric order, NOT chronologically. -->
+    <release>
+         <Version>
+           <name>solr-9.7.0</name>
+           <created>2024-09-09</created>
+           <revision>9.7.0</revision>
+         </Version>
+    </release>
     <release>
          <Version>
            <name>solr-9.6.1</name>

diff --git a/dev-tools/scripts/releaseWizard.yaml b/dev-tools/scripts/releaseWizard.yaml
@@ -87,7 +87,7 @@ templates:
 
     The Solr PMC is pleased to announce the release of Apache Solr {{ release_version }}.
 
-    Solr is the popular, blazing fast, open source NoSQL search platform from the Apache Solr project. Its major features include powerful full-text search, hit highlighting, faceted search, dynamic clustering, database integration, rich document handling, and geospatial search. Solr is highly scalable, providing fault tolerant distributed search and indexing, and powers the search and navigation features of many of the world's largest internet sites.
+    Solr is the blazing-fast, open source, multi-modal search platform built on Apache Lucene. It powers full-text, vector, analytics, and geospatial search at many of the world's largest organizations. Other major features include Kubernetes and docker integration, streaming, highlighting, faceting, and spellchecking.
 
     Solr {{ release_version }} is available for immediate download at:
 

diff --git a/gradle/testing/retry-test.gradle b/gradle/testing/retry-test.gradle
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+allprojects {
+    plugins.withType(JavaPlugin) {
+        tasks.withType(Test) {
+            develocity.testRetry {
+                if (isCIBuild) {
+                    // For more info on params: https://github.com/gradle/test-retry-gradle-plugin
+                    maxRetries = 3
+                    maxFailures = 10
+                    failOnPassedAfterRetry = true
+                }
+            }
+        }
+    }
+}
diff --git a/gradle/validation/jar-checks.gradle b/gradle/validation/jar-checks.gradle
@@ -20,6 +20,10 @@
 // 2) notice file
 // 3) checksum validation/ generation.
 
+// WARNING: The tasks in this file share internal state between tasks without using files.
+// Because of this all tasks here must always execute together, so they cannot define task outputs.
+// TODO: Rewrite the internal state to use state files containing the ext.jarInfos and its referencedFiles
+
 import org.apache.commons.codec.digest.DigestUtils
 
 // This should be false only for debugging.
@@ -210,13 +214,6 @@ subprojects {
     description = "Validate license and notice files of dependencies"
     dependsOn collectJarInfos
 
-    def outputFileName = 'validateJarLicenses'
-    inputs.dir(file(project.rootDir.path + '/solr/licenses'))
-        .withPropertyName('licenses')
-        .withPathSensitivity(PathSensitivity.RELATIVE)
-    outputs.file(layout.buildDirectory.file(outputFileName))
-        .withPropertyName('validateJarLicensesResult')
-
     doLast {
       def errors = []
       jarInfos.each { dep ->
@@ -262,8 +259,7 @@ subprojects {
           }
         }
       }
-      def f = new File(project.buildDir.path + "/" + outputFileName)
-      f.text = errors
+
       if (errors) {
         def msg = "Certain license/ notice files are missing:\n  - " + errors.join("\n  - ")
         if (failOnError) {

diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
@@ -110,7 +110,16 @@ New Features
 Improvements
 ---------------------
 * SOLR-17397: SkipExistingDocumentsProcessor now functions correctly with child documents.  (Tim Owens via Eric Pugh)
-* SOLR-17180: Deprecate snapshotscli.sh in favour of bin/solr snapshot sub commands.  Now able to manage Snapshots from the CLI.  HDFS module specific snapshot script now ships as part of that module in the modules/hdfs/bin directory. (Eric Pugh)  
+
+* SOLR-17180: Deprecate snapshotscli.sh in favour of bin/solr snapshot sub commands.  Now able to manage Snapshots from the CLI.  HDFS module specific snapshot script now ships as part of that module in the modules/hdfs/bin directory. (Eric Pugh)
+
+* SOLR-17419: An alternate ShardHandlerFactory is now available, ParallelHttpShardHandlerFactory,
+  which may help reduce distributed-search latency in collections with many shards, especially
+  when PKI is used between nodes. (Jason Gerlowski)
+
+* SOLR-17382: Deprecate -a and -addlopts in favour of --jvm-opts for passing options into the JVM in bin/solr.   (Eric Pugh, Christos Malliaridis)
+
+* SOLR-17431: Deprecate -p parameter where it doesn't refer to a port in bin/solr. (Eric Pugh, Christos Malliaridis)
 
 Optimizations
 ---------------------
@@ -135,6 +144,13 @@ Bug Fixes
 ---------------------
 * SOLR-12429: Uploading a configset with a symbolic link produces a IOException. Now a error message to user generated instead. (Eric Pugh)
 
+* SOLR-17421: Fixed a rare case where overseer was stuck after a failure when changing overseer to honor the
+  node role for preferred overseer. (Pierre Salagnac)
+
+* SOLR-17416: Fixed ExportHandler bug that silently suppressed errors and returned partial results in some situations (hossman)
+
+* SOLR-16254: Clarify when a bin/solr create needs to be run on the same server as Solr. (Eric Pugh)
+
 Dependency Upgrades
 ---------------------
 (No changes)
@@ -145,6 +161,10 @@ Other Changes
 
 * SOLR-17399: Replace the use of the deprecated java.util.Locale constructor with Locale Builder API. (Sanjay Dutt)
 
+* SOLR-17142: Fix Gradle build sometimes gives spurious "unreferenced license file" warnings. (Uwe Schindler)
+
+* SOLR-11318: Introduce unit testing for AssertTool. (Eric Pugh, Jason Gerlowski)
+
 ==================  9.7.0 ==================
 New Features
 ---------------------
@@ -215,6 +235,10 @@ Improvements
   support docValues will have them enabled by default. This behavior can be reverted by setting
   'docValues' parameter for a field or a field type to false, the default for schema versions 1.6 and below. (Houston Putman)
 
+* SOLR-17418: Streamline ConfigSet modification logic.  (Houston Putman, Liu Huajin)
+
+* SOLR-17434: Avoid exposing Solr's IP in the Location header.  (David Smiley)
+
 Optimizations
 ---------------------
 * SOLR-17257: Both Minimize Cores and the Affinity replica placement strategies would over-gather

diff --git a/solr/bin/solr b/solr/bin/solr
@@ -367,7 +367,7 @@ function print_usage() {
 
   if [[ "$CMD" == "start" || "$CMD" == "restart" ]]; then
     echo ""
-    echo "Usage: solr $CMD [-f] [-c] [--host host] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-s solr.solr.home] [-t solr.data.home] [-a \"additional-options\"] [-V]"
+    echo "Usage: solr $CMD [-f] [-c] [--host host] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-s solr.solr.home] [-t solr.data.home] [--jvm-opts \"jvm-opts\"] [-V]"
     echo ""
     echo "  -f                    Start Solr in foreground; default starts Solr in the background"
     echo "                          and sends stdout / stderr to solr-PORT-console.log"
@@ -410,9 +410,9 @@ function print_usage() {
     echo "      schemaless:         Schema-less example (schema is inferred from data during indexing)"
     echo "      films:              Example of starting with _default configset and adding explicit fields dynamically"
     echo ""
-    echo "  -a <jvmParams>        Additional parameters to pass to the JVM when starting Solr, such as to setup"
+    echo "  --jvm-opts <jvmParams> Additional parameters to pass to the JVM when starting Solr, such as to setup"
     echo "                          Java debug options. For example, to enable a Java debugger to attach to the Solr JVM"
-    echo "                          you could pass: -a \"-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=18983\""
+    echo "                          you could pass: --jvm-opts \"-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=18983\""
     echo "                          In most cases, you should wrap the additional parameters in double quotes."
     echo ""
     echo "  -j <jettyParams>      Additional parameters to pass to Jetty when starting Solr."
@@ -831,9 +831,9 @@ if [ $# -gt 0 ]; then
             PASS_TO_RUN_EXAMPLE+=("-z" "$ZK_HOST")
             shift 2
         ;;
-        -a|--additional-options|-addlopts)
+        -a|--jvm-opts|-addlopts)
             ADDITIONAL_CMD_OPTS="$2"
-            PASS_TO_RUN_EXAMPLE+=("-a" "$ADDITIONAL_CMD_OPTS")
+            PASS_TO_RUN_EXAMPLE+=("--jvm-opts" "$ADDITIONAL_CMD_OPTS")
             shift 2
         ;;
         -j|--jettyconfig|-jettyconfig)

diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd
@@ -307,7 +307,7 @@ goto done
 
 :start_usage
 @echo.
-@echo Usage: solr %SCRIPT_CMD% [-f] [-c] [--host hostname] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-s solr.solr.home] [-t solr.data.home] [-a "additional-options"] [-V]
+@echo Usage: solr %SCRIPT_CMD% [-f] [-c] [--host hostname] [-p port] [-d directory] [-z zkHost] [-m memory] [-e example] [-s solr.solr.home] [-t solr.data.home] [--jvm-opts "jvm-opts"] [-V]
 @echo.
 @echo   -f            Start Solr in foreground; default starts Solr in the background
 @echo                   and sends stdout / stderr to solr-PORT-console.log
@@ -350,9 +350,9 @@ goto done
 @echo       schemaless:     Schema-less example (schema is inferred from data during indexing)
 @echo       films:          Example of starting with _default configset and defining explicit fields dynamically
 @echo.
-@echo   -a opts       Additional parameters to pass to the JVM when starting Solr, such as to setup
+@echo   --jvm-opts opts Additional parameters to pass to the JVM when starting Solr, such as to setup
 @echo                 Java debug options. For example, to enable a Java debugger to attach to the Solr JVM
-@echo                 you could pass: -a "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=18983"
+@echo                 you could pass: --jvm-opts "-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=18983"
 @echo                 In most cases, you should wrap the additional parameters in double quotes.
 @echo.
 @echo   -j opts       Additional parameters to pass to Jetty when starting Solr.
@@ -426,7 +426,7 @@ IF "%1"=="-s" goto set_solr_url
 IF "%1"=="--solr-url" goto set_solr_url
 IF "%1"=="-solrUrl" goto set_solr_url
 IF "%1"=="-a" goto set_addl_opts
-IF "%1"=="--additional-options" goto set_addl_opts
+IF "%1"=="--jvm-opts" goto set_addl_opts
 IF "%1"=="-j" goto set_addl_jetty_config
 IF "%1"=="--jettyconfig" goto set_addl_jetty_config
 IF "%1"=="--noprompt" goto set_noprompt
@@ -582,7 +582,7 @@ IF "%firstChar%"=="-" (
 )
 
 set SOLR_HOST=%~2
-set "PASS_TO_RUN_EXAMPLE=-h %~2 !PASS_TO_RUN_EXAMPLE!"
+set "PASS_TO_RUN_EXAMPLE=--host %~2 !PASS_TO_RUN_EXAMPLE!"
 SHIFT
 SHIFT
 goto parse_args

diff --git a/solr/bin/solr.in.sh b/solr/bin/solr.in.sh
@@ -102,7 +102,7 @@
 
 # Anything you add to the SOLR_OPTS variable will be included in the java
 # start command line as-is, in ADDITION to other options. If you specify the
-# -a option on start script, those options will be appended as well. Examples:
+# --jvm-opts option on start script, those options will be appended as well. Examples:
 #SOLR_OPTS="$SOLR_OPTS -Dsolr.autoSoftCommit.maxTime=3000"
 #SOLR_OPTS="$SOLR_OPTS -Dsolr.autoCommit.maxTime=60000"
 
@@ -301,4 +301,4 @@
 
 # Solr internally doesn't use cookies other than for modules such as Kerberos/Hadoop Auth. If you don't need any of those
 # And you don't need them for an external system (such as a load balancer), you can disable the use of a CookieStore with:
-# SOLR_OPTS="$SOLR_OPTS -Dsolr.http.disableCookies=true"
+# SOLR_OPTS="$SOLR_OPTS -Dsolr.http.disableCookies=true"
diff --git a/solr/core/src/java/org/apache/solr/cli/ApiTool.java b/solr/core/src/java/org/apache/solr/cli/ApiTool.java
@@ -125,7 +125,7 @@ protected String callGet(String url, String credentials) throws Exception {
    * @return Solr base url with port and root (from above example http://localhost:8983/solr)
    */
   public static String getSolrUrlFromUri(URI uri) {
-    return uri.getScheme() + "://" + uri.getAuthority() + "/" + uri.getPath().split("/")[1];
+    return uri.resolve("/" + uri.getPath().split("/")[1]).toString();
   }
 
   public static ModifiableSolrParams getSolrParamsFromUri(URI uri) {

diff --git a/solr/core/src/java/org/apache/solr/cli/AuthTool.java b/solr/core/src/java/org/apache/solr/cli/AuthTool.java
@@ -75,7 +75,7 @@ public String getUsage() {
 
   @Override
   public String getHeader() {
-    return "\nUpdates or enables/disables authentication.  Must be run on the machine hosting Solr.\n"
+    return "Updates or enables/disables authentication.  Must be run on the Solr server itself.\n"
         + "\n"
         + "List of options:";
   }

diff --git a/solr/core/src/java/org/apache/solr/cli/ConfigTool.java b/solr/core/src/java/org/apache/solr/cli/ConfigTool.java
@@ -22,6 +22,8 @@
 import java.util.List;
 import java.util.Map;
 import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.DeprecatedAttributes;
+import org.apache.commons.cli.MissingArgumentException;
 import org.apache.commons.cli.Option;
 import org.apache.solr.client.solrj.SolrClient;
 import org.apache.solr.common.util.NamedList;
@@ -66,11 +68,26 @@ public List<Option> getOptions() {
             .desc(
                 "Config API action, one of: set-property, unset-property, set-user-property, unset-user-property; default is 'set-property'.")
             .build(),
-        Option.builder("p")
+        Option.builder()
             .longOpt("property")
             .argName("PROP")
             .hasArg()
-            .required(true)
+            .required(
+                false) // Should be TRUE but have a deprecated option to deal with first, so we
+            // enforce in code
+            .desc(
+                "Name of the Config API property to apply the action to, such as: 'updateHandler.autoSoftCommit.maxTime'.")
+            .build(),
+        Option.builder("p")
+            .deprecated(
+                DeprecatedAttributes.builder()
+                    .setForRemoval(true)
+                    .setSince("9.8")
+                    .setDescription("Use --property instead")
+                    .get())
+            .hasArg()
+            .argName("PROP")
+            .required(false)
             .desc(
                 "Name of the Config API property to apply the action to, such as: 'updateHandler.autoSoftCommit.maxTime'.")
             .build(),
@@ -93,9 +110,13 @@ public void runImpl(CommandLine cli) throws Exception {
     String solrUrl = SolrCLI.normalizeSolrUrl(cli);
     String action = cli.getOptionValue("action", "set-property");
     String collection = cli.getOptionValue("name");
-    String property = cli.getOptionValue("property");
+    String property = SolrCLI.getOptionWithDeprecatedAndDefault(cli, "property", "p", null);
     String value = cli.getOptionValue("value");
 
+    if (property == null) {
+      throw new MissingArgumentException("'property' is a required option.");
+    }
+
     Map<String, Object> jsonObj = new HashMap<>();
     if (value != null) {
       Map<String, String> setMap = new HashMap<>();

diff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/cli/CreateTool.java
@@ -41,7 +41,7 @@
 import org.apache.solr.client.solrj.request.CoreAdminRequest;
 import org.apache.solr.client.solrj.request.GenericSolrRequest;
 import org.apache.solr.client.solrj.response.CoreAdminResponse;
-import org.apache.solr.common.cloud.ZkMaintenanceUtils;
+import org.apache.solr.cloud.ZkConfigSetService;
 import org.apache.solr.common.cloud.ZkStateReader;
 import org.apache.solr.common.params.CollectionAdminParams;
 import org.apache.solr.common.params.CommonParams;
@@ -66,6 +66,14 @@ public String getName() {
     return "create";
   }
 
+  @Override
+  public String getHeader() {
+    return "Creates a core or collection depending on whether Solr is running in standalone (core) or SolrCloud mode (collection).\n"
+        + "If you are using standalone mode you must run this command on the Solr server itself.\n"
+        + "\n"
+        + "List of options:";
+  }
+
   @Override
   public List<Option> getOptions() {
     return List.of(
@@ -311,7 +319,10 @@ protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli
         confName = collectionName;
       }
 
+      // TODO: This should be done using the configSet API
       final Path configsetsDirPath = SolrCLI.getConfigSetsDir(solrInstallDirPath);
+      ConfigSetService configSetService =
+          new ZkConfigSetService(ZkStateReader.from(cloudSolrClient).getZkClient());
       Path confPath = ConfigSetService.getConfigsetPath(confDir, configsetsDirPath.toString());
 
       echoIfVerbose(
@@ -322,11 +333,8 @@ protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli
               + " to ZooKeeper at "
               + cloudSolrClient.getClusterStateProvider().getQuorumHosts(),
           cli);
-      ZkMaintenanceUtils.uploadToZK(
-          ZkStateReader.from(cloudSolrClient).getZkClient(),
-          confPath,
-          ZkMaintenanceUtils.CONFIGS_ZKNODE + "/" + confName,
-          ZkMaintenanceUtils.UPLOAD_FILENAME_EXCLUDE_PATTERN);
+      // We will trust the config since we have the Zookeeper Address
+      configSetService.uploadConfig(confName, confPath, true);
     }
 
     // since creating a collection is a heavy-weight operation, check for existence first

diff --git a/solr/core/src/java/org/apache/solr/cli/DeleteTool.java b/solr/core/src/java/org/apache/solr/cli/DeleteTool.java
@@ -61,7 +61,7 @@ public String getName() {
 
   @Override
   public String getHeader() {
-    return "\nDeletes a core or collection depending on whether Solr is running in standalone (core) or SolrCloud"
+    return "Deletes a core or collection depending on whether Solr is running in standalone (core) or SolrCloud"
         + " mode (collection). If you're deleting a collection in SolrCloud mode, the default behavior is to also"
         + " delete the configuration directory from Zookeeper so long as it is not being used by another collection.\n"
         + " You can override this behavior by passing -deleteConfig false when running this command.\n"