Merge branch 'main' into introduce-version-catalog

# Conflicts:
#	solr/prometheus-exporter/build.gradle
#	versions.lock
#	versions.props

.asf.yaml

@@ -28,6 +28,7 @@ github:
     branch_9_4: {}
     branch_9_5: {}
     branch_9_6: {}
+    branch_9_7: {}
     branch_9x: {}
 
   protected_tags:

.github/workflows/bin-solr-test.yml

@@ -10,6 +10,7 @@ on:
       - 'solr/bin/**'
       - 'solr/packaging/**'
       - 'solr/core/src/java/org/apache/solr/cli/**'
+      - 'solr/prometheus-exporter/**'
 
 jobs:
   test:
@@ -47,4 +48,3 @@ jobs:
       with:
         name: logs
         path: solr/packaging/build/test-output
-

.github/workflows/gradle-precommit.yml

@@ -41,4 +41,4 @@ jobs:
     - name: Run gradle check (without tests)
       run: ./gradlew check -x test -Ptask.times=true
 
-    - uses: gradle/wrapper-validation-action@v2
+    - uses: gradle/wrapper-validation-action@v3

README.md

@@ -21,6 +21,9 @@
 Solr is the popular, blazing fast open source search platform for all your
 enterprise, e-commerce, and analytics needs, built on [Apache Lucene](https://lucene.apache.org/).
 
+[![Build Status](https://ci-builds.apache.org/job/Solr/job/Solr-Artifacts-main/badge/icon?subject=Solr%20Artifacts)](https://ci-builds.apache.org/job/Solr/job/Solr-Artifacts-main/)
+[![Build Status](https://ci-builds.apache.org/job/Solr/job/Solr-Check-main/badge/icon?subject=Solr%20Check)](https://ci-builds.apache.org/job/Solr/job/Solr-Check-main/)
+
 For a complete description of the Solr project, team composition, source
 code repositories, and other details, please see the Solr web site at
 https://solr.apache.org/

dev-tools/scripts/releaseWizard.py

@@ -1123,7 +1123,7 @@ def file_to_string(filename):
         return f.read().strip()
 
 def download_keys():
-    download('KEYS', "https://archive.apache.org/dist/solr/KEYS", state.config_path)
+    download('KEYS', "https://downloads.apache.org/solr/KEYS", state.config_path)
 
 def keys_downloaded():
     return os.path.exists(os.path.join(state.config_path, "KEYS"))

dev-tools/scripts/releaseWizard.yaml

@@ -689,7 +689,7 @@ groups:
       {% if release_type == 'major' -%}
       . Change name of version `main ({{ release_version_major }}.0)` into `{{ release_version_major }}.0`
       {%- endif %}
-      . Create a new (unreleased) version `{{ get_next_version }}`
+      . Create a new (unreleased) version `{{ release_version_major }}.{{ release_version_minor + 1 }}`
     types:
     - major
     - minor

gradle/libs.versions.toml

@@ -120,7 +120,7 @@ minJava = "11"
 mockito = "5.12.0"
 morethan-jmhreport = "0.9.0"
 navsecurity = "0.5.10"
-netty = "4.1.111.Final"
+netty = "4.1.112.Final"
 nimbusds-josejwt = "9.30.2"
 nodegradle-node = "7.0.1"
 openapi = "7.6.0"

gradle/testing/randomization.gradle

@@ -126,7 +126,6 @@ configure(allprojects.findAll {project -> project.path.startsWith(":solr") }) {
   plugins.withType(JavaPlugin) {
     ext {
       testOptions += [
-          [propName: 'solr.directoryFactory', value: "org.apache.solr.core.MockDirectoryFactory", description: "Solr directory factory."],
           [propName: 'tests.src.home', value: null, description: "See SOLR-14023."],
           [propName: 'solr.tests.use.numeric.points', value: null, description: "Point implementation to use (true=numerics, false=trie)."],
       ]

solr/CHANGES.txt

@@ -100,10 +100,45 @@ Other Changes
 
 * SOLR-17279: Introduce SecurityJson.java file to Test Framework to consolidate setting up authentication in tests. (Rudy Seitz via Eric Pugh)
 
+==================  9.8.0 ==================
+New Features
+---------------------
+(No changes)
+
+Improvements
+---------------------
+* SOLR-17397: SkipExistingDocumentsProcessor now functions correctly with child documents.  (Tim Owens via Eric Pugh)
+
+Optimizations
+---------------------
+* SOLR-14985: Solrj CloudSolrClient with Solr URLs had serious performance regressions (since the
+  beginning?) in which its collection state cache was not being used, resulting in many extra
+  requests to Solr for cluster information. (Aparna Suresh, shalin, David Smiley)
+
+* SOLR-17102: The VersionBucket indexing lock mechanism was replaced with something just as fast yet
+  that which consumes almost no memory, saving 1MB of memory per SolrCore.  (David Smiley)
+
+* SOLR-17381: Make CLUSTERSTATUS request configurable to improve performance by allowing retrieval of specific information,
+  reducing unnecessary data fetching. (Aparna Suresh, David Smiley)
+
+* SOLR-17396: Reduce thread contention in ZkStateReader.getCollectionProperties(). (Aparna Suresh, David Smiley, Paul McArthur)
+
+Bug Fixes
+---------------------
+(No changes)
+
+Dependency Upgrades
+---------------------
+(No changes)
+
+Other Changes
+---------------------
+(No changes)
+
 ==================  9.7.0 ==================
 New Features
 ---------------------
-* SOLR-13350: Multithreaded search execution (Ishan Chattopadhyaya, Mark Miller, Christine Poerschke, David Smiley, noble)
+* SOLR-13350, SOLR-17298: Opt-in multithreaded search execution (Ishan Chattopadhyaya, Mark Miller, Christine Poerschke, David Smiley, noble, Gus Heck)
 
 * SOLR-17192: Put an UpdateRequestProcessor-enforced soft-limit on the number of fields allowed in a core.  The `NumFieldLimitingUpdateRequestProcessorFactory`
   limit may be adjusted by raising the factory's `maxFields` setting, toggled in and out of "warning-only" mode using the `warnOnly` setting, or disabled entirely
@@ -124,8 +159,18 @@ New Features
 
 * SOLR-10255: Add support for docValues to solr.BinaryField. (Alexey Serba via Mikhail Khludnev, David Smiley)
 
+* SOLR-17276: Prometheus Exporter: now scrapes metrics at a fixed rate instead of delay. (Rafał Harabień)
+
 Improvements
 ---------------------
+* SOLR-10808, SOLR-12963: The Solr schema version has been increased to 1.7.
+  Starting in schema version 1.7, most fields/fieldTypes that support docValues will have them enabled by default.
+  These field types include primitive (Numeric, Date, Bool, String, Enum, UUID), sorting (SortableTextField, SortableBinaryField, CollationField, ICUCollationField) and LatLonPointSpacialField.
+  This behavior can be reverted by setting the 'docValues' parameter for a field or a field type to false, the default for schema versions 1.6 and below.
+  Also in schema version 1.7, all fields/fieldTypes will be unable to be uninverted by default.
+  This behavior can be reverted by setting the 'uninvertible' parameter for a field or a field type to true, the default for schema versions 1.6 and below.
+  (Houston Putman, hossman)
+
 * SOLR-17137: Enable Prometheus exporter to communicate with SSL protected Solr. (Eivind Bergstøl via Eric Pugh)
 
 * SOLR-16921: use -solrUrl to derive the zk host connection for bin/solr zk subcommands (Eric Pugh)
@@ -146,7 +191,7 @@ Improvements
 
 * SOLR-16824: Adopt Linux standard pattern of -- for long option commands, and make all commands "kebab" formatting.  I.e -zkHost is now -zk-host.  The old parameters
   such as -zkHost continue to be supported in the 9.x line of Solr.  -u is now used to specify user credentials everywhere, this only impacts the bin/solr assert
-  commands "same user" check which has -u as the short form of --same-user. (Eric Pugh, janhoy, Jason Gerlowski)
+  commands "same user" check which has -u as the short form of --same-user. (Eric Pugh, janhoy, Jason Gerlowski, Christos Malliaridis)
 
 * SOLR-17346: Synchronise stopwords from snowball with those in Lucene (Alastair Porter via Houston Putman)
 
@@ -167,7 +212,7 @@ Optimizations
 
 * SOLR-17099: Do not return spurious tags when fetching metrics from a Solr node to another. (Pierre Salagnac)
 
-* SOLR-17269: Prevent the "Coordinator node" feature from registering synthetic cores in ZooKeeper (Patson Luk)
+* SOLR-17269, SOLR-17386: Prevent the "Coordinator node" feature from registering synthetic cores in ZooKeeper (ellaeln, Patson Luk, David Smiley, Christine Poerschke)
 
 * SOLR-17330: When not set, loadOnStartup defaults to true, which is the default choice for a core. (Pierre Salagnac via Eric Pugh)
 
@@ -212,6 +257,14 @@ Bug Fixes
 
 * SOLR-17369: Fix "flags" usage in FunctionQParser that caused some issues in vectorSimilarity() with BYTE vector constants (hossman)
 
+* SOLR-17337: Display all custom distributed stages in debug output. (Torsten Bøgh Köster, Christine Poerschke)
+
+* SOLR-17394: Detect and handle non-200 HTTP status codes for requests  made by IndexFetcher (Jason Gerlowski)
+
+* SOLR-17391: Fixed performance regression of misconfigured threadpools from SOLR-16879 (Solr 9.4).
+  Shard splits and concurrent/large collection backup/restore performance was serial.  UpdateLog
+  replay was a little suboptimal in thread usage too.  (Pierre Salagnac, Hakan Özler, David Smiley)
+
 Dependency Upgrades
 ---------------------
 (No changes)
@@ -240,6 +293,12 @@ Other Changes
 
 * SOLR-15831: Refactor bin/solr and bin/solr.cmd commands integration with AuthTool, DeleteTool, and PackageTool to delegate arg parsing to Java code.  Removed limitation of PackageTool only being executed on an active Solr node.  (Eric Pugh)
 
+* SOLR-17322: Once again allow rank queries to use custom TopDocsCollectors that operate on types that extend
+  ScoreDocs (covariant generic types) broken in Solr 9.0. (Stephen Woods via Christine Poerschke)
+
+* SOLR-16996: Update Solr Exporter for Prometheus cli to use commons-cli instead of argparse4j.  (Christos Malliaridis via Eric Pugh)
+
+
 ==================  9.6.1 ==================
 Bug Fixes
 ---------------------

solr/bin/solr.cmd

@@ -246,6 +246,7 @@ set FIRST_ARG=%1
 
 IF [%1]==[] goto usage
 
+REM -help is a special case to faciliate folks learning about how to use Solr.
 IF "%1"=="-help" goto run_solrcli
 IF "%1"=="-usage" goto run_solrcli
 IF "%1"=="-h" goto run_solrcli
@@ -293,7 +294,6 @@ IF "%FIRST_ARG%"=="-help" goto run_solrcli
 IF "%FIRST_ARG%"=="-usage" goto run_solrcli
 IF "%FIRST_ARG%"=="-h" goto run_solrcli
 IF "%FIRST_ARG%"=="--help" goto run_solrcli
-IF "%FIRST_ARG%"=="-help" goto run_solrcli
 IF "%FIRST_ARG%"=="/?" goto run_solrcli
 IF "%SCRIPT_CMD%"=="start" goto start_usage
 IF "%SCRIPT_CMD%"=="restart" goto start_usage
@@ -316,7 +316,7 @@ goto done
 @echo   -f            Start Solr in foreground; default starts Solr in the background
 @echo                   and sends stdout / stderr to solr-PORT-console.log
 @echo.
-@echo   -c or -cloud  Start Solr in SolrCloud mode; if -z not supplied and ZK_HOST not defined in
+@echo   -c or --cloud Start Solr in SolrCloud mode; if -z not supplied and ZK_HOST not defined in
 @echo                   solr.in.cmd, an embedded ZooKeeper instance is started on Solr port+1000,
 @echo                   such as 9983 if Solr is bound to 8983
 @echo.
@@ -1493,10 +1493,10 @@ IF "!ZK_OP!"=="" (
 set CONNECTION_PARAMS=""
 
 IF "!ZK_OP!"=="" (
-  set CONNECTION_PARAMS="-solrUrl !ZK_SOLR_URL!"
+  set CONNECTION_PARAMS="--solr-url !ZK_SOLR_URL!"
 )
 ELSE (
-  set CONNECTION_PARAMS="-zkHost ZK_HOST!"
+  set CONNECTION_PARAMS="--zk-host ZK_HOST!"
 )
 
 IF "!ZK_OP!"=="upconfig" (

solr/core/src/java/org/apache/solr/cli/CreateTool.java

@@ -411,7 +411,7 @@ private void printDefaultConfigsetWarningIfNecessary(CommandLine cli) {
       final String configCommand =
           String.format(
               Locale.ROOT,
-              "bin/solr config -c %s -solrUrl %s -action set-user-property -property update.autoCreateFields -value false",
+              "bin/solr config -c %s -s %s --action set-user-property --property update.autoCreateFields --value false",
               collectionName,
               solrUrl);
       echo(

solr/core/src/java/org/apache/solr/cli/PackageTool.java

@@ -205,8 +205,9 @@ public void runImpl(CommandLine cli) throws Exception {
                           : new String[] {};
                   packageManager.undeploy(packageName, collections, cli.hasOption("cluster"));
                 } else {
+
                   printRed(
-                      "Either specify -cluster to undeploy cluster level plugins or -collections <list-of-collections> to undeploy collection level plugins");
+                      "Either specify --cluster to undeploy cluster level plugins or -collections <list-of-collections> to undeploy collection level plugins");
                 }
                 break;
               }
@@ -259,7 +260,7 @@ public String getHeader() {
     format(sb, "");
     formatGreen(
         sb,
-        "bin/solr package deploy <package-name>[:<version>] [-y] [--update] -collections <comma-separated-collections> [-p <param1>=<val1> -p <param2>=<val2> ...] ");
+        "bin/solr package deploy <package-name>[:<version>] [-y] [--update] --collections <comma-separated-collections> [-p <param1>=<val1> -p <param2>=<val2> ...] ");
     format(
         sb,
         "Bootstraps a previously installed package into the specified collections. It the package accepts parameters for its setup commands, they can be specified (as per package documentation).");
@@ -277,7 +278,7 @@ public String getHeader() {
     format(sb, "Print a list of collections on which a given package has been deployed.");
     format(sb, "");
     formatGreen(
-        sb, "bin/solr package undeploy <package-name> -collections <comma-separated-collections>");
+        sb, "bin/solr package undeploy <package-name> --collections <comma-separated-collections>");
     format(sb, "Undeploy a package from specified collection(s)");
     format(sb, "");
     formatGreen(sb, "bin/solr package uninstall <package-name>:<version>");

solr/core/src/java/org/apache/solr/core/CoreContainer.java

@@ -26,6 +26,7 @@
 import static org.apache.solr.common.params.CommonParams.METRICS_PATH;
 import static org.apache.solr.common.params.CommonParams.ZK_PATH;
 import static org.apache.solr.common.params.CommonParams.ZK_STATUS_PATH;
+import static org.apache.solr.search.CpuAllowedLimit.TIMING_CONTEXT;
 import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
 
 import com.github.benmanes.caffeine.cache.Interner;
@@ -62,6 +63,7 @@
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.solr.api.ClusterPluginsSource;
 import org.apache.solr.api.ContainerPluginsRegistry;
 import org.apache.solr.api.JerseyResource;
@@ -155,6 +157,7 @@
 import org.apache.solr.util.OrderedExecutor;
 import org.apache.solr.util.RefCounted;
 import org.apache.solr.util.StartupLoggingUtils;
+import org.apache.solr.util.ThreadCpuTimer;
 import org.apache.solr.util.stats.MetricUtils;
 import org.apache.zookeeper.KeeperException;
 import org.glassfish.hk2.utilities.binding.AbstractBinder;
@@ -237,7 +240,7 @@ public JerseyAppHandlerCache getJerseyAppHandlerCache() {
       ExecutorUtil.newMDCAwareCachedThreadPool(
           new SolrNamedThreadFactory("coreContainerWorkExecutor"));
 
-  private final OrderedExecutor replayUpdatesExecutor;
+  private final OrderedExecutor<BytesRef> replayUpdatesExecutor;
 
   protected volatile LogWatcher<?> logging = null;
 
@@ -418,7 +421,7 @@ public CoreContainer(NodeConfig config, CoresLocator locator, boolean asyncSolrC
     this.containerProperties = new Properties(config.getSolrProperties());
     this.asyncSolrCoreLoad = asyncSolrCoreLoad;
     this.replayUpdatesExecutor =
-        new OrderedExecutor(
+        new OrderedExecutor<>(
             cfg.getReplayUpdatesThreads(),
             ExecutorUtil.newMDCAwareCachedThreadPool(
                 cfg.getReplayUpdatesThreads(), // thread count
@@ -448,7 +451,8 @@ public CoreContainer(NodeConfig config, CoresLocator locator, boolean asyncSolrC
           ExecutorUtil.newMDCAwareFixedThreadPool(
               indexSearcherExecutorThreads, // thread count
               indexSearcherExecutorThreads * 1000, // queue size
-              new SolrNamedThreadFactory("searcherCollector"));
+              new SolrNamedThreadFactory("searcherCollector"),
+              () -> ThreadCpuTimer.reset(TIMING_CONTEXT));
     } else {
       this.collectorExecutor = null;
     }
@@ -724,7 +728,7 @@ public Tracer getTracer() {
     return tracer;
   }
 
-  public OrderedExecutor getReplayUpdatesExecutor() {
+  public OrderedExecutor<BytesRef> getReplayUpdatesExecutor() {
     return replayUpdatesExecutor;
   }
 
@@ -2054,9 +2058,10 @@ public void reload(String name, UUID coreId) {
 
         DocCollection docCollection = null;
         if (getZkController() != null) {
-          docCollection = getZkController().getClusterState().getCollection(cd.getCollectionName());
+          docCollection =
+              getZkController().getClusterState().getCollectionOrNull(cd.getCollectionName());
           // turn off indexing now, before the new core is registered
-          if (docCollection.getBool(ZkStateReader.READ_ONLY, false)) {
+          if (docCollection != null && docCollection.getBool(ZkStateReader.READ_ONLY, false)) {
             newCore.readOnly = true;
           }
         }