Merge branch 'main' into introduce-version-catalog

# Conflicts:
#	solr/prometheus-exporter/build.gradle
#	versions.lock
#	versions.props

.asf.yaml

@@ -28,6 +28,7 @@ github:
  branch_9_4: {}
  branch_9_5: {}
  branch_9_6: {}
+ branch_9_7: {}
  branch_9x: {}
 
  protected_tags:

.github/workflows/bin-solr-test.yml

@@ -10,6 +10,7 @@ on:
  - 'solr/bin/**'
  - 'solr/packaging/**'
  - 'solr/core/src/java/org/apache/solr/cli/**'
+ - 'solr/prometheus-exporter/**'
 
 jobs:
  test:
@@ -47,4 +48,3 @@ jobs:
  with:
  name: logs
  path: solr/packaging/build/test-output
-

.github/workflows/gradle-precommit.yml

@@ -41,4 +41,4 @@ jobs:
  - name: Run gradle check (without tests)
  run: ./gradlew check -x test -Ptask.times=true
 
- - uses: gradle/wrapper-validation-action@v2
+ - uses: gradle/wrapper-validation-action@v3

README.md

@@ -21,6 +21,9 @@
 Solr is the popular, blazing fast open source search platform for all your
 enterprise, e-commerce, and analytics needs, built on [Apache Lucene](https://lucene.apache.org/).
 
+[![Build Status](https://ci-builds.apache.org/job/Solr/job/Solr-Artifacts-main/badge/icon?subject=Solr%20Artifacts)](https://ci-builds.apache.org/job/Solr/job/Solr-Artifacts-main/)
+[![Build Status](https://ci-builds.apache.org/job/Solr/job/Solr-Check-main/badge/icon?subject=Solr%20Check)](https://ci-builds.apache.org/job/Solr/job/Solr-Check-main/)
+
 For a complete description of the Solr project, team composition, source
 code repositories, and other details, please see the Solr web site at
 https://solr.apache.org/

dev-tools/scripts/releaseWizard.py

@@ -1123,7 +1123,7 @@ def file_to_string(filename):
  return f.read().strip()
 
 def download_keys():
- download('KEYS', "https://archive.apache.org/dist/solr/KEYS", state.config_path)
+ download('KEYS', "https://downloads.apache.org/solr/KEYS", state.config_path)
 
 def keys_downloaded():
  return os.path.exists(os.path.join(state.config_path, "KEYS"))

dev-tools/scripts/releaseWizard.yaml

@@ -689,7 +689,7 @@ groups:
  {% if release_type == 'major' -%}
  . Change name of version `main ({{ release_version_major }}.0)` into `{{ release_version_major }}.0`
  {%- endif %}
- . Create a new (unreleased) version `{{ get_next_version }}`
+ . Create a new (unreleased) version `{{ release_version_major }}.{{ release_version_minor + 1 }}`
  types:
  - major
  - minor

gradle/libs.versions.toml

@@ -120,7 +120,7 @@ minJava = "11"
 mockito = "5.12.0"
 morethan-jmhreport = "0.9.0"
 navsecurity = "0.5.10"
-netty = "4.1.111.Final"
+netty = "4.1.112.Final"
 nimbusds-josejwt = "9.30.2"
 nodegradle-node = "7.0.1"
 openapi = "7.6.0"

gradle/testing/randomization.gradle

@@ -126,7 +126,6 @@ configure(allprojects.findAll {project -> project.path.startsWith(":solr") }) {
  plugins.withType(JavaPlugin) {
  ext {
  testOptions += [
- [propName: 'solr.directoryFactory', value: "org.apache.solr.core.MockDirectoryFactory", description: "Solr directory factory."],
  [propName: 'tests.src.home', value: null, description: "See SOLR-14023."],
  [propName: 'solr.tests.use.numeric.points', value: null, description: "Point implementation to use (true=numerics, false=trie)."],
  ]

solr/CHANGES.txt

@@ -100,10 +100,45 @@ Other Changes
 
 * SOLR-17279: Introduce SecurityJson.java file to Test Framework to consolidate setting up authentication in tests. (Rudy Seitz via Eric Pugh)
 
+================== 9.8.0 ==================
+New Features
+---------------------
+(No changes)
+
+Improvements
+---------------------
+* SOLR-17397: SkipExistingDocumentsProcessor now functions correctly with child documents. (Tim Owens via Eric Pugh)
+
+Optimizations
+---------------------
+* SOLR-14985: Solrj CloudSolrClient with Solr URLs had serious performance regressions (since the
+ beginning?) in which its collection state cache was not being used, resulting in many extra
+ requests to Solr for cluster information. (Aparna Suresh, shalin, David Smiley)
+
+* SOLR-17102: The VersionBucket indexing lock mechanism was replaced with something just as fast yet
+ that which consumes almost no memory, saving 1MB of memory per SolrCore. (David Smiley)
+
+* SOLR-17381: Make CLUSTERSTATUS request configurable to improve performance by allowing retrieval of specific information,
+ reducing unnecessary data fetching. (Aparna Suresh, David Smiley)
+
+* SOLR-17396: Reduce thread contention in ZkStateReader.getCollectionProperties(). (Aparna Suresh, David Smiley, Paul McArthur)
+
+Bug Fixes
+---------------------
+(No changes)
+
+Dependency Upgrades
+---------------------
+(No changes)
+
+Other Changes
+---------------------
+(No changes)
+
 ================== 9.7.0 ==================
 New Features
 ---------------------
-* SOLR-13350: Multithreaded search execution (Ishan Chattopadhyaya, Mark Miller, Christine Poerschke, David Smiley, noble)
+* SOLR-13350, SOLR-17298: Opt-in multithreaded search execution (Ishan Chattopadhyaya, Mark Miller, Christine Poerschke, David Smiley, noble, Gus Heck)
 
 * SOLR-17192: Put an UpdateRequestProcessor-enforced soft-limit on the number of fields allowed in a core. The `NumFieldLimitingUpdateRequestProcessorFactory`
  limit may be adjusted by raising the factory's `maxFields` setting, toggled in and out of "warning-only" mode using the `warnOnly` setting, or disabled entirely
@@ -124,8 +159,18 @@ New Features
 
 * SOLR-10255: Add support for docValues to solr.BinaryField. (Alexey Serba via Mikhail Khludnev, David Smiley)
 
+* SOLR-17276: Prometheus Exporter: now scrapes metrics at a fixed rate instead of delay. (Rafał Harabień)
+
 Improvements
 ---------------------
+* SOLR-10808, SOLR-12963: The Solr schema version has been increased to 1.7.
+ Starting in schema version 1.7, most fields/fieldTypes that support docValues will have them enabled by default.
+ These field types include primitive (Numeric, Date, Bool, String, Enum, UUID), sorting (SortableTextField, SortableBinaryField, CollationField, ICUCollationField) and LatLonPointSpacialField.
+ This behavior can be reverted by setting the 'docValues' parameter for a field or a field type to false, the default for schema versions 1.6 and below.
+ Also in schema version 1.7, all fields/fieldTypes will be unable to be uninverted by default.
+ This behavior can be reverted by setting the 'uninvertible' parameter for a field or a field type to true, the default for schema versions 1.6 and below.
+ (Houston Putman, hossman)
+
 * SOLR-17137: Enable Prometheus exporter to communicate with SSL protected Solr. (Eivind Bergstøl via Eric Pugh)
 
 * SOLR-16921: use -solrUrl to derive the zk host connection for bin/solr zk subcommands (Eric Pugh)
@@ -146,7 +191,7 @@ Improvements
 
 * SOLR-16824: Adopt Linux standard pattern of -- for long option commands, and make all commands "kebab" formatting. I.e -zkHost is now -zk-host. The old parameters
  such as -zkHost continue to be supported in the 9.x line of Solr. -u is now used to specify user credentials everywhere, this only impacts the bin/solr assert
- commands "same user" check which has -u as the short form of --same-user. (Eric Pugh, janhoy, Jason Gerlowski)
+ commands "same user" check which has -u as the short form of --same-user. (Eric Pugh, janhoy, Jason Gerlowski, Christos Malliaridis)
 
 * SOLR-17346: Synchronise stopwords from snowball with those in Lucene (Alastair Porter via Houston Putman)
 
@@ -167,7 +212,7 @@ Optimizations
 
 * SOLR-17099: Do not return spurious tags when fetching metrics from a Solr node to another. (Pierre Salagnac)
 
-* SOLR-17269: Prevent the "Coordinator node" feature from registering synthetic cores in ZooKeeper (Patson Luk)
+* SOLR-17269, SOLR-17386: Prevent the "Coordinator node" feature from registering synthetic cores in ZooKeeper (ellaeln, Patson Luk, David Smiley, Christine Poerschke)
 
 * SOLR-17330: When not set, loadOnStartup defaults to true, which is the default choice for a core. (Pierre Salagnac via Eric Pugh)
 
@@ -212,6 +257,14 @@ Bug Fixes
 
 * SOLR-17369: Fix "flags" usage in FunctionQParser that caused some issues in vectorSimilarity() with BYTE vector constants (hossman)
 
+* SOLR-17337: Display all custom distributed stages in debug output. (Torsten Bøgh Köster, Christine Poerschke)
+
+* SOLR-17394: Detect and handle non-200 HTTP status codes for requests made by IndexFetcher (Jason Gerlowski)
+
+* SOLR-17391: Fixed performance regression of misconfigured threadpools from SOLR-16879 (Solr 9.4).
+ Shard splits and concurrent/large collection backup/restore performance was serial. UpdateLog
+ replay was a little suboptimal in thread usage too. (Pierre Salagnac, Hakan Özler, David Smiley)
+
 Dependency Upgrades
 ---------------------
 (No changes)
@@ -240,6 +293,12 @@ Other Changes
 
 * SOLR-15831: Refactor bin/solr and bin/solr.cmd commands integration with AuthTool, DeleteTool, and PackageTool to delegate arg parsing to Java code. Removed limitation of PackageTool only being executed on an active Solr node. (Eric Pugh)
 
+* SOLR-17322: Once again allow rank queries to use custom TopDocsCollectors that operate on types that extend
+ ScoreDocs (covariant generic types) broken in Solr 9.0. (Stephen Woods via Christine Poerschke)
+
+* SOLR-16996: Update Solr Exporter for Prometheus cli to use commons-cli instead of argparse4j. (Christos Malliaridis via Eric Pugh)
+
+
 ================== 9.6.1 ==================
 Bug Fixes
 ---------------------

solr/bin/solr.cmd

@@ -246,6 +246,7 @@ set FIRST_ARG=%1
 
 IF [%1]==[] goto usage
 
+REM -help is a special case to faciliate folks learning about how to use Solr.
 IF "%1"=="-help" goto run_solrcli
 IF "%1"=="-usage" goto run_solrcli
 IF "%1"=="-h" goto run_solrcli
@@ -293,7 +294,6 @@ IF "%FIRST_ARG%"=="-help" goto run_solrcli
 IF "%FIRST_ARG%"=="-usage" goto run_solrcli
 IF "%FIRST_ARG%"=="-h" goto run_solrcli
 IF "%FIRST_ARG%"=="--help" goto run_solrcli
-IF "%FIRST_ARG%"=="-help" goto run_solrcli
 IF "%FIRST_ARG%"=="/?" goto run_solrcli
 IF "%SCRIPT_CMD%"=="start" goto start_usage
 IF "%SCRIPT_CMD%"=="restart" goto start_usage
@@ -316,7 +316,7 @@ goto done
 @echo -f Start Solr in foreground; default starts Solr in the background
 @echo and sends stdout / stderr to solr-PORT-console.log
 @echo.
-@echo -c or -cloud  Start Solr in SolrCloud mode; if -z not supplied and ZK_HOST not defined in
+@echo -c or --cloud Start Solr in SolrCloud mode; if -z not supplied and ZK_HOST not defined in
 @echo solr.in.cmd, an embedded ZooKeeper instance is started on Solr port+1000,
 @echo such as 9983 if Solr is bound to 8983
 @echo.
@@ -1493,10 +1493,10 @@ IF "!ZK_OP!"=="" (
 set CONNECTION_PARAMS=""
 
 IF "!ZK_OP!"=="" (
- set CONNECTION_PARAMS="-solrUrl !ZK_SOLR_URL!"
+ set CONNECTION_PARAMS="--solr-url !ZK_SOLR_URL!"
 )
 ELSE (
- set CONNECTION_PARAMS="-zkHost ZK_HOST!"
+ set CONNECTION_PARAMS="--zk-host ZK_HOST!"
 )
 
 IF "!ZK_OP!"=="upconfig" (

solr/core/src/java/org/apache/solr/cli/CreateTool.java

@@ -411,7 +411,7 @@ private void printDefaultConfigsetWarningIfNecessary(CommandLine cli) {
  final String configCommand =
  String.format(
  Locale.ROOT,
- "bin/solr config -c %s -solrUrl %s -action set-user-property -property update.autoCreateFields -value false",
+ "bin/solr config -c %s -s %s --action set-user-property --property update.autoCreateFields --value false",
  collectionName,
  solrUrl);
  echo(

solr/core/src/java/org/apache/solr/cli/PackageTool.java

@@ -205,8 +205,9 @@ public void runImpl(CommandLine cli) throws Exception {
  : new String[] {};
  packageManager.undeploy(packageName, collections, cli.hasOption("cluster"));
  } else {
+
  printRed(
- "Either specify -cluster to undeploy cluster level plugins or -collections <list-of-collections> to undeploy collection level plugins");
+ "Either specify --cluster to undeploy cluster level plugins or -collections <list-of-collections> to undeploy collection level plugins");
  }
  break;
  }
@@ -259,7 +260,7 @@ public String getHeader() {
  format(sb, "");
  formatGreen(
  sb,
- "bin/solr package deploy <package-name>[:<version>] [-y] [--update] -collections <comma-separated-collections> [-p <param1>=<val1> -p <param2>=<val2> ...] ");
+ "bin/solr package deploy <package-name>[:<version>] [-y] [--update] --collections <comma-separated-collections> [-p <param1>=<val1> -p <param2>=<val2> ...] ");
  format(
  sb,
  "Bootstraps a previously installed package into the specified collections. It the package accepts parameters for its setup commands, they can be specified (as per package documentation).");
@@ -277,7 +278,7 @@ public String getHeader() {
  format(sb, "Print a list of collections on which a given package has been deployed.");
  format(sb, "");
  formatGreen(
- sb, "bin/solr package undeploy <package-name> -collections <comma-separated-collections>");
+ sb, "bin/solr package undeploy <package-name> --collections <comma-separated-collections>");
  format(sb, "Undeploy a package from specified collection(s)");
  format(sb, "");
  formatGreen(sb, "bin/solr package uninstall <package-name>:<version>");

solr/core/src/java/org/apache/solr/core/CoreContainer.java

@@ -26,6 +26,7 @@
 import static org.apache.solr.common.params.CommonParams.METRICS_PATH;
 import static org.apache.solr.common.params.CommonParams.ZK_PATH;
 import static org.apache.solr.common.params.CommonParams.ZK_STATUS_PATH;
+import static org.apache.solr.search.CpuAllowedLimit.TIMING_CONTEXT;
 import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP;
 
 import com.github.benmanes.caffeine.cache.Interner;
@@ -62,6 +63,7 @@
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
 import org.apache.solr.api.ClusterPluginsSource;
 import org.apache.solr.api.ContainerPluginsRegistry;
 import org.apache.solr.api.JerseyResource;
@@ -155,6 +157,7 @@
 import org.apache.solr.util.OrderedExecutor;
 import org.apache.solr.util.RefCounted;
 import org.apache.solr.util.StartupLoggingUtils;
+import org.apache.solr.util.ThreadCpuTimer;
 import org.apache.solr.util.stats.MetricUtils;
 import org.apache.zookeeper.KeeperException;
 import org.glassfish.hk2.utilities.binding.AbstractBinder;
@@ -237,7 +240,7 @@ public JerseyAppHandlerCache getJerseyAppHandlerCache() {
  ExecutorUtil.newMDCAwareCachedThreadPool(
  new SolrNamedThreadFactory("coreContainerWorkExecutor"));
 
- private final OrderedExecutor replayUpdatesExecutor;
+ private final OrderedExecutor<BytesRef> replayUpdatesExecutor;
 
  protected volatile LogWatcher<?> logging = null;
 
@@ -418,7 +421,7 @@ public CoreContainer(NodeConfig config, CoresLocator locator, boolean asyncSolrC
  this.containerProperties = new Properties(config.getSolrProperties());
  this.asyncSolrCoreLoad = asyncSolrCoreLoad;
  this.replayUpdatesExecutor =
- new OrderedExecutor(
+ new OrderedExecutor<>(
  cfg.getReplayUpdatesThreads(),
  ExecutorUtil.newMDCAwareCachedThreadPool(
  cfg.getReplayUpdatesThreads(), // thread count
@@ -448,7 +451,8 @@ public CoreContainer(NodeConfig config, CoresLocator locator, boolean asyncSolrC
  ExecutorUtil.newMDCAwareFixedThreadPool(
  indexSearcherExecutorThreads, // thread count
  indexSearcherExecutorThreads * 1000, // queue size
- new SolrNamedThreadFactory("searcherCollector"));
+ new SolrNamedThreadFactory("searcherCollector"),
+ () -> ThreadCpuTimer.reset(TIMING_CONTEXT));
  } else {
  this.collectorExecutor = null;
  }
@@ -724,7 +728,7 @@ public Tracer getTracer() {
  return tracer;
  }
 
- public OrderedExecutor getReplayUpdatesExecutor() {
+ public OrderedExecutor<BytesRef> getReplayUpdatesExecutor() {
  return replayUpdatesExecutor;
  }
 
@@ -2054,9 +2058,10 @@ public void reload(String name, UUID coreId) {
 
  DocCollection docCollection = null;
  if (getZkController() != null) {
- docCollection = getZkController().getClusterState().getCollection(cd.getCollectionName());
+ docCollection =
+ getZkController().getClusterState().getCollectionOrNull(cd.getCollectionName());
  // turn off indexing now, before the new core is registered
- if (docCollection.getBool(ZkStateReader.READ_ONLY, false)) {
+ if (docCollection != null && docCollection.getBool(ZkStateReader.READ_ONLY, false)) {
  newCore.readOnly = true;
  }
  }