diff --git a/.asf.yaml b/.asf.yaml index 602feeb246a..57999445552 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -28,6 +28,7 @@ github: branch_9_4: {} branch_9_5: {} branch_9_6: {} + branch_9_7: {} branch_9x: {} protected_tags: diff --git a/.github/workflows/bin-solr-test.yml b/.github/workflows/bin-solr-test.yml index 1f07d4367a6..0754fcd46bd 100644 --- a/.github/workflows/bin-solr-test.yml +++ b/.github/workflows/bin-solr-test.yml @@ -10,6 +10,7 @@ on: - 'solr/bin/**' - 'solr/packaging/**' - 'solr/core/src/java/org/apache/solr/cli/**' + - 'solr/prometheus-exporter/**' jobs: test: @@ -47,4 +48,3 @@ jobs: with: name: logs path: solr/packaging/build/test-output - diff --git a/.github/workflows/gradle-precommit.yml b/.github/workflows/gradle-precommit.yml index 5c477134b2a..12a00fce4b4 100644 --- a/.github/workflows/gradle-precommit.yml +++ b/.github/workflows/gradle-precommit.yml @@ -41,4 +41,4 @@ jobs: - name: Run gradle check (without tests) run: ./gradlew check -x test -Ptask.times=true - - uses: gradle/wrapper-validation-action@v2 + - uses: gradle/wrapper-validation-action@v3 diff --git a/README.md b/README.md index 12fdc47b94b..584bb136cf1 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,9 @@ Solr is the popular, blazing fast open source search platform for all your enterprise, e-commerce, and analytics needs, built on [Apache Lucene](https://lucene.apache.org/). +[![Build Status](https://ci-builds.apache.org/job/Solr/job/Solr-Artifacts-main/badge/icon?subject=Solr%20Artifacts)](https://ci-builds.apache.org/job/Solr/job/Solr-Artifacts-main/) +[![Build Status](https://ci-builds.apache.org/job/Solr/job/Solr-Check-main/badge/icon?subject=Solr%20Check)](https://ci-builds.apache.org/job/Solr/job/Solr-Check-main/) + For a complete description of the Solr project, team composition, source code repositories, and other details, please see the Solr web site at https://solr.apache.org/ diff --git a/dev-tools/scripts/releaseWizard.py b/dev-tools/scripts/releaseWizard.py index a13d7ec2cfe..4520a3c832a 100755 --- a/dev-tools/scripts/releaseWizard.py +++ b/dev-tools/scripts/releaseWizard.py @@ -1123,7 +1123,7 @@ def file_to_string(filename): return f.read().strip() def download_keys(): - download('KEYS', "https://archive.apache.org/dist/solr/KEYS", state.config_path) + download('KEYS', "https://downloads.apache.org/solr/KEYS", state.config_path) def keys_downloaded(): return os.path.exists(os.path.join(state.config_path, "KEYS")) diff --git a/dev-tools/scripts/releaseWizard.yaml b/dev-tools/scripts/releaseWizard.yaml index b604c0cef09..84192113dd5 100644 --- a/dev-tools/scripts/releaseWizard.yaml +++ b/dev-tools/scripts/releaseWizard.yaml @@ -689,7 +689,7 @@ groups: {% if release_type == 'major' -%} . Change name of version `main ({{ release_version_major }}.0)` into `{{ release_version_major }}.0` {%- endif %} - . Create a new (unreleased) version `{{ get_next_version }}` + . Create a new (unreleased) version `{{ release_version_major }}.{{ release_version_minor + 1 }}` types: - major - minor diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 620a57f5a76..798f02162d3 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -120,7 +120,7 @@ minJava = "11" mockito = "5.12.0" morethan-jmhreport = "0.9.0" navsecurity = "0.5.10" -netty = "4.1.111.Final" +netty = "4.1.112.Final" nimbusds-josejwt = "9.30.2" nodegradle-node = "7.0.1" openapi = "7.6.0" diff --git a/gradle/testing/randomization.gradle b/gradle/testing/randomization.gradle index 2e31b881516..9052a31d2a9 100644 --- a/gradle/testing/randomization.gradle +++ b/gradle/testing/randomization.gradle @@ -126,7 +126,6 @@ configure(allprojects.findAll {project -> project.path.startsWith(":solr") }) { plugins.withType(JavaPlugin) { ext { testOptions += [ - [propName: 'solr.directoryFactory', value: "org.apache.solr.core.MockDirectoryFactory", description: "Solr directory factory."], [propName: 'tests.src.home', value: null, description: "See SOLR-14023."], [propName: 'solr.tests.use.numeric.points', value: null, description: "Point implementation to use (true=numerics, false=trie)."], ] diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index 1077e653d98..8328743d647 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -100,10 +100,45 @@ Other Changes * SOLR-17279: Introduce SecurityJson.java file to Test Framework to consolidate setting up authentication in tests. (Rudy Seitz via Eric Pugh) +================== 9.8.0 ================== +New Features +--------------------- +(No changes) + +Improvements +--------------------- +* SOLR-17397: SkipExistingDocumentsProcessor now functions correctly with child documents. (Tim Owens via Eric Pugh) + +Optimizations +--------------------- +* SOLR-14985: Solrj CloudSolrClient with Solr URLs had serious performance regressions (since the + beginning?) in which its collection state cache was not being used, resulting in many extra + requests to Solr for cluster information. (Aparna Suresh, shalin, David Smiley) + +* SOLR-17102: The VersionBucket indexing lock mechanism was replaced with something just as fast yet + that which consumes almost no memory, saving 1MB of memory per SolrCore. (David Smiley) + +* SOLR-17381: Make CLUSTERSTATUS request configurable to improve performance by allowing retrieval of specific information, + reducing unnecessary data fetching. (Aparna Suresh, David Smiley) + +* SOLR-17396: Reduce thread contention in ZkStateReader.getCollectionProperties(). (Aparna Suresh, David Smiley, Paul McArthur) + +Bug Fixes +--------------------- +(No changes) + +Dependency Upgrades +--------------------- +(No changes) + +Other Changes +--------------------- +(No changes) + ================== 9.7.0 ================== New Features --------------------- -* SOLR-13350: Multithreaded search execution (Ishan Chattopadhyaya, Mark Miller, Christine Poerschke, David Smiley, noble) +* SOLR-13350, SOLR-17298: Opt-in multithreaded search execution (Ishan Chattopadhyaya, Mark Miller, Christine Poerschke, David Smiley, noble, Gus Heck) * SOLR-17192: Put an UpdateRequestProcessor-enforced soft-limit on the number of fields allowed in a core. The `NumFieldLimitingUpdateRequestProcessorFactory` limit may be adjusted by raising the factory's `maxFields` setting, toggled in and out of "warning-only" mode using the `warnOnly` setting, or disabled entirely @@ -124,8 +159,18 @@ New Features * SOLR-10255: Add support for docValues to solr.BinaryField. (Alexey Serba via Mikhail Khludnev, David Smiley) +* SOLR-17276: Prometheus Exporter: now scrapes metrics at a fixed rate instead of delay. (Rafał Harabień) + Improvements --------------------- +* SOLR-10808, SOLR-12963: The Solr schema version has been increased to 1.7. + Starting in schema version 1.7, most fields/fieldTypes that support docValues will have them enabled by default. + These field types include primitive (Numeric, Date, Bool, String, Enum, UUID), sorting (SortableTextField, SortableBinaryField, CollationField, ICUCollationField) and LatLonPointSpacialField. + This behavior can be reverted by setting the 'docValues' parameter for a field or a field type to false, the default for schema versions 1.6 and below. + Also in schema version 1.7, all fields/fieldTypes will be unable to be uninverted by default. + This behavior can be reverted by setting the 'uninvertible' parameter for a field or a field type to true, the default for schema versions 1.6 and below. + (Houston Putman, hossman) + * SOLR-17137: Enable Prometheus exporter to communicate with SSL protected Solr. (Eivind Bergstøl via Eric Pugh) * SOLR-16921: use -solrUrl to derive the zk host connection for bin/solr zk subcommands (Eric Pugh) @@ -146,7 +191,7 @@ Improvements * SOLR-16824: Adopt Linux standard pattern of -- for long option commands, and make all commands "kebab" formatting. I.e -zkHost is now -zk-host. The old parameters such as -zkHost continue to be supported in the 9.x line of Solr. -u is now used to specify user credentials everywhere, this only impacts the bin/solr assert - commands "same user" check which has -u as the short form of --same-user. (Eric Pugh, janhoy, Jason Gerlowski) + commands "same user" check which has -u as the short form of --same-user. (Eric Pugh, janhoy, Jason Gerlowski, Christos Malliaridis) * SOLR-17346: Synchronise stopwords from snowball with those in Lucene (Alastair Porter via Houston Putman) @@ -167,7 +212,7 @@ Optimizations * SOLR-17099: Do not return spurious tags when fetching metrics from a Solr node to another. (Pierre Salagnac) -* SOLR-17269: Prevent the "Coordinator node" feature from registering synthetic cores in ZooKeeper (Patson Luk) +* SOLR-17269, SOLR-17386: Prevent the "Coordinator node" feature from registering synthetic cores in ZooKeeper (ellaeln, Patson Luk, David Smiley, Christine Poerschke) * SOLR-17330: When not set, loadOnStartup defaults to true, which is the default choice for a core. (Pierre Salagnac via Eric Pugh) @@ -212,6 +257,14 @@ Bug Fixes * SOLR-17369: Fix "flags" usage in FunctionQParser that caused some issues in vectorSimilarity() with BYTE vector constants (hossman) +* SOLR-17337: Display all custom distributed stages in debug output. (Torsten Bøgh Köster, Christine Poerschke) + +* SOLR-17394: Detect and handle non-200 HTTP status codes for requests made by IndexFetcher (Jason Gerlowski) + +* SOLR-17391: Fixed performance regression of misconfigured threadpools from SOLR-16879 (Solr 9.4). + Shard splits and concurrent/large collection backup/restore performance was serial. UpdateLog + replay was a little suboptimal in thread usage too. (Pierre Salagnac, Hakan Özler, David Smiley) + Dependency Upgrades --------------------- (No changes) @@ -240,6 +293,12 @@ Other Changes * SOLR-15831: Refactor bin/solr and bin/solr.cmd commands integration with AuthTool, DeleteTool, and PackageTool to delegate arg parsing to Java code. Removed limitation of PackageTool only being executed on an active Solr node. (Eric Pugh) +* SOLR-17322: Once again allow rank queries to use custom TopDocsCollectors that operate on types that extend + ScoreDocs (covariant generic types) broken in Solr 9.0. (Stephen Woods via Christine Poerschke) + +* SOLR-16996: Update Solr Exporter for Prometheus cli to use commons-cli instead of argparse4j. (Christos Malliaridis via Eric Pugh) + + ================== 9.6.1 ================== Bug Fixes --------------------- diff --git a/solr/bin/solr.cmd b/solr/bin/solr.cmd index 0eb25adce14..58fb4e6599e 100755 --- a/solr/bin/solr.cmd +++ b/solr/bin/solr.cmd @@ -246,6 +246,7 @@ set FIRST_ARG=%1 IF [%1]==[] goto usage +REM -help is a special case to faciliate folks learning about how to use Solr. IF "%1"=="-help" goto run_solrcli IF "%1"=="-usage" goto run_solrcli IF "%1"=="-h" goto run_solrcli @@ -293,7 +294,6 @@ IF "%FIRST_ARG%"=="-help" goto run_solrcli IF "%FIRST_ARG%"=="-usage" goto run_solrcli IF "%FIRST_ARG%"=="-h" goto run_solrcli IF "%FIRST_ARG%"=="--help" goto run_solrcli -IF "%FIRST_ARG%"=="-help" goto run_solrcli IF "%FIRST_ARG%"=="/?" goto run_solrcli IF "%SCRIPT_CMD%"=="start" goto start_usage IF "%SCRIPT_CMD%"=="restart" goto start_usage @@ -316,7 +316,7 @@ goto done @echo -f Start Solr in foreground; default starts Solr in the background @echo and sends stdout / stderr to solr-PORT-console.log @echo. -@echo -c or -cloud Start Solr in SolrCloud mode; if -z not supplied and ZK_HOST not defined in +@echo -c or --cloud Start Solr in SolrCloud mode; if -z not supplied and ZK_HOST not defined in @echo solr.in.cmd, an embedded ZooKeeper instance is started on Solr port+1000, @echo such as 9983 if Solr is bound to 8983 @echo. @@ -1493,10 +1493,10 @@ IF "!ZK_OP!"=="" ( set CONNECTION_PARAMS="" IF "!ZK_OP!"=="" ( - set CONNECTION_PARAMS="-solrUrl !ZK_SOLR_URL!" + set CONNECTION_PARAMS="--solr-url !ZK_SOLR_URL!" ) ELSE ( - set CONNECTION_PARAMS="-zkHost ZK_HOST!" + set CONNECTION_PARAMS="--zk-host ZK_HOST!" ) IF "!ZK_OP!"=="upconfig" ( diff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/cli/CreateTool.java index f828ac77423..61031ae5cc9 100644 --- a/solr/core/src/java/org/apache/solr/cli/CreateTool.java +++ b/solr/core/src/java/org/apache/solr/cli/CreateTool.java @@ -411,7 +411,7 @@ private void printDefaultConfigsetWarningIfNecessary(CommandLine cli) { final String configCommand = String.format( Locale.ROOT, - "bin/solr config -c %s -solrUrl %s -action set-user-property -property update.autoCreateFields -value false", + "bin/solr config -c %s -s %s --action set-user-property --property update.autoCreateFields --value false", collectionName, solrUrl); echo( diff --git a/solr/core/src/java/org/apache/solr/cli/PackageTool.java b/solr/core/src/java/org/apache/solr/cli/PackageTool.java index a8fdd5afd18..c3f4a3a9e96 100644 --- a/solr/core/src/java/org/apache/solr/cli/PackageTool.java +++ b/solr/core/src/java/org/apache/solr/cli/PackageTool.java @@ -205,8 +205,9 @@ public void runImpl(CommandLine cli) throws Exception { : new String[] {}; packageManager.undeploy(packageName, collections, cli.hasOption("cluster")); } else { + printRed( - "Either specify -cluster to undeploy cluster level plugins or -collections to undeploy collection level plugins"); + "Either specify --cluster to undeploy cluster level plugins or -collections to undeploy collection level plugins"); } break; } @@ -259,7 +260,7 @@ public String getHeader() { format(sb, ""); formatGreen( sb, - "bin/solr package deploy [:] [-y] [--update] -collections [-p = -p = ...] "); + "bin/solr package deploy [:] [-y] [--update] --collections [-p = -p = ...] "); format( sb, "Bootstraps a previously installed package into the specified collections. It the package accepts parameters for its setup commands, they can be specified (as per package documentation)."); @@ -277,7 +278,7 @@ public String getHeader() { format(sb, "Print a list of collections on which a given package has been deployed."); format(sb, ""); formatGreen( - sb, "bin/solr package undeploy -collections "); + sb, "bin/solr package undeploy --collections "); format(sb, "Undeploy a package from specified collection(s)"); format(sb, ""); formatGreen(sb, "bin/solr package uninstall :"); diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java index 174deadd4fb..57370af1f18 100644 --- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java +++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java @@ -26,6 +26,7 @@ import static org.apache.solr.common.params.CommonParams.METRICS_PATH; import static org.apache.solr.common.params.CommonParams.ZK_PATH; import static org.apache.solr.common.params.CommonParams.ZK_STATUS_PATH; +import static org.apache.solr.search.CpuAllowedLimit.TIMING_CONTEXT; import static org.apache.solr.security.AuthenticationPlugin.AUTHENTICATION_PLUGIN_PROP; import com.github.benmanes.caffeine.cache.Interner; @@ -62,6 +63,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.apache.solr.api.ClusterPluginsSource; import org.apache.solr.api.ContainerPluginsRegistry; import org.apache.solr.api.JerseyResource; @@ -155,6 +157,7 @@ import org.apache.solr.util.OrderedExecutor; import org.apache.solr.util.RefCounted; import org.apache.solr.util.StartupLoggingUtils; +import org.apache.solr.util.ThreadCpuTimer; import org.apache.solr.util.stats.MetricUtils; import org.apache.zookeeper.KeeperException; import org.glassfish.hk2.utilities.binding.AbstractBinder; @@ -237,7 +240,7 @@ public JerseyAppHandlerCache getJerseyAppHandlerCache() { ExecutorUtil.newMDCAwareCachedThreadPool( new SolrNamedThreadFactory("coreContainerWorkExecutor")); - private final OrderedExecutor replayUpdatesExecutor; + private final OrderedExecutor replayUpdatesExecutor; protected volatile LogWatcher logging = null; @@ -418,7 +421,7 @@ public CoreContainer(NodeConfig config, CoresLocator locator, boolean asyncSolrC this.containerProperties = new Properties(config.getSolrProperties()); this.asyncSolrCoreLoad = asyncSolrCoreLoad; this.replayUpdatesExecutor = - new OrderedExecutor( + new OrderedExecutor<>( cfg.getReplayUpdatesThreads(), ExecutorUtil.newMDCAwareCachedThreadPool( cfg.getReplayUpdatesThreads(), // thread count @@ -448,7 +451,8 @@ public CoreContainer(NodeConfig config, CoresLocator locator, boolean asyncSolrC ExecutorUtil.newMDCAwareFixedThreadPool( indexSearcherExecutorThreads, // thread count indexSearcherExecutorThreads * 1000, // queue size - new SolrNamedThreadFactory("searcherCollector")); + new SolrNamedThreadFactory("searcherCollector"), + () -> ThreadCpuTimer.reset(TIMING_CONTEXT)); } else { this.collectorExecutor = null; } @@ -724,7 +728,7 @@ public Tracer getTracer() { return tracer; } - public OrderedExecutor getReplayUpdatesExecutor() { + public OrderedExecutor getReplayUpdatesExecutor() { return replayUpdatesExecutor; } @@ -2054,9 +2058,10 @@ public void reload(String name, UUID coreId) { DocCollection docCollection = null; if (getZkController() != null) { - docCollection = getZkController().getClusterState().getCollection(cd.getCollectionName()); + docCollection = + getZkController().getClusterState().getCollectionOrNull(cd.getCollectionName()); // turn off indexing now, before the new core is registered - if (docCollection.getBool(ZkStateReader.READ_ONLY, false)) { + if (docCollection != null && docCollection.getBool(ZkStateReader.READ_ONLY, false)) { newCore.readOnly = true; } } diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 2b040f14f37..4c3d09980ca 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -766,29 +766,16 @@ public SolrCore reload(ConfigSet coreConfig) throws IOException { // only one reload at a time synchronized (getUpdateHandler().getSolrCoreState().getReloadLock()) { solrCoreState.increfSolrCoreState(); - final SolrCore currentCore; - if (!getNewIndexDir().equals(getIndexDir())) { - // the directory is changing, don't pass on state - currentCore = null; - } else { - currentCore = this; - } + + // if directory is changing, then don't pass on state + boolean cloneCoreState = getNewIndexDir().equals(getIndexDir()); boolean success = false; SolrCore core = null; try { CoreDescriptor cd = new CoreDescriptor(name, getCoreDescriptor()); cd.loadExtraProperties(); // Reload the extra properties - core = - new SolrCore( - coreContainer, - cd, - coreConfig, - getDataDir(), - updateHandler, - solrDelPolicy, - currentCore, - true); + core = cloneForReloadCore(cd, coreConfig, cloneCoreState); // we open a new IndexWriter to pick up the latest config core.getUpdateHandler().getSolrCoreState().newIndexWriter(core, false); @@ -805,6 +792,24 @@ public SolrCore reload(ConfigSet coreConfig) throws IOException { } } + /** + * Clones the current core for core reload, with the provided CoreDescriptor and ConfigSet. + * + * @return the cloned core to be used for {@link SolrCore#reload} + */ + protected SolrCore cloneForReloadCore( + CoreDescriptor newCoreDescriptor, ConfigSet newCoreConfig, boolean cloneCoreState) { + return new SolrCore( + coreContainer, + newCoreDescriptor, + newCoreConfig, + getDataDir(), + updateHandler, + solrDelPolicy, + cloneCoreState ? this : null, + true); + } + private DirectoryFactory initDirectoryFactory() { return DirectoryFactory.loadDirectoryFactory( solrConfig, coreContainer, coreMetricManager.getRegistryName()); @@ -1054,7 +1059,7 @@ public CoreContainer getCoreContainer() { this(coreContainer, cd, configSet, null, null, null, null, false); } - private SolrCore( + protected SolrCore( CoreContainer coreContainer, CoreDescriptor coreDescriptor, ConfigSet configSet, diff --git a/solr/core/src/java/org/apache/solr/core/SyntheticSolrCore.java b/solr/core/src/java/org/apache/solr/core/SyntheticSolrCore.java index e97c5780d54..db30474ffd4 100644 --- a/solr/core/src/java/org/apache/solr/core/SyntheticSolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SyntheticSolrCore.java @@ -22,6 +22,7 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.rest.RestManager; +import org.apache.solr.update.UpdateHandler; /** * A synthetic core that is created only in memory and not registered against Zookeeper. @@ -34,7 +35,20 @@ */ public class SyntheticSolrCore extends SolrCore { public SyntheticSolrCore(CoreContainer coreContainer, CoreDescriptor cd, ConfigSet configSet) { - super(coreContainer, cd, configSet); + this(coreContainer, cd, configSet, null, null, null, null, false); + } + + public SyntheticSolrCore( + CoreContainer coreContainer, + CoreDescriptor coreDescriptor, + ConfigSet configSet, + String dataDir, + UpdateHandler updateHandler, + IndexDeletionPolicyWrapper delPolicy, + SolrCore prev, + boolean reload) { + super( + coreContainer, coreDescriptor, configSet, dataDir, updateHandler, delPolicy, prev, reload); } public static SyntheticSolrCore createAndRegisterCore( @@ -72,4 +86,18 @@ protected RestManager initRestManager() throws SolrException { // We do not expect RestManager ops on Coordinator Nodes return new RestManager(); } + + @Override + protected SyntheticSolrCore cloneForReloadCore( + CoreDescriptor newCoreDescriptor, ConfigSet newCoreConfig, boolean cloneCurrentState) { + return new SyntheticSolrCore( + getCoreContainer(), + newCoreDescriptor, + newCoreConfig, + getDataDir(), + getUpdateHandler(), + getDeletionPolicy(), + cloneCurrentState ? this : null, + true); + } } diff --git a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java index 662a94943c9..d439f7196cd 100644 --- a/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java +++ b/solr/core/src/java/org/apache/solr/handler/IndexFetcher.java @@ -71,6 +71,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Properties; import java.util.Set; import java.util.concurrent.CountDownLatch; @@ -1988,16 +1989,38 @@ private FastInputStream getStream() throws IOException { req.setBasePath(leaderBaseUrl); if (useExternalCompression) req.addHeader("Accept-Encoding", "gzip"); response = solrClient.request(req, leaderCoreName); + final var responseStatus = (Integer) response.get("responseStatus"); is = (InputStream) response.get("stream"); + + if (responseStatus != 200) { + final var errorMsg = + String.format( + Locale.ROOT, + "Unexpected status code [%d] when downloading file [%s].", + responseStatus, + fileName); + closeStreamAndThrowIOE(is, errorMsg, Optional.empty()); + } + if (useInternalCompression) { is = new InflaterInputStream(is); } return new FastInputStream(is); } catch (Exception e) { - // close stream on error - IOUtils.closeQuietly(is); - throw new IOException("Could not download file '" + fileName + "'", e); + closeStreamAndThrowIOE(is, "Could not download file '" + fileName + "'", Optional.of(e)); + } + + // Unreachable b/c closeStreamAndThrowIOE will always throw + return null; + } + + private void closeStreamAndThrowIOE( + InputStream is, String exceptionMessage, Optional e) throws IOException { + IOUtils.closeQuietly(is); + if (e.isPresent()) { + throw new IOException(exceptionMessage, e.get()); } + throw new IOException(exceptionMessage); } } diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java index ff020ddebd3..d85b939fbb5 100644 --- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java +++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java @@ -43,8 +43,9 @@ import org.apache.solr.metrics.SolrMetricsContext; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; -import org.apache.solr.request.SolrRequestInfo; import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.CpuAllowedLimit; +import org.apache.solr.search.QueryLimits; import org.apache.solr.search.SyntaxError; import org.apache.solr.security.PermissionNameProvider; import org.apache.solr.update.processor.DistributedUpdateProcessor; @@ -62,6 +63,7 @@ public abstract class RequestHandlerBase ApiSupport, PermissionNameProvider { + public static final String REQUEST_CPU_TIMER_CONTEXT = "publishCpuTime"; protected NamedList initArgs = null; protected SolrParams defaults; protected SolrParams appends; @@ -217,12 +219,8 @@ public abstract void handleRequestBody(SolrQueryRequest req, SolrQueryResponse r @Override public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) { - ThreadCpuTimer threadCpuTimer = null; if (publishCpuTime) { - threadCpuTimer = - SolrRequestInfo.getRequestInfo() == null - ? new ThreadCpuTimer() - : SolrRequestInfo.getRequestInfo().getThreadCpuTimer(); + ThreadCpuTimer.beginContext(REQUEST_CPU_TIMER_CONTEXT); } HandlerMetrics metrics = getMetricsForThisRequest(req); metrics.requests.inc(); @@ -246,21 +244,36 @@ public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) { processErrorMetricsOnException(normalized, metrics); rsp.setException(normalized); } finally { - long elapsed = timer.stop(); - metrics.totalTime.inc(elapsed); - - if (publishCpuTime && threadCpuTimer != null) { - Optional cpuTime = threadCpuTimer.getElapsedCpuMs(); - if (cpuTime.isPresent()) { - // add CPU_TIME if not already added by SearchHandler - NamedList header = rsp.getResponseHeader(); - if (header != null) { - if (header.get(ThreadCpuTimer.CPU_TIME) == null) { - header.add(ThreadCpuTimer.CPU_TIME, cpuTime.get()); + try { + long elapsed = timer.stop(); + metrics.totalTime.inc(elapsed); + + if (publishCpuTime) { + Optional cpuTime = ThreadCpuTimer.readMSandReset(REQUEST_CPU_TIMER_CONTEXT); + if (QueryLimits.getCurrentLimits().isLimitsEnabled()) { + // prefer the value from the limit if available to avoid confusing users with trivial + // differences. Not fond of the spotless formatting here... + cpuTime = + Optional.ofNullable( + (Long) + QueryLimits.getCurrentLimits() + .currentLimitValueFor(CpuAllowedLimit.class) + .orElse(cpuTime.orElse(null))); + } + if (cpuTime.isPresent()) { + // add CPU_TIME if not already added by SearchHandler + NamedList header = rsp.getResponseHeader(); + if (header != null) { + if (header.get(ThreadCpuTimer.CPU_TIME) == null) { + header.add(ThreadCpuTimer.CPU_TIME, cpuTime.get()); + } } + rsp.addToLog(ThreadCpuTimer.LOCAL_CPU_TIME, cpuTime.get()); } - rsp.addToLog(ThreadCpuTimer.LOCAL_CPU_TIME, cpuTime.get()); } + } finally { + // whatever happens be sure to clear things out at end of request. + ThreadCpuTimer.reset(); } } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java index f897aea129c..6c5998d17a2 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java @@ -34,9 +34,9 @@ import org.apache.solr.common.cloud.PerReplicaStates; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.ShardParams; +import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.Utils; @@ -44,9 +44,14 @@ public class ClusterStatus { private final ZkStateReader zkStateReader; - private final ZkNodeProps message; + private final SolrParams solrParams; private final String collection; // maybe null + public static final String INCLUDE_ALL = "includeAll"; + public static final String LIVENODES_PROP = "liveNodes"; + public static final String CLUSTER_PROP = "clusterProperties"; + public static final String ALIASES_PROP = "aliases"; + /** Shard / collection health state. */ public enum Health { /** All replicas up, leader exists. */ @@ -89,16 +94,72 @@ public static Health combine(Collection states) { } } - public ClusterStatus(ZkStateReader zkStateReader, ZkNodeProps props) { + public ClusterStatus(ZkStateReader zkStateReader, SolrParams params) { this.zkStateReader = zkStateReader; - this.message = props; - collection = props.getStr(ZkStateReader.COLLECTION_PROP); + this.solrParams = params; + collection = params.get(ZkStateReader.COLLECTION_PROP); } public void getClusterStatus(NamedList results) throws KeeperException, InterruptedException { + NamedList clusterStatus = new SimpleOrderedMap<>(); + + boolean includeAll = solrParams.getBool(INCLUDE_ALL, true); + boolean withLiveNodes = solrParams.getBool(LIVENODES_PROP, includeAll); + boolean withClusterProperties = solrParams.getBool(CLUSTER_PROP, includeAll); + boolean withRoles = solrParams.getBool(ZkStateReader.ROLES_PROP, includeAll); + boolean withCollection = includeAll || (collection != null); + boolean withAliases = solrParams.getBool(ALIASES_PROP, includeAll); + + List liveNodes = null; + if (withLiveNodes || collection != null) { + liveNodes = + zkStateReader.getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true); + // add live_nodes + if (withLiveNodes) clusterStatus.add("live_nodes", liveNodes); + } + + Aliases aliases = null; + if (withCollection || withAliases) { + aliases = zkStateReader.getAliases(); + } + + if (withCollection) { + assert liveNodes != null; + fetchClusterStatusForCollOrAlias(clusterStatus, liveNodes, aliases); + } + + if (withAliases) { + addAliasMap(aliases, clusterStatus); + } + + if (withClusterProperties) { + Map clusterProps = zkStateReader.getClusterProperties(); + if (clusterProps == null) { + clusterProps = Collections.emptyMap(); + } + clusterStatus.add("properties", clusterProps); + } + + // add the roles map + if (withRoles) { + Map roles = Collections.emptyMap(); + if (zkStateReader.getZkClient().exists(ZkStateReader.ROLES, true)) { + roles = + (Map) + Utils.fromJSON( + zkStateReader.getZkClient().getData(ZkStateReader.ROLES, null, null, true)); + } + clusterStatus.add("roles", roles); + } + + results.add("cluster", clusterStatus); + } + + private void fetchClusterStatusForCollOrAlias( + NamedList clusterStatus, List liveNodes, Aliases aliases) { + // read aliases - Aliases aliases = zkStateReader.getAliases(); Map> collectionVsAliases = new HashMap<>(); Map> aliasVsCollections = aliases.getCollectionAliasListMap(); for (Map.Entry> entry : aliasVsCollections.entrySet()) { @@ -112,18 +173,10 @@ public void getClusterStatus(NamedList results) } } - Map roles = null; - if (zkStateReader.getZkClient().exists(ZkStateReader.ROLES, true)) { - roles = - (Map) - Utils.fromJSON( - zkStateReader.getZkClient().getData(ZkStateReader.ROLES, null, null, true)); - } - ClusterState clusterState = zkStateReader.getClusterState(); - String routeKey = message.getStr(ShardParams._ROUTE_); - String shard = message.getStr(ZkStateReader.SHARD_ID_PROP); + String routeKey = solrParams.get(ShardParams._ROUTE_); + String shard = solrParams.get(ZkStateReader.SHARD_ID_PROP); Map collectionsMap = null; if (collection == null) { @@ -139,6 +192,7 @@ public void getClusterStatus(NamedList results) if (didNotFindCollection && isAlias) { // In this case this.collection is an alias name not a collection // get all collections and filter out collections not in the alias + // clusterState.getCollectionsMap() should be replaced with an inexpensive call collectionsMap = clusterState.getCollectionsMap().entrySet().stream() .filter((entry) -> aliasVsCollections.get(collection).contains(entry.getKey())) @@ -191,43 +245,25 @@ public void getClusterStatus(NamedList results) } String configName = clusterStateCollection.getConfigName(); collectionStatus.put("configName", configName); - if (message.getBool("prs", false) && clusterStateCollection.isPerReplicaState()) { + if (solrParams.getBool("prs", false) && clusterStateCollection.isPerReplicaState()) { PerReplicaStates prs = clusterStateCollection.getPerReplicaStates(); collectionStatus.put("PRS", prs); } collectionProps.add(name, collectionStatus); } - List liveNodes = - zkStateReader.getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true); - // now we need to walk the collectionProps tree to cross-check replica state with live nodes crossCheckReplicaStateWithLiveNodes(liveNodes, collectionProps); - NamedList clusterStatus = new SimpleOrderedMap<>(); clusterStatus.add("collections", collectionProps); + } - // read cluster properties - Map clusterProps = zkStateReader.getClusterProperties(); - if (clusterProps != null && !clusterProps.isEmpty()) { - clusterStatus.add("properties", clusterProps); - } - + private void addAliasMap(Aliases aliases, NamedList clusterStatus) { // add the alias map too Map collectionAliasMap = aliases.getCollectionAliasMap(); // comma delim if (!collectionAliasMap.isEmpty()) { clusterStatus.add("aliases", collectionAliasMap); } - - // add the roles map - if (roles != null) { - clusterStatus.add("roles", roles); - } - - // add live_nodes - clusterStatus.add("live_nodes", liveNodes); - - results.add("cluster", clusterStatus); } /** diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java index 485670a4421..7ee6da6a0c1 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java @@ -100,7 +100,6 @@ import static org.apache.solr.common.params.CommonParams.VALUE_LONG; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION; import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY; -import static org.apache.solr.common.params.ShardParams._ROUTE_; import static org.apache.solr.common.util.StrUtils.formatString; import java.lang.invoke.MethodHandles; @@ -975,10 +974,7 @@ public Map execute( CLUSTERSTATUS_OP( CLUSTERSTATUS, (req, rsp, h) -> { - Map all = - copy(req.getParams(), null, COLLECTION_PROP, SHARD_ID_PROP, _ROUTE_, "prs"); - new ClusterStatus( - h.coreContainer.getZkController().getZkStateReader(), new ZkNodeProps(all)) + new ClusterStatus(h.coreContainer.getZkController().getZkStateReader(), req.getParams()) .getClusterStatus(rsp.getValues()); return null; }), diff --git a/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java b/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java index 0565290cf06..001b6c3338a 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java @@ -19,6 +19,7 @@ import static org.apache.solr.common.params.CommonParams.FQ; import static org.apache.solr.common.params.CommonParams.JSON; +import com.google.common.annotations.VisibleForTesting; import java.io.IOException; import java.lang.reflect.Array; import java.util.ArrayList; @@ -181,16 +182,28 @@ public void modifyRequest(ResponseBuilder rb, SearchComponent who, ShardRequest } } + @VisibleForTesting + protected String getDistributedStageName(int stage) { + String stageName = stages.get(stage); + + if (stageName == null) { + stageName = "STAGE_" + Integer.toString(stage); + } + + return stageName; + } + @Override public void handleResponses(ResponseBuilder rb, ShardRequest sreq) { if (rb.isDebugTrack() && rb.isDistrib && !rb.finished.isEmpty()) { @SuppressWarnings("unchecked") NamedList stageList = (NamedList) - ((NamedList) rb.getDebugInfo().get("track")).get(stages.get(rb.stage)); + ((NamedList) rb.getDebugInfo().get("track")) + .get(getDistributedStageName(rb.stage)); if (stageList == null) { stageList = new SimpleOrderedMap<>(); - rb.addDebug(stageList, "track", stages.get(rb.stage)); + rb.addDebug(stageList, "track", getDistributedStageName(rb.stage)); } for (ShardResponse response : sreq.responses) { stageList.add(response.getShard(), getTrackResponse(response)); diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java index b7c080ec3b5..3c558feffc3 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java @@ -371,7 +371,7 @@ public void process(ResponseBuilder rb) throws IOException { return; } - final boolean multiThreaded = params.getBool("multiThreaded", true); + final boolean multiThreaded = params.getBool(CommonParams.MULTI_THREADED, false); // -1 as flag if not set. long timeAllowed = params.getLong(CommonParams.TIME_ALLOWED, -1L); diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/SolrMetric.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/SolrMetric.java index 4668e615c85..e95316c0565 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/SolrMetric.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/SolrMetric.java @@ -25,9 +25,9 @@ /** * Base class is a wrapper to categorize and export {@link com.codahale.metrics.Metric} to {@link * io.prometheus.metrics.model.snapshots.DataPointSnapshot} and register to a {@link - * SolrPrometheusExporter}. {@link com.codahale.metrics.MetricRegistry} does not support tags unlike - * prometheus. Metrics registered to the registry need to be parsed out from the metric name to be - * exported to {@link io.prometheus.metrics.model.snapshots.DataPointSnapshot} + * SolrPrometheusFormatter}. {@link com.codahale.metrics.MetricRegistry} does not support tags + * unlike prometheus. Metrics registered to the registry need to be parsed out from the metric name + * to be exported to {@link io.prometheus.metrics.model.snapshots.DataPointSnapshot} */ public abstract class SolrMetric { public Metric dropwizardMetric; @@ -49,7 +49,7 @@ public SolrMetric(Metric dropwizardMetric, String metricName) { /* * Export metric to Prometheus with labels */ - public abstract void toPrometheus(SolrPrometheusExporter exporter); + public abstract void toPrometheus(SolrPrometheusFormatter formatter); public Labels getLabels() { return Labels.of(new ArrayList<>(labels.keySet()), new ArrayList<>(labels.values())); diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/SolrNoOpMetric.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/SolrNoOpMetric.java index 0be1761dc51..4d3dcbe87ab 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/SolrNoOpMetric.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/SolrNoOpMetric.java @@ -25,5 +25,5 @@ public SolrMetric parseLabels() { } @Override - public void toPrometheus(SolrPrometheusExporter exporter) {} + public void toPrometheus(SolrPrometheusFormatter formatter) {} } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/SolrPrometheusExporter.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/SolrPrometheusFormatter.java similarity index 98% rename from solr/core/src/java/org/apache/solr/metrics/prometheus/SolrPrometheusExporter.java rename to solr/core/src/java/org/apache/solr/metrics/prometheus/SolrPrometheusFormatter.java index 6fbb4c0d8ed..8c1ba8652ab 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/SolrPrometheusExporter.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/SolrPrometheusFormatter.java @@ -31,15 +31,15 @@ import java.util.Map; /** - * Base class for all {@link SolrPrometheusExporter} holding {@link MetricSnapshot}s. Can export + * Base class for all {@link SolrPrometheusFormatter} holding {@link MetricSnapshot}s. Can export * {@link com.codahale.metrics.Metric} to {@link MetricSnapshot} to be outputted for {@link * org.apache.solr.response.PrometheusResponseWriter} */ -public abstract class SolrPrometheusExporter { +public abstract class SolrPrometheusFormatter { protected final Map> metricCounters; protected final Map> metricGauges; - public SolrPrometheusExporter() { + public SolrPrometheusFormatter() { this.metricCounters = new HashMap<>(); this.metricGauges = new HashMap<>(); } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/PrometheusCoreExporterInfo.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/PrometheusCoreFormatterInfo.java similarity index 96% rename from solr/core/src/java/org/apache/solr/metrics/prometheus/core/PrometheusCoreExporterInfo.java rename to solr/core/src/java/org/apache/solr/metrics/prometheus/core/PrometheusCoreFormatterInfo.java index 32ed8104276..51f5d9765d9 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/PrometheusCoreExporterInfo.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/PrometheusCoreFormatterInfo.java @@ -18,7 +18,7 @@ import java.util.regex.Pattern; -public interface PrometheusCoreExporterInfo { +public interface PrometheusCoreFormatterInfo { /** Category of prefix Solr Core dropwizard handler metric names */ enum CoreCategory { ADMIN, diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreCacheMetric.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreCacheMetric.java index 6c71664102e..e8352d3575c 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreCacheMetric.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreCacheMetric.java @@ -18,7 +18,7 @@ import com.codahale.metrics.Gauge; import com.codahale.metrics.Metric; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; /** Dropwizard metrics of name CACHE.* */ public class SolrCoreCacheMetric extends SolrCoreMetric { @@ -44,9 +44,9 @@ public SolrCoreMetric parseLabels() { } @Override - public void toPrometheus(SolrPrometheusExporter exporter) { + public void toPrometheus(SolrPrometheusFormatter formatter) { if (dropwizardMetric instanceof Gauge) { - exporter.exportGauge(CORE_CACHE_SEARCHER_METRICS, (Gauge) dropwizardMetric, getLabels()); + formatter.exportGauge(CORE_CACHE_SEARCHER_METRICS, (Gauge) dropwizardMetric, getLabels()); } } } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreHandlerMetric.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreHandlerMetric.java index 3a05fbeb5b0..af9641729a9 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreHandlerMetric.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreHandlerMetric.java @@ -21,7 +21,7 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.Metric; import com.codahale.metrics.Timer; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; /** Dropwizard metrics of name ADMIN/QUERY/UPDATE/REPLICATION.* */ public class SolrCoreHandlerMetric extends SolrCoreMetric { @@ -53,26 +53,26 @@ public SolrCoreMetric parseLabels() { } @Override - public void toPrometheus(SolrPrometheusExporter exporter) { + public void toPrometheus(SolrPrometheusFormatter formatter) { if (dropwizardMetric instanceof Meter) { - exporter.exportMeter(CORE_REQUESTS_TOTAL, (Meter) dropwizardMetric, getLabels()); + formatter.exportMeter(CORE_REQUESTS_TOTAL, (Meter) dropwizardMetric, getLabels()); } else if (dropwizardMetric instanceof Counter) { if (metricName.endsWith("requests")) { - exporter.exportCounter(CORE_REQUESTS_TOTAL, (Counter) dropwizardMetric, getLabels()); + formatter.exportCounter(CORE_REQUESTS_TOTAL, (Counter) dropwizardMetric, getLabels()); } else if (metricName.endsWith("totalTime")) { // Do not need type label for total time labels.remove("type"); - exporter.exportCounter(CORE_REQUESTS_TOTAL_TIME, (Counter) dropwizardMetric, getLabels()); + formatter.exportCounter(CORE_REQUESTS_TOTAL_TIME, (Counter) dropwizardMetric, getLabels()); } } else if (dropwizardMetric instanceof Gauge) { if (!metricName.endsWith("handlerStart")) { - exporter.exportGauge( + formatter.exportGauge( CORE_REQUESTS_UPDATE_HANDLER, (Gauge) dropwizardMetric, getLabels()); } } else if (dropwizardMetric instanceof Timer) { // Do not need type label for request times labels.remove("type"); - exporter.exportTimer(CORE_REQUEST_TIMES, (Timer) dropwizardMetric, getLabels()); + formatter.exportTimer(CORE_REQUEST_TIMES, (Timer) dropwizardMetric, getLabels()); } } } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreHighlighterMetric.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreHighlighterMetric.java index 3d0c57d6043..52bc1292c25 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreHighlighterMetric.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreHighlighterMetric.java @@ -18,7 +18,7 @@ import com.codahale.metrics.Counter; import com.codahale.metrics.Metric; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; /** Dropwizard metrics of name HIGHLIGHTER.* */ public class SolrCoreHighlighterMetric extends SolrCoreMetric { @@ -44,7 +44,7 @@ public SolrCoreMetric parseLabels() { } @Override - public void toPrometheus(SolrPrometheusExporter exporter) { - exporter.exportCounter(CORE_HIGHLIGHER_METRICS, (Counter) dropwizardMetric, getLabels()); + public void toPrometheus(SolrPrometheusFormatter formatter) { + formatter.exportCounter(CORE_HIGHLIGHER_METRICS, (Counter) dropwizardMetric, getLabels()); } } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreIndexMetric.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreIndexMetric.java index 75a2457668a..048fc8d2ca2 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreIndexMetric.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreIndexMetric.java @@ -18,7 +18,7 @@ import com.codahale.metrics.Gauge; import com.codahale.metrics.Metric; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; /** Dropwizard metrics of name INDEX.* */ public class SolrCoreIndexMetric extends SolrCoreMetric { @@ -40,9 +40,9 @@ public SolrCoreMetric parseLabels() { } @Override - public void toPrometheus(SolrPrometheusExporter exporter) { + public void toPrometheus(SolrPrometheusFormatter formatter) { if (metricName.endsWith("sizeInBytes")) { - exporter.exportGauge(CORE_INDEX_METRICS, (Gauge) dropwizardMetric, getLabels()); + formatter.exportGauge(CORE_INDEX_METRICS, (Gauge) dropwizardMetric, getLabels()); } } } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreMetric.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreMetric.java index d977106d241..5010425a023 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreMetric.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreMetric.java @@ -16,7 +16,7 @@ */ package org.apache.solr.metrics.prometheus.core; -import static org.apache.solr.metrics.prometheus.core.PrometheusCoreExporterInfo.CLOUD_CORE_PATTERN; +import static org.apache.solr.metrics.prometheus.core.PrometheusCoreFormatterInfo.CLOUD_CORE_PATTERN; import com.codahale.metrics.Metric; import java.util.regex.Matcher; diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreSearcherMetric.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreSearcherMetric.java index 13d53f38db5..93071f9b8ad 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreSearcherMetric.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreSearcherMetric.java @@ -22,7 +22,7 @@ import com.codahale.metrics.Gauge; import com.codahale.metrics.Metric; import com.codahale.metrics.Timer; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; /** Dropwizard metrics of name SEARCHER.* */ public class SolrCoreSearcherMetric extends SolrCoreMetric { @@ -50,15 +50,16 @@ public SolrCoreMetric parseLabels() { } @Override - public void toPrometheus(SolrPrometheusExporter exporter) { + public void toPrometheus(SolrPrometheusFormatter formatter) { if (dropwizardMetric instanceof Gauge) { if (metricName.endsWith("liveDocsCache")) { - exporter.exportGauge(CORE_CACHE_SEARCHER_METRICS, (Gauge) dropwizardMetric, getLabels()); + formatter.exportGauge( + CORE_CACHE_SEARCHER_METRICS, (Gauge) dropwizardMetric, getLabels()); } else { - exporter.exportGauge(CORE_SEARCHER_METRICS, (Gauge) dropwizardMetric, getLabels()); + formatter.exportGauge(CORE_SEARCHER_METRICS, (Gauge) dropwizardMetric, getLabels()); } } else if (dropwizardMetric instanceof Timer) { - exporter.exportTimer(CORE_SEARCHER_TIMES, (Timer) dropwizardMetric, getLabels()); + formatter.exportTimer(CORE_SEARCHER_TIMES, (Timer) dropwizardMetric, getLabels()); } } } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreTlogMetric.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreTlogMetric.java index 8aba9cfd7ab..29b13e1caee 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreTlogMetric.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrCoreTlogMetric.java @@ -18,7 +18,7 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.Metric; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; /** Dropwizard metrics of name TLOG.* */ public class SolrCoreTlogMetric extends SolrCoreMetric { @@ -44,9 +44,9 @@ public SolrCoreMetric parseLabels() { } @Override - public void toPrometheus(SolrPrometheusExporter exporter) { + public void toPrometheus(SolrPrometheusFormatter formatter) { if (dropwizardMetric instanceof Meter) { - exporter.exportMeter(CORE_TLOG_METRICS, (Meter) dropwizardMetric, getLabels()); + formatter.exportMeter(CORE_TLOG_METRICS, (Meter) dropwizardMetric, getLabels()); } } } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrPrometheusCoreExporter.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrPrometheusCoreFormatter.java similarity index 88% rename from solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrPrometheusCoreExporter.java rename to solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrPrometheusCoreFormatter.java index 99066acea32..6b2a49154f3 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrPrometheusCoreExporter.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/SolrPrometheusCoreFormatter.java @@ -20,18 +20,18 @@ import com.google.common.base.Enums; import org.apache.solr.metrics.prometheus.SolrMetric; import org.apache.solr.metrics.prometheus.SolrNoOpMetric; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; /** * This class maintains a {@link io.prometheus.metrics.model.snapshots.MetricSnapshot}s exported * from solr.core {@link com.codahale.metrics.MetricRegistry} */ -public class SolrPrometheusCoreExporter extends SolrPrometheusExporter - implements PrometheusCoreExporterInfo { +public class SolrPrometheusCoreFormatter extends SolrPrometheusFormatter + implements PrometheusCoreFormatterInfo { public final String coreName; public final boolean cloudMode; - public SolrPrometheusCoreExporter(String coreName, boolean cloudMode) { + public SolrPrometheusCoreFormatter(String coreName, boolean cloudMode) { super(); this.coreName = coreName; this.cloudMode = cloudMode; @@ -46,7 +46,7 @@ public void exportDropwizardMetric(Metric dropwizardMetric, String metricName) { @Override public SolrMetric categorizeMetric(Metric dropwizardMetric, String metricName) { String metricCategory = metricName.split("\\.", 2)[0]; - if (!Enums.getIfPresent(PrometheusCoreExporterInfo.CoreCategory.class, metricCategory) + if (!Enums.getIfPresent(PrometheusCoreFormatterInfo.CoreCategory.class, metricCategory) .isPresent()) { return new SolrNoOpMetric(); } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/package-info.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/package-info.java index 701dcf9e3bf..acabe654867 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/core/package-info.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/core/package-info.java @@ -16,7 +16,7 @@ */ /** - * The {@link org.apache.solr.metrics.prometheus.core.SolrPrometheusCoreExporter} is responsible for - * exporting solr.core registry metrics to Prometheus. + * The {@link org.apache.solr.metrics.prometheus.core.SolrPrometheusCoreFormatter} is responsible + * for exporting solr.core registry metrics to Prometheus. */ package org.apache.solr.metrics.prometheus.core; diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/jetty/SolrJettyDispatchesMetric.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/jetty/SolrJettyDispatchesMetric.java index dbded5d36ac..cd057418845 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/jetty/SolrJettyDispatchesMetric.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/jetty/SolrJettyDispatchesMetric.java @@ -19,7 +19,7 @@ import com.codahale.metrics.Metric; import com.codahale.metrics.Timer; import org.apache.solr.metrics.prometheus.SolrMetric; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; /* Dropwizard metrics of name *.dispatches */ public class SolrJettyDispatchesMetric extends SolrJettyMetric { @@ -40,7 +40,7 @@ public SolrMetric parseLabels() { } @Override - public void toPrometheus(SolrPrometheusExporter exporter) { - exporter.exportTimerCount(JETTY_DISPATCHES_TOTAL, (Timer) dropwizardMetric, getLabels()); + public void toPrometheus(SolrPrometheusFormatter formatter) { + formatter.exportTimerCount(JETTY_DISPATCHES_TOTAL, (Timer) dropwizardMetric, getLabels()); } } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/jetty/SolrJettyReqRespMetric.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/jetty/SolrJettyReqRespMetric.java index 51c1fd80b3a..374f0bdfd59 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/jetty/SolrJettyReqRespMetric.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/jetty/SolrJettyReqRespMetric.java @@ -21,7 +21,7 @@ import com.codahale.metrics.Metric; import com.codahale.metrics.Timer; import org.apache.solr.metrics.prometheus.SolrMetric; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; /* Dropwizard metrics of name *.xx-responses and *-requests */ public class SolrJettyReqRespMetric extends SolrJettyMetric { @@ -51,14 +51,14 @@ public SolrMetric parseLabels() { } @Override - public void toPrometheus(SolrPrometheusExporter exporter) { + public void toPrometheus(SolrPrometheusFormatter formatter) { if (metricName.endsWith("xx-responses")) { - exporter.exportMeter(JETTY_RESPONSES_TOTAL, (Meter) dropwizardMetric, getLabels()); + formatter.exportMeter(JETTY_RESPONSES_TOTAL, (Meter) dropwizardMetric, getLabels()); } else if (metricName.endsWith("-requests")) { if (dropwizardMetric instanceof Counter) { - exporter.exportCounter(JETTY_REQUESTS_TOTAL, (Counter) dropwizardMetric, getLabels()); + formatter.exportCounter(JETTY_REQUESTS_TOTAL, (Counter) dropwizardMetric, getLabels()); } else if (dropwizardMetric instanceof Timer) { - exporter.exportTimerCount(JETTY_REQUESTS_TOTAL, (Timer) dropwizardMetric, getLabels()); + formatter.exportTimerCount(JETTY_REQUESTS_TOTAL, (Timer) dropwizardMetric, getLabels()); } } } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/jetty/SolrPrometheusJettyExporter.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/jetty/SolrPrometheusJettyFormatter.java similarity index 90% rename from solr/core/src/java/org/apache/solr/metrics/prometheus/jetty/SolrPrometheusJettyExporter.java rename to solr/core/src/java/org/apache/solr/metrics/prometheus/jetty/SolrPrometheusJettyFormatter.java index 0cfafb7f4cd..88eae10d39f 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/jetty/SolrPrometheusJettyExporter.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/jetty/SolrPrometheusJettyFormatter.java @@ -19,14 +19,14 @@ import com.codahale.metrics.Metric; import org.apache.solr.metrics.prometheus.SolrMetric; import org.apache.solr.metrics.prometheus.SolrNoOpMetric; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; /** * This class maintains a {@link io.prometheus.metrics.model.snapshots.MetricSnapshot}s exported * from solr.jetty {@link com.codahale.metrics.MetricRegistry} */ -public class SolrPrometheusJettyExporter extends SolrPrometheusExporter { - public SolrPrometheusJettyExporter() { +public class SolrPrometheusJettyFormatter extends SolrPrometheusFormatter { + public SolrPrometheusJettyFormatter() { super(); } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/jetty/package-info.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/jetty/package-info.java index 9f4123c13f1..c3c039c879e 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/jetty/package-info.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/jetty/package-info.java @@ -16,7 +16,7 @@ */ /** - * The {@link org.apache.solr.metrics.prometheus.jetty.SolrPrometheusJettyExporter} is responsible + * The {@link org.apache.solr.metrics.prometheus.jetty.SolrPrometheusJettyFormatter} is responsible * for exporting solr.jetty registry metrics to Prometheus. */ package org.apache.solr.metrics.prometheus.jetty; diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/PrometheusJvmExporterInfo.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/PrometheusJvmFormatterInfo.java similarity index 95% rename from solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/PrometheusJvmExporterInfo.java rename to solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/PrometheusJvmFormatterInfo.java index dfe9075eb17..660154ece4a 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/PrometheusJvmExporterInfo.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/PrometheusJvmFormatterInfo.java @@ -16,7 +16,7 @@ */ package org.apache.solr.metrics.prometheus.jvm; -public interface PrometheusJvmExporterInfo { +public interface PrometheusJvmFormatterInfo { /** Category of prefix Solr JVM dropwizard handler metric names */ enum JvmCategory { buffers, diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrJvmBuffersMetric.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrJvmBuffersMetric.java index 2cd09ac64e8..58f7ae697a5 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrJvmBuffersMetric.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrJvmBuffersMetric.java @@ -18,7 +18,7 @@ import com.codahale.metrics.Gauge; import com.codahale.metrics.Metric; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; /* Dropwizard metrics of name buffers.* */ public class SolrJvmBuffersMetric extends SolrJvmMetric { @@ -44,13 +44,13 @@ public SolrJvmMetric parseLabels() { } @Override - public void toPrometheus(SolrPrometheusExporter exporter) { + public void toPrometheus(SolrPrometheusFormatter formatter) { String[] parsedMetric = metricName.split("\\."); String metricType = parsedMetric[parsedMetric.length - 1]; if (metricType.equals("Count")) { - exporter.exportGauge(JVM_BUFFERS, (Gauge) dropwizardMetric, getLabels()); + formatter.exportGauge(JVM_BUFFERS, (Gauge) dropwizardMetric, getLabels()); } else if (metricType.equals(("MemoryUsed")) || metricType.equals(("TotalCapacity"))) { - exporter.exportGauge(JVM_BUFFERS_BYTES, (Gauge) dropwizardMetric, getLabels()); + formatter.exportGauge(JVM_BUFFERS_BYTES, (Gauge) dropwizardMetric, getLabels()); } } } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrJvmGcMetrics.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrJvmGcMetrics.java index 50aca3bff27..7dbb8baa817 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrJvmGcMetrics.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrJvmGcMetrics.java @@ -19,7 +19,7 @@ import com.codahale.metrics.Gauge; import com.codahale.metrics.Metric; import org.apache.solr.metrics.prometheus.SolrMetric; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; /* Dropwizard metrics of name gc.* */ public class SolrJvmGcMetrics extends SolrJvmMetric { @@ -44,11 +44,11 @@ public SolrMetric parseLabels() { } @Override - public void toPrometheus(SolrPrometheusExporter exporter) { + public void toPrometheus(SolrPrometheusFormatter formatter) { if (metricName.endsWith(".count")) { - exporter.exportGauge(JVM_GC, (Gauge) dropwizardMetric, getLabels()); + formatter.exportGauge(JVM_GC, (Gauge) dropwizardMetric, getLabels()); } else if (metricName.endsWith(".time")) { - exporter.exportGauge(JVM_GC_SECONDS, (Gauge) dropwizardMetric, getLabels()); + formatter.exportGauge(JVM_GC_SECONDS, (Gauge) dropwizardMetric, getLabels()); } } } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrJvmMemoryMetric.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrJvmMemoryMetric.java index 180edae2100..5ba1189c283 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrJvmMemoryMetric.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrJvmMemoryMetric.java @@ -19,7 +19,7 @@ import com.codahale.metrics.Gauge; import com.codahale.metrics.Metric; import org.apache.solr.metrics.prometheus.SolrMetric; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; /* Dropwizard metrics of name memory.* */ public class SolrJvmMemoryMetric extends SolrJvmMetric { @@ -46,7 +46,7 @@ public SolrMetric parseLabels() { } @Override - public void toPrometheus(SolrPrometheusExporter exporter) { + public void toPrometheus(SolrPrometheusFormatter formatter) { String[] parsedMetric = metricName.split("\\."); String metricType = parsedMetric[1]; switch (metricType) { @@ -54,11 +54,11 @@ public void toPrometheus(SolrPrometheusExporter exporter) { case "non-heap": case "total": labels.put("memory", parsedMetric[1]); - exporter.exportGauge(JVM_MEMORY, (Gauge) dropwizardMetric, getLabels()); + formatter.exportGauge(JVM_MEMORY, (Gauge) dropwizardMetric, getLabels()); break; case "pools": labels.put("space", parsedMetric[2]); - exporter.exportGauge(JVM_MEMORY_POOL_BYTES, (Gauge) dropwizardMetric, getLabels()); + formatter.exportGauge(JVM_MEMORY_POOL_BYTES, (Gauge) dropwizardMetric, getLabels()); break; } } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrJvmOsMetric.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrJvmOsMetric.java index fc42c6ac8ef..9b5d4ce1d69 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrJvmOsMetric.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrJvmOsMetric.java @@ -19,7 +19,7 @@ import com.codahale.metrics.Gauge; import com.codahale.metrics.Metric; import org.apache.solr.metrics.prometheus.SolrMetric; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; /* Dropwizard metrics of name os.* and threads.* */ public class SolrJvmOsMetric extends SolrJvmMetric { @@ -48,11 +48,11 @@ public SolrMetric parseLabels() { } @Override - public void toPrometheus(SolrPrometheusExporter exporter) { + public void toPrometheus(SolrPrometheusFormatter formatter) { if (metricName.startsWith("threads.")) { - exporter.exportGauge(JVM_OS_THREADS, (Gauge) dropwizardMetric, getLabels()); + formatter.exportGauge(JVM_OS_THREADS, (Gauge) dropwizardMetric, getLabels()); } else { - exporter.exportGauge(JVM_OS, (Gauge) dropwizardMetric, getLabels()); + formatter.exportGauge(JVM_OS, (Gauge) dropwizardMetric, getLabels()); } } } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrPrometheusJvmExporter.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrPrometheusJvmFormatter.java similarity index 90% rename from solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrPrometheusJvmExporter.java rename to solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrPrometheusJvmFormatter.java index 91a5ef07229..888a0cb9927 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrPrometheusJvmExporter.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/SolrPrometheusJvmFormatter.java @@ -20,15 +20,15 @@ import com.google.common.base.Enums; import org.apache.solr.metrics.prometheus.SolrMetric; import org.apache.solr.metrics.prometheus.SolrNoOpMetric; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; /** * This class maintains a {@link io.prometheus.metrics.model.snapshots.MetricSnapshot}s exported * from solr.jvm {@link com.codahale.metrics.MetricRegistry} */ -public class SolrPrometheusJvmExporter extends SolrPrometheusExporter - implements PrometheusJvmExporterInfo { - public SolrPrometheusJvmExporter() { +public class SolrPrometheusJvmFormatter extends SolrPrometheusFormatter + implements PrometheusJvmFormatterInfo { + public SolrPrometheusJvmFormatter() { super(); } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/package-info.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/package-info.java index 230f9576576..5150b3e2a19 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/package-info.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/jvm/package-info.java @@ -16,7 +16,7 @@ */ /** - * The {@link org.apache.solr.metrics.prometheus.jvm.SolrPrometheusJvmExporter} is responsible for + * The {@link org.apache.solr.metrics.prometheus.jvm.SolrPrometheusJvmFormatter} is responsible for * exporting solr.jvm registry metrics to Prometheus. */ package org.apache.solr.metrics.prometheus.jvm; diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/node/PrometheusNodeExporterInfo.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/node/PrometheusNodeFormatterInfo.java similarity index 95% rename from solr/core/src/java/org/apache/solr/metrics/prometheus/node/PrometheusNodeExporterInfo.java rename to solr/core/src/java/org/apache/solr/metrics/prometheus/node/PrometheusNodeFormatterInfo.java index 67caa63ef24..ab462751d60 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/node/PrometheusNodeExporterInfo.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/node/PrometheusNodeFormatterInfo.java @@ -16,7 +16,7 @@ */ package org.apache.solr.metrics.prometheus.node; -public interface PrometheusNodeExporterInfo { +public interface PrometheusNodeFormatterInfo { /** Category of prefix Solr Node dropwizard handler metric names */ enum NodeCategory { ADMIN, diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/node/SolrNodeContainerMetric.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/node/SolrNodeContainerMetric.java index bb48eea9790..2e3e16645c2 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/node/SolrNodeContainerMetric.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/node/SolrNodeContainerMetric.java @@ -19,7 +19,7 @@ import com.codahale.metrics.Gauge; import com.codahale.metrics.Metric; import org.apache.solr.metrics.prometheus.SolrMetric; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; /* Dropwizard metrics of name CONTAINER.* */ public class SolrNodeContainerMetric extends SolrNodeMetric { @@ -44,14 +44,14 @@ public SolrMetric parseLabels() { } @Override - public void toPrometheus(SolrPrometheusExporter exporter) { + public void toPrometheus(SolrPrometheusFormatter formatter) { String[] parsedMetric = metricName.split("\\."); if (metricName.startsWith("CONTAINER.cores.")) { labels.put("item", parsedMetric[2]); - exporter.exportGauge(NODE_CORES, (Gauge) dropwizardMetric, getLabels()); + formatter.exportGauge(NODE_CORES, (Gauge) dropwizardMetric, getLabels()); } else if (metricName.startsWith("CONTAINER.fs.coreRoot.")) { labels.put("item", parsedMetric[3]); - exporter.exportGauge(NODE_CORE_FS_BYTES, (Gauge) dropwizardMetric, getLabels()); + formatter.exportGauge(NODE_CORE_FS_BYTES, (Gauge) dropwizardMetric, getLabels()); } } } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/node/SolrNodeHandlerMetric.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/node/SolrNodeHandlerMetric.java index 2d28ce90703..25f014d677c 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/node/SolrNodeHandlerMetric.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/node/SolrNodeHandlerMetric.java @@ -21,7 +21,7 @@ import com.codahale.metrics.Meter; import com.codahale.metrics.Metric; import org.apache.solr.metrics.prometheus.SolrMetric; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; /* Dropwizard metrics of name ADMIN.* and UPDATE.* */ public class SolrNodeHandlerMetric extends SolrNodeMetric { @@ -49,16 +49,16 @@ public SolrMetric parseLabels() { } @Override - public void toPrometheus(SolrPrometheusExporter exporter) { + public void toPrometheus(SolrPrometheusFormatter formatter) { if (metricName.endsWith(".totalTime")) { labels.remove("type"); - exporter.exportCounter(NODE_SECONDS_TOTAL, (Counter) dropwizardMetric, getLabels()); + formatter.exportCounter(NODE_SECONDS_TOTAL, (Counter) dropwizardMetric, getLabels()); } else if (metricName.endsWith("Connections")) { - exporter.exportGauge(NODE_CONNECTIONS, (Gauge) dropwizardMetric, getLabels()); + formatter.exportGauge(NODE_CONNECTIONS, (Gauge) dropwizardMetric, getLabels()); } else if (dropwizardMetric instanceof Meter) { - exporter.exportMeter(NODE_REQUESTS, (Meter) dropwizardMetric, getLabels()); + formatter.exportMeter(NODE_REQUESTS, (Meter) dropwizardMetric, getLabels()); } else if (dropwizardMetric instanceof Counter) { - exporter.exportCounter(NODE_REQUESTS, (Counter) dropwizardMetric, getLabels()); + formatter.exportCounter(NODE_REQUESTS, (Counter) dropwizardMetric, getLabels()); } } } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/node/SolrPrometheusNodeExporter.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/node/SolrPrometheusNodeFormatter.java similarity index 91% rename from solr/core/src/java/org/apache/solr/metrics/prometheus/node/SolrPrometheusNodeExporter.java rename to solr/core/src/java/org/apache/solr/metrics/prometheus/node/SolrPrometheusNodeFormatter.java index 40a76bfd85c..eb542e05b77 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/node/SolrPrometheusNodeExporter.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/node/SolrPrometheusNodeFormatter.java @@ -25,15 +25,15 @@ import io.prometheus.metrics.model.snapshots.Labels; import org.apache.solr.metrics.prometheus.SolrMetric; import org.apache.solr.metrics.prometheus.SolrNoOpMetric; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; /** * This class maintains a {@link io.prometheus.metrics.model.snapshots.MetricSnapshot}s exported * from solr.node {@link com.codahale.metrics.MetricRegistry} */ -public class SolrPrometheusNodeExporter extends SolrPrometheusExporter - implements PrometheusNodeExporterInfo { - public SolrPrometheusNodeExporter() { +public class SolrPrometheusNodeFormatter extends SolrPrometheusFormatter + implements PrometheusNodeFormatterInfo { + public SolrPrometheusNodeFormatter() { super(); } @@ -51,7 +51,7 @@ public void exportDropwizardMetric(Metric dropwizardMetric, String metricName) { @Override public SolrMetric categorizeMetric(Metric dropwizardMetric, String metricName) { String metricCategory = metricName.split("\\.", 2)[0]; - if (!Enums.getIfPresent(PrometheusNodeExporterInfo.NodeCategory.class, metricCategory) + if (!Enums.getIfPresent(PrometheusNodeFormatterInfo.NodeCategory.class, metricCategory) .isPresent()) { return new SolrNoOpMetric(); } diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/node/package-info.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/node/package-info.java index 31ca46347d0..64de64aa73c 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/node/package-info.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/node/package-info.java @@ -16,7 +16,7 @@ */ /** - * The {@link org.apache.solr.metrics.prometheus.node.SolrPrometheusNodeExporter} is responsible for - * exporting solr.node registry metrics to Prometheus. + * The {@link org.apache.solr.metrics.prometheus.node.SolrPrometheusNodeFormatter} is responsible + * for exporting solr.node registry metrics to Prometheus. */ package org.apache.solr.metrics.prometheus.node; diff --git a/solr/core/src/java/org/apache/solr/metrics/prometheus/package-info.java b/solr/core/src/java/org/apache/solr/metrics/prometheus/package-info.java index 33da9290400..68f0c2a983e 100644 --- a/solr/core/src/java/org/apache/solr/metrics/prometheus/package-info.java +++ b/solr/core/src/java/org/apache/solr/metrics/prometheus/package-info.java @@ -16,7 +16,7 @@ */ /** - * The {@link org.apache.solr.metrics.prometheus.SolrPrometheusExporter} is responsible for + * The {@link org.apache.solr.metrics.prometheus.SolrPrometheusFormatter} is responsible for * collecting Prometheus metrics from exporting {@link com.codahale.metrics.Metric}'s from {@link * com.codahale.metrics.MetricRegistry} {@link org.apache.solr.metrics.prometheus.SolrMetric} is a * wrapper to export {@link com.codahale.metrics.Metric} to {@link diff --git a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java index 1b671c0e172..15b6def6ecb 100644 --- a/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java +++ b/solr/core/src/java/org/apache/solr/packagemanager/PackageManager.java @@ -121,7 +121,7 @@ public void uninstall(String packageName, String version) + packageName + " is currently deployed on collection: " + collection - + ". Undeploy the package with undeploy -collections [,,...] before attempting to uninstall the package."); + + ". Undeploy the package with undeploy --collections [,,...] before attempting to uninstall the package."); System.exit(1); } } @@ -138,7 +138,7 @@ public void uninstall(String packageName, String version) + packageName + "is currently deployed as a cluster-level plugin (" + clusterPackageInstance.getCustomData() - + "). Undeploy the package with undeploy -collections [,,...] before uninstalling the package."); + + "). Undeploy the package with undeploy --collections [,,...] before uninstalling the package."); System.exit(1); } } diff --git a/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java b/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java index b7d0a11eb97..193b838e9cd 100644 --- a/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java +++ b/solr/core/src/java/org/apache/solr/request/SolrRequestInfo.java @@ -33,7 +33,6 @@ import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.search.QueryLimits; import org.apache.solr.servlet.SolrDispatchFilter; -import org.apache.solr.util.ThreadCpuTimer; import org.apache.solr.util.TimeZoneUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -46,7 +45,6 @@ public class SolrRequestInfo { private static final ThreadLocal> threadLocal = ThreadLocal.withInitial(ArrayDeque::new); static final Object LIMITS_KEY = new Object(); - static final Object CPU_TIMER_KEY = new Object(); private int refCount = 1; // prevent closing when still used @@ -80,10 +78,12 @@ public static void setRequestInfo(SolrRequestInfo info) { assert false : "SolrRequestInfo Stack is full"; log.error("SolrRequestInfo Stack is full"); } else if (!stack.isEmpty() && info.req != null) { - // New SRI instances inherit limits and thread CPU from prior SRI regardless of parameters. + // New SRI instances inherit limits from prior SRI regardless of parameters. // This ensures these two properties cannot be changed or removed for a given thread once set. // if req is null then limits will be an empty instance with no limits anyway. - info.req.getContext().put(CPU_TIMER_KEY, stack.peek().getThreadCpuTimer()); + + // protected by !stack.isEmpty() + // noinspection DataFlowIssue info.req.getContext().put(LIMITS_KEY, stack.peek().getLimits()); } // this creates both new QueryLimits and new ThreadCpuTime if not already set @@ -244,27 +244,12 @@ private void initQueryLimits() { */ public QueryLimits getLimits() { // make sure the ThreadCpuTime is always initialized - getThreadCpuTimer(); return req == null || rsp == null ? QueryLimits.NONE : (QueryLimits) req.getContext().computeIfAbsent(LIMITS_KEY, (k) -> new QueryLimits(req, rsp)); } - /** - * Get the thread CPU time monitor for the current request. This will either trigger the creation - * of a new instance if it hasn't been yet created, or will retrieve the already existing instance - * from the "bottom" of the request stack. - * - * @return the {@link ThreadCpuTimer} object for the current request. - */ - public ThreadCpuTimer getThreadCpuTimer() { - return req == null - ? new ThreadCpuTimer() - : (ThreadCpuTimer) - req.getContext().computeIfAbsent(CPU_TIMER_KEY, k -> new ThreadCpuTimer()); - } - public SolrDispatchFilter.Action getAction() { return action; } diff --git a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java index e67b20a0007..5c6210603da 100644 --- a/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java +++ b/solr/core/src/java/org/apache/solr/response/PrometheusResponseWriter.java @@ -33,11 +33,11 @@ import java.util.stream.Collectors; import org.apache.solr.common.util.NamedList; import org.apache.solr.metrics.AggregateMetric; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; -import org.apache.solr.metrics.prometheus.core.SolrPrometheusCoreExporter; -import org.apache.solr.metrics.prometheus.jetty.SolrPrometheusJettyExporter; -import org.apache.solr.metrics.prometheus.jvm.SolrPrometheusJvmExporter; -import org.apache.solr.metrics.prometheus.node.SolrPrometheusNodeExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; +import org.apache.solr.metrics.prometheus.core.SolrPrometheusCoreFormatter; +import org.apache.solr.metrics.prometheus.jetty.SolrPrometheusJettyFormatter; +import org.apache.solr.metrics.prometheus.jvm.SolrPrometheusJvmFormatter; +import org.apache.solr.metrics.prometheus.node.SolrPrometheusNodeFormatter; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.util.stats.MetricUtils; import org.slf4j.Logger; @@ -49,6 +49,7 @@ */ @SuppressWarnings(value = "unchecked") public class PrometheusResponseWriter extends RawResponseWriter { + private static final String CONTENT_TYPE_PROMETHEUS = "text/plain; version=0.0.4"; private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @Override @@ -58,15 +59,20 @@ public void write(OutputStream out, SolrQueryRequest request, SolrQueryResponse (NamedList) response.getValues().get("metrics"); var prometheusTextFormatWriter = new PrometheusTextFormatWriter(false); for (Map.Entry prometheusRegistry : prometheusRegistries) { - var prometheusExporter = (SolrPrometheusExporter) prometheusRegistry.getValue(); - prometheusTextFormatWriter.write(out, prometheusExporter.collect()); + var prometheusFormatter = (SolrPrometheusFormatter) prometheusRegistry.getValue(); + prometheusTextFormatWriter.write(out, prometheusFormatter.collect()); } } + @Override + public String getContentType(SolrQueryRequest request, SolrQueryResponse response) { + return CONTENT_TYPE_PROMETHEUS; + } + /** * Provides a representation of the given Dropwizard metric registry as {@link - * SolrPrometheusCoreExporter}-s. Only those metrics are converted which match at least one of the - * given MetricFilter instances. + * SolrPrometheusCoreFormatter}-s. Only those metrics are converted which match at least one of + * the given MetricFilter instances. * * @param registry the {@link MetricRegistry} to be converted * @param shouldMatchFilters a list of {@link MetricFilter} instances. A metric must match any @@ -77,7 +83,7 @@ public void write(OutputStream out, SolrQueryRequest request, SolrQueryResponse * @param skipHistograms discard any {@link Histogram}-s and histogram parts of {@link Timer}-s. * @param skipAggregateValues discard internal values of {@link AggregateMetric}-s. * @param compact use compact representation for counters and gauges. - * @param consumer consumer that accepts produced {@link SolrPrometheusCoreExporter}-s + * @param consumer consumer that accepts produced {@link SolrPrometheusCoreFormatter}-s */ public static void toPrometheus( MetricRegistry registry, @@ -88,10 +94,10 @@ public static void toPrometheus( boolean skipHistograms, boolean skipAggregateValues, boolean compact, - Consumer consumer) { + Consumer consumer) { Map dropwizardMetrics = registry.getMetrics(); - var exporter = getExporterType(registryName); - if (exporter == null) { + var formatter = getFormatterType(registryName); + if (formatter == null) { return; } @@ -107,17 +113,17 @@ public static void toPrometheus( (metricName, metric) -> { try { Metric dropwizardMetric = dropwizardMetrics.get(metricName); - exporter.exportDropwizardMetric(dropwizardMetric, metricName); + formatter.exportDropwizardMetric(dropwizardMetric, metricName); } catch (Exception e) { // Do not fail entirely for metrics exporting. Log and try to export next metric log.warn("Error occurred exporting Dropwizard Metric to Prometheus", e); } }); - consumer.accept(exporter); + consumer.accept(formatter); } - public static SolrPrometheusExporter getExporterType(String registryName) { + public static SolrPrometheusFormatter getFormatterType(String registryName) { String coreName; boolean cloudMode = false; String[] parsedRegistry = registryName.split("\\."); @@ -132,13 +138,13 @@ public static SolrPrometheusExporter getExporterType(String registryName) { } else { coreName = registryName; } - return new SolrPrometheusCoreExporter(coreName, cloudMode); + return new SolrPrometheusCoreFormatter(coreName, cloudMode); case "jvm": - return new SolrPrometheusJvmExporter(); + return new SolrPrometheusJvmFormatter(); case "jetty": - return new SolrPrometheusJettyExporter(); + return new SolrPrometheusJettyFormatter(); case "node": - return new SolrPrometheusNodeExporter(); + return new SolrPrometheusNodeFormatter(); default: return null; } diff --git a/solr/core/src/java/org/apache/solr/schema/BinaryField.java b/solr/core/src/java/org/apache/solr/schema/BinaryField.java index 37bfc653236..94bc94fd86f 100644 --- a/solr/core/src/java/org/apache/solr/schema/BinaryField.java +++ b/solr/core/src/java/org/apache/solr/schema/BinaryField.java @@ -152,11 +152,6 @@ public List createFields(SchemaField field, Object val) { protected void checkSupportsDocValues() { // we support DocValues } - @Override - protected boolean doesTypeSupportDocValues() { - return true; - } - @Override public Object toNativeType(Object val) { if (val instanceof byte[]) { diff --git a/solr/core/src/java/org/apache/solr/schema/CollationField.java b/solr/core/src/java/org/apache/solr/schema/CollationField.java index 739bae5f211..91f67d459e7 100644 --- a/solr/core/src/java/org/apache/solr/schema/CollationField.java +++ b/solr/core/src/java/org/apache/solr/schema/CollationField.java @@ -246,7 +246,7 @@ protected void checkSupportsDocValues() { // we support DocValues } @Override - protected boolean doesTypeSupportDocValues() { + protected boolean enableDocValuesByDefault() { return true; } diff --git a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java index d83462235b0..4d528361dd4 100644 --- a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java +++ b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java @@ -179,7 +179,7 @@ public VectorEncoding getVectorEncoding() { } @Override - protected boolean doesTypeSupportDocValues() { + protected boolean enableDocValuesByDefault() { return false; } diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java index 9eae31f1a02..cb60b1f332a 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java @@ -188,9 +188,9 @@ protected void setArgs(IndexSchema schema, Map args) { args.remove("compressThreshold"); } if (schemaVersion >= 1.6f) properties |= USE_DOCVALUES_AS_STORED; - if (schemaVersion >= 1.7f && doesTypeSupportDocValues()) properties |= DOC_VALUES; + if (schemaVersion >= 1.7f && enableDocValuesByDefault()) properties |= DOC_VALUES; - properties |= UNINVERTIBLE; + if (schemaVersion < 1.7f) properties |= UNINVERTIBLE; this.args = Collections.unmodifiableMap(args); Map initArgs = new HashMap<>(args); @@ -1161,15 +1161,17 @@ protected void checkSupportsDocValues() { ErrorCode.SERVER_ERROR, "Field type " + this + " does not support doc values"); } - /** Returns whether this field type supports docValues. By default none do. */ - protected boolean doesTypeSupportDocValues() { - try { - // TODO: In Solr 10.0 change this such that checkSupportsDocValues() calls this method instead - checkSupportsDocValues(); - return true; - } catch (Exception ignored) { - return false; - } + /** + * Returns whether this field type should enable docValues by default for schemaVersion >= 1.7. + * This should not be enabled for fields that did not have docValues implemented by Solr 9.7, as + * users may have indexed documents without docValues (since they weren't supported). Flipping the + * default docValues values when they upgrade to a new version will break their index + * compatibility. + * + *

New field types can enable this without issue, as long as they support docValues. + */ + protected boolean enableDocValuesByDefault() { + return false; } public static final String TYPE = "type"; diff --git a/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.java b/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.java index b237b79ad37..5010ba27075 100644 --- a/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.java +++ b/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.java @@ -70,7 +70,7 @@ protected void checkSupportsDocValues() { // we support DocValues } @Override - protected boolean doesTypeSupportDocValues() { + protected boolean enableDocValuesByDefault() { return true; } diff --git a/solr/core/src/java/org/apache/solr/schema/PointType.java b/solr/core/src/java/org/apache/solr/schema/PointType.java index 2f80fc24722..3d68af9449c 100644 --- a/solr/core/src/java/org/apache/solr/schema/PointType.java +++ b/solr/core/src/java/org/apache/solr/schema/PointType.java @@ -172,11 +172,6 @@ protected void checkSupportsDocValues() { } } - @Override - protected boolean doesTypeSupportDocValues() { - return true; - } - /** * Calculates the range and creates a RangeQuery (bounding box) wrapped in a BooleanQuery (unless * the dimension is 1, one range for every dimension, AND'd together by a Boolean diff --git a/solr/core/src/java/org/apache/solr/schema/PrimitiveFieldType.java b/solr/core/src/java/org/apache/solr/schema/PrimitiveFieldType.java index 2ce3203ae71..cc3fb3f7ff0 100644 --- a/solr/core/src/java/org/apache/solr/schema/PrimitiveFieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/PrimitiveFieldType.java @@ -39,7 +39,7 @@ protected void checkSupportsDocValues() { // primitive types support DocValues } @Override - protected boolean doesTypeSupportDocValues() { + protected boolean enableDocValuesByDefault() { return true; } diff --git a/solr/core/src/java/org/apache/solr/schema/SortableTextField.java b/solr/core/src/java/org/apache/solr/schema/SortableTextField.java index b61d668f333..9bc6a3f7cb9 100644 --- a/solr/core/src/java/org/apache/solr/schema/SortableTextField.java +++ b/solr/core/src/java/org/apache/solr/schema/SortableTextField.java @@ -151,7 +151,7 @@ protected void checkSupportsDocValues() { } @Override - protected boolean doesTypeSupportDocValues() { + protected boolean enableDocValuesByDefault() { return true; } @@ -227,7 +227,7 @@ public ValueSource getSingleValueSource( return new SortedSetFieldSource(field.getName(), selectorType); } - /** {@inheritDoc} this field type is not uninvertable, this method always returns null */ + /** {@inheritDoc} this field type is not uninvertible, this method always returns null */ @Override public Type getUninversionType(SchemaField sf) { return null; diff --git a/solr/core/src/java/org/apache/solr/search/AbstractReRankQuery.java b/solr/core/src/java/org/apache/solr/search/AbstractReRankQuery.java index f2eb6de2c3b..62c4bd98bbc 100644 --- a/solr/core/src/java/org/apache/solr/search/AbstractReRankQuery.java +++ b/solr/core/src/java/org/apache/solr/search/AbstractReRankQuery.java @@ -75,7 +75,7 @@ public MergeStrategy getMergeStrategy() { @Override @SuppressWarnings({"unchecked"}) - public TopDocsCollector getTopDocsCollector( + public TopDocsCollector getTopDocsCollector( int len, QueryCommand cmd, IndexSearcher searcher) throws IOException { if (this.boostedPriority == null) { SolrRequestInfo info = SolrRequestInfo.getRequestInfo(); diff --git a/solr/core/src/java/org/apache/solr/search/CpuAllowedLimit.java b/solr/core/src/java/org/apache/solr/search/CpuAllowedLimit.java index aa3a1fae7b2..72362159f0b 100644 --- a/solr/core/src/java/org/apache/solr/search/CpuAllowedLimit.java +++ b/solr/core/src/java/org/apache/solr/search/CpuAllowedLimit.java @@ -16,14 +16,23 @@ */ package org.apache.solr.search; +import static org.apache.solr.util.ThreadCpuTimer.readNSAndReset; + import com.google.common.annotations.VisibleForTesting; +import java.lang.invoke.MethodHandles; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.util.Locale; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import net.jcip.annotations.NotThreadSafe; -import org.apache.lucene.index.QueryTimeout; import org.apache.solr.common.params.CommonParams; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestInfo; +import org.apache.solr.response.SolrQueryResponse; import org.apache.solr.util.ThreadCpuTimer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * Enforces a CPU-time based timeout on a given SolrQueryRequest, as specified by the {@code @@ -37,14 +46,18 @@ * @see ThreadCpuTimer */ @NotThreadSafe -public class CpuAllowedLimit implements QueryTimeout { - private final ThreadCpuTimer threadCpuTimer; +public class CpuAllowedLimit implements QueryLimit { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private final long requestedTimeoutNs; + private volatile long timedOutAt = 0L; + AtomicLong accumulatedTime = new AtomicLong(0); + public static final String TIMING_CONTEXT = CpuAllowedLimit.class.getName(); /** * Create an object to represent a CPU time limit for the current request. NOTE: this * implementation will attempt to obtain an existing thread CPU time monitor, created when {@link - * SolrRequestInfo#getThreadCpuTimer()} is initialized. + * QueryLimits#QueryLimits(SolrQueryRequest, SolrQueryResponse)} is called. * * @param req solr request with a {@code cpuAllowed} parameter */ @@ -52,11 +65,6 @@ public CpuAllowedLimit(SolrQueryRequest req) { if (!ThreadCpuTimer.isSupported()) { throw new IllegalArgumentException("Thread CPU time monitoring is not available."); } - SolrRequestInfo solrRequestInfo = SolrRequestInfo.getRequestInfo(); - // get existing timer if available to ensure sub-queries can't reset/exceed the intended time - // constraint. - threadCpuTimer = - solrRequestInfo != null ? solrRequestInfo.getThreadCpuTimer() : new ThreadCpuTimer(); long reqCpuLimit = req.getParams().getLong(CommonParams.CPU_ALLOWED, -1L); if (reqCpuLimit <= 0L) { @@ -65,11 +73,15 @@ public CpuAllowedLimit(SolrQueryRequest req) { } // calculate the time when the limit is reached, e.g. account for the time already spent requestedTimeoutNs = TimeUnit.NANOSECONDS.convert(reqCpuLimit, TimeUnit.MILLISECONDS); + + // here we rely on the current thread never creating a second CpuAllowedLimit within the same + // request, and also rely on it always creating a new CpuAllowedLimit object for each + // request that requires it. + ThreadCpuTimer.beginContext(TIMING_CONTEXT); } @VisibleForTesting CpuAllowedLimit(long limitMs) { - this.threadCpuTimer = new ThreadCpuTimer(); requestedTimeoutNs = TimeUnit.NANOSECONDS.convert(limitMs, TimeUnit.MILLISECONDS); } @@ -81,6 +93,38 @@ static boolean hasCpuLimit(SolrQueryRequest req) { /** Return true if usage has exceeded the limit. */ @Override public boolean shouldExit() { - return threadCpuTimer.getElapsedCpuNs() > requestedTimeoutNs; + if (timedOutAt > 0L) { + return true; + } + // if unsupported, use zero, and thus never exit, expect jvm and/or cpu branch + // prediction to short circuit things if unsupported. + Long delta = readNSAndReset(TIMING_CONTEXT).orElse(0L); + try { + if (accumulatedTime.addAndGet(delta) > requestedTimeoutNs) { + timedOutAt = accumulatedTime.get(); + return true; + } + return false; + } finally { + if (log.isTraceEnabled()) { + java.text.DecimalFormatSymbols symbols = new DecimalFormatSymbols(Locale.US); + DecimalFormat formatter = new DecimalFormat("#,###", symbols); + String threadName = Thread.currentThread().getName(); + String deltaFmt = formatter.format(delta); + String accumulated = formatter.format(accumulatedTime.get()); + String timeoutForComparison = formatter.format(requestedTimeoutNs); + log.trace( + "++++++++++++ SHOULD_EXIT - measuredDelta:{} accumulated:{} vs {} ++++ ON:{}", + deltaFmt, + accumulated, + timeoutForComparison, + threadName); + } + } + } + + @Override + public Object currentValue() { + return timedOutAt > 0 ? timedOutAt : accumulatedTime.get(); } } diff --git a/solr/core/src/java/org/apache/solr/search/ExportQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/ExportQParserPlugin.java index e43f229d8a5..8529533dfb3 100644 --- a/solr/core/src/java/org/apache/solr/search/ExportQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/ExportQParserPlugin.java @@ -104,7 +104,7 @@ public Query rewrite(IndexReader reader) throws IOException { } @Override - public TopDocsCollector getTopDocsCollector( + public TopDocsCollector getTopDocsCollector( int len, QueryCommand cmd, IndexSearcher searcher) throws IOException { int leafCount = searcher.getTopReaderContext().leaves().size(); FixedBitSet[] sets = new FixedBitSet[leafCount]; diff --git a/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java b/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java index f9810b1a083..b56931a650e 100644 --- a/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/MultiThreadedSearcher.java @@ -120,10 +120,7 @@ SearchResult searchCollectorManagers( } static boolean allowMT(DelegatingCollector postFilter, QueryCommand cmd, Query query) { - if (postFilter != null - || cmd.getSegmentTerminateEarly() - || cmd.getTimeAllowed() > 0 - || !cmd.getMultiThreaded()) { + if (postFilter != null || cmd.getSegmentTerminateEarly() || !cmd.getMultiThreaded()) { return false; } else { MTCollectorQueryCheck allowMT = new MTCollectorQueryCheck(); diff --git a/solr/core/src/java/org/apache/solr/search/QueryLimit.java b/solr/core/src/java/org/apache/solr/search/QueryLimit.java new file mode 100644 index 00000000000..1043707f06b --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/QueryLimit.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search; + +import org.apache.lucene.index.QueryTimeout; + +public interface QueryLimit extends QueryTimeout { + /** + * A value representing the portion of the specified limit that has been consumed. Reading this + * value should never affect the outcome (other than the time it takes to do it). + * + * @return an expression of the amount of the limit used so far, numeric if possible, if + * non-numeric it should have toString() suitable for logging or similar expression to the + * user. + */ + Object currentValue(); +} diff --git a/solr/core/src/java/org/apache/solr/search/QueryLimits.java b/solr/core/src/java/org/apache/solr/search/QueryLimits.java index 2e232bccde6..86c7f488de3 100644 --- a/solr/core/src/java/org/apache/solr/search/QueryLimits.java +++ b/solr/core/src/java/org/apache/solr/search/QueryLimits.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Optional; import org.apache.lucene.index.QueryTimeout; import org.apache.solr.common.params.CommonParams; import org.apache.solr.request.SolrQueryRequest; @@ -34,7 +35,7 @@ * return true the next time it is checked (it may be checked in either Lucene code or Solr code) */ public class QueryLimits implements QueryTimeout { - private final List limits = + private final List limits = new ArrayList<>(3); // timeAllowed, cpu, and memory anticipated public static QueryLimits NONE = new QueryLimits(); @@ -149,6 +150,15 @@ public String limitStatusMessage() { return sb.toString(); } + public Optional currentLimitValueFor(Class limitClass) { + for (QueryLimit limit : limits) { + if (limit.getClass().isAssignableFrom(limitClass)) { + return Optional.of(limit.currentValue()); + } + } + return Optional.empty(); + } + /** Return true if there are any limits enabled for the current request. */ public boolean isLimitsEnabled() { return !limits.isEmpty(); diff --git a/solr/core/src/java/org/apache/solr/search/RankQuery.java b/solr/core/src/java/org/apache/solr/search/RankQuery.java index ec6ae26f5a3..3f281ebbd29 100644 --- a/solr/core/src/java/org/apache/solr/search/RankQuery.java +++ b/solr/core/src/java/org/apache/solr/search/RankQuery.java @@ -29,7 +29,7 @@ */ public abstract class RankQuery extends ExtendedQueryBase { - public abstract TopDocsCollector getTopDocsCollector( + public abstract TopDocsCollector getTopDocsCollector( int len, QueryCommand cmd, IndexSearcher searcher) throws IOException; public abstract MergeStrategy getMergeStrategy(); diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java index 8204487fed3..db3940f464e 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java @@ -1921,9 +1921,7 @@ public ScoreMode scoreMode() { final TopDocs topDocs; final ScoreMode scoreModeUsed; if (!MultiThreadedSearcher.allowMT(pf.postFilter, cmd, query)) { - if (log.isDebugEnabled()) { - log.debug("skipping collector manager"); - } + log.trace("SINGLE THREADED search, skipping collector manager in getDocListNC"); final TopDocsCollector topCollector = buildTopDocsCollector(len, cmd); MaxScoreCollector maxScoreCollector = null; Collector collector = topCollector; @@ -1942,9 +1940,7 @@ public ScoreMode scoreMode() { ? (maxScoreCollector == null ? Float.NaN : maxScoreCollector.getMaxScore()) : 0.0f; } else { - if (log.isDebugEnabled()) { - log.debug("using CollectorManager"); - } + log.trace("MULTI-THREADED search, using CollectorManager int getDocListNC"); final MultiThreadedSearcher.SearchResult searchResult = new MultiThreadedSearcher(this) .searchCollectorManagers(len, cmd, query, true, needScores, false); @@ -2046,6 +2042,8 @@ public ScoreMode scoreMode() { } else { final TopDocs topDocs; if (!MultiThreadedSearcher.allowMT(pf.postFilter, cmd, query)) { + log.trace("SINGLE THREADED search, skipping collector manager in getDocListAndSetNC"); + @SuppressWarnings({"rawtypes"}) final TopDocsCollector topCollector = buildTopDocsCollector(len, cmd); final DocSetCollector setCollector = new DocSetCollector(maxDoc); @@ -2072,7 +2070,7 @@ public ScoreMode scoreMode() { ? (maxScoreCollector == null ? Float.NaN : maxScoreCollector.getMaxScore()) : 0.0f; } else { - log.debug("using CollectorManager"); + log.trace("MULTI-THREADED search, using CollectorManager in getDocListAndSetNC"); boolean needMaxScore = needScores; MultiThreadedSearcher.SearchResult searchResult = diff --git a/solr/core/src/java/org/apache/solr/search/TimeAllowedLimit.java b/solr/core/src/java/org/apache/solr/search/TimeAllowedLimit.java index 432993d6c43..f837712176c 100644 --- a/solr/core/src/java/org/apache/solr/search/TimeAllowedLimit.java +++ b/solr/core/src/java/org/apache/solr/search/TimeAllowedLimit.java @@ -19,7 +19,6 @@ import static java.lang.System.nanoTime; import java.util.concurrent.TimeUnit; -import org.apache.lucene.index.QueryTimeout; import org.apache.solr.common.params.CommonParams; import org.apache.solr.request.SolrQueryRequest; @@ -30,9 +29,10 @@ * has {@code timeAllowed} set. Essentially only one timeAllowed can be specified for any thread * executing a query. This is to ensure that subqueries don't escape from the intended limit */ -public class TimeAllowedLimit implements QueryTimeout { +public class TimeAllowedLimit implements QueryLimit { private final long timeoutAt; + private final long timingSince; /** * Create an object to represent a time limit for the current request. @@ -50,9 +50,12 @@ public TimeAllowedLimit(SolrQueryRequest req) { throw new IllegalArgumentException( "Check for limit with hasTimeLimit(req) before creating a TimeAllowedLimit"); } - long timeAllowed = reqTimeAllowed - (long) req.getRequestTimer().getTime(); + long timeAlreadySpent = (long) req.getRequestTimer().getTime(); + long now = nanoTime(); + long timeAllowed = reqTimeAllowed - timeAlreadySpent; long nanosAllowed = TimeUnit.NANOSECONDS.convert(timeAllowed, TimeUnit.MILLISECONDS); - timeoutAt = nanoTime() + nanosAllowed; + timeoutAt = now + nanosAllowed; + timingSince = now - timeAlreadySpent; } /** Return true if the current request has a parameter with a valid value of the limit. */ @@ -65,4 +68,9 @@ static boolean hasTimeLimit(SolrQueryRequest req) { public boolean shouldExit() { return timeoutAt - nanoTime() < 0L; } + + @Override + public Object currentValue() { + return nanoTime() - timingSince; + } } diff --git a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java index 78a3576420d..174095a2ea4 100644 --- a/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java +++ b/solr/core/src/java/org/apache/solr/uninverting/UninvertingReader.java @@ -228,7 +228,7 @@ public static LeafReader wrap(LeafReader in, Function mapping) { ArrayList newFieldInfos = new ArrayList<>(in.getFieldInfos().size()); for (FieldInfo fi : in.getFieldInfos()) { DocValuesType type = fi.getDocValuesType(); - // fields which currently don't have docValues, but are uninvertable (indexed or points data + // fields which currently don't have docValues, but are uninvertible (indexed or points data // present) if (type == DocValuesType.NONE && (fi.getIndexOptions() != IndexOptions.NONE diff --git a/solr/core/src/java/org/apache/solr/update/TimedVersionBucket.java b/solr/core/src/java/org/apache/solr/update/TimedVersionBucket.java deleted file mode 100644 index cdddede5a79..00000000000 --- a/solr/core/src/java/org/apache/solr/update/TimedVersionBucket.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.update; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.Condition; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; -import org.apache.solr.common.SolrException; - -/** - * This implementation uses lock and condition and will throw exception if it can't obtain the lock - * within lockTimeoutMs. - * - * @lucene.internal - */ -public class TimedVersionBucket extends VersionBucket { - - private final Lock lock = new ReentrantLock(true); - private final Condition condition = lock.newCondition(); - - /** - * This will run the function with the lock. It will throw exception if it can't obtain the lock - * within lockTimeoutMs. - */ - @Override - public R runWithLock(int lockTimeoutMs, CheckedFunction function) - throws IOException { - if (tryLock(lockTimeoutMs)) { - return function.apply(); - } else { - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Unable to get version bucket lock in " + lockTimeoutMs + " ms"); - } - } - - @Override - public void unlock() { - lock.unlock(); - } - - @Override - public void signalAll() { - condition.signalAll(); - } - - @Override - public void awaitNanos(long nanosTimeout) { - try { - if (nanosTimeout > 0) { - condition.awaitNanos(nanosTimeout); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } - } - - protected boolean tryLock(int lockTimeoutMs) { - try { - return lock.tryLock(lockTimeoutMs, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RuntimeException(e); - } - } -} diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLocks.java b/solr/core/src/java/org/apache/solr/update/UpdateLocks.java new file mode 100644 index 00000000000..5fbd27c422c --- /dev/null +++ b/solr/core/src/java/org/apache/solr/update/UpdateLocks.java @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.update; + +import com.carrotsearch.hppc.IntObjectHashMap; +import java.io.IOException; +import java.util.ArrayDeque; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import org.apache.lucene.util.BytesRef; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrException.ErrorCode; +import org.apache.solr.util.IOFunction; + +/** + * Locks associated with updates in connection with the {@link UpdateLog}. + * + * @lucene.internal + */ +public class UpdateLocks { + + private final long docLockTimeoutMs; + + private final ReadWriteLock blockUpdatesLock = new ReentrantReadWriteLock(true); + + // SolrCloud's first approach was a fixed size array of locks (lock striping) using an ID's hash. + // Sized too small, there was too much lock sharing; sized too big, there was memory waste. + // Here we have a Map keyed by hash and a pool of locks to re-use. Synchronization is needed 2x. + // Note: ConcurrentHashMap was also explored but HPPC came out on top, probably because + // we can use a hashcode directly as the key, and it's GC friendly (zero-allocation). + + /** Maps a ID hashcode to a lock. Synchronize to manipulate. */ + private final IntObjectHashMap hashToLock = + new IntObjectHashMap<>(32) { + @Override + protected int hashKey(int key) { + return key; // our keys are themselves hash-codes + } + }; + + /** A pool of locks to avoid creating & GC'ing them too much. Must synchronize on hashToLock. */ + private final ArrayDeque lockPool = new ArrayDeque<>(16); + + public UpdateLocks(long docLockTimeoutMs) { + this.docLockTimeoutMs = docLockTimeoutMs; + } + + /** + * Acquires a lock for the given doc ID, executes the function, and releases the lock. The + * provided {@link Condition} can be used for wait/notify control. The per-doc locking is needed + * indirectly due to SolrCloud's internal out-of-order versioned update design. + */ + public R runWithLock(BytesRef id, IOFunction function) throws IOException { + final var startTimeNanos = System.nanoTime(); + + lockForUpdate(); + try { + // note: if we didn't need a Condition, then we could have reused + // OrderedExecutor.SparseStripedLock over here, which is also a mechanism invented for + // per-doc locking. + + // hashToLock isn't concurrent, but we synchronize on it briefly twice to do cheap work + + final int hash = id.hashCode(); + final LockAndCondition lock; + // get or insert lock, increment refcount + synchronized (hashToLock) { + final int idx = hashToLock.indexOf(hash); + if (hashToLock.indexExists(idx)) { + lock = hashToLock.indexGet(idx); + assert lock.refCount >= 1; + lock.refCount++; + } else { + lock = borrowLock(); + hashToLock.indexInsert(idx, hash, lock); + } + } + + // try-finally ensuring we decrement the refCount + try { + return runWithLockInternal(id, function, lock, startTimeNanos); + } finally { + // decrement refcount, remove lock if unreferenced + synchronized (hashToLock) { + assert lock.refCount > 0; // because we incremented it + if (--lock.refCount == 0) { // typical + hashToLock.remove(hash); + returnLock(lock); + } + } + } + + } finally { + unlockForUpdate(); + } + } + + private LockAndCondition borrowLock() { + assert Thread.holdsLock(hashToLock); + if (lockPool.isEmpty()) { + return new LockAndCondition(); + } else { + return lockPool.removeLast(); + } + } + + private void returnLock(LockAndCondition lock) { + assert Thread.holdsLock(hashToLock); + if (lockPool.size() < 16) { + lockPool.add(lock); + lock.refCount = 1; // ready for next use + } + } + + private R runWithLockInternal( + BytesRef id, IOFunction function, LockAndCondition lock, long startTimeNanos) + throws IOException { + // Acquire the lock + try { + if (docLockTimeoutMs == 0) { + lock.lock.lockInterruptibly(); + } else { + long remainingNs = + TimeUnit.MILLISECONDS.toNanos(docLockTimeoutMs) - (System.nanoTime() - startTimeNanos); + boolean timedOut = !lock.lock.tryLock(remainingNs, TimeUnit.NANOSECONDS); + if (timedOut) { + throw new SolrException( + ErrorCode.SERVER_ERROR, + "Unable to lock doc " + id + " in " + docLockTimeoutMs + " ms"); + } + } + } catch (InterruptedException e) { + // don't set interrupt status; we're ending the request + throw new SolrException(ErrorCode.SERVER_ERROR, "Unable to lock doc " + id, e); + } + // try-finally ensuring we unlock + try { + // We have the lock; do stuff with it + return function.apply(lock.condition); + } finally { + // Release the lock + lock.lock.unlock(); + } + } + + private static class LockAndCondition { + final Lock lock; + final Condition condition; + int refCount; // only access when synchronized on hashToLock + + LockAndCondition() { + lock = new ReentrantLock(true); // fair + condition = lock.newCondition(); + refCount = 1; + } + } + + public void lockForUpdate() { + blockUpdatesLock.readLock().lock(); + } + + public void unlockForUpdate() { + blockUpdatesLock.readLock().unlock(); + } + + public void blockUpdates() { + blockUpdatesLock.writeLock().lock(); + } + + public void unblockUpdates() { + blockUpdatesLock.writeLock().unlock(); + } +} diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java index 4ebe4f311d1..9bc685f276b 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java @@ -66,6 +66,7 @@ import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.CollectionUtil; +import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.IOUtils; import org.apache.solr.common.util.SolrNamedThreadFactory; @@ -223,15 +224,14 @@ public String toString() { protected final int numDeletesByQueryToKeep = 100; protected int numRecordsToKeep; protected int maxNumLogsToKeep; - // This should only be used to initialize VersionInfo... the actual number of buckets may be - // rounded up to a power of two. - protected int numVersionBuckets; protected boolean existOldBufferLog = false; // keep track of deletes only... this is not updated on an add protected LinkedHashMap oldDeletes = new OldDeletesLinkedHashMap(this.numDeletesToKeep); + private UpdateLocks updateLocks; + /** Holds the query and the version for a DeleteByQuery command */ public static class DBQ { public String q; // the query string @@ -331,6 +331,10 @@ public VersionInfo getVersionInfo() { return versionInfo; } + public UpdateLocks getLocks() { + return updateLocks; + } + public int getNumRecordsToKeep() { return numRecordsToKeep; } @@ -339,10 +343,6 @@ public int getMaxNumLogsToKeep() { return maxNumLogsToKeep; } - public int getNumVersionBuckets() { - return numVersionBuckets; - } - protected static int objToInt(Object obj, int def) { if (obj != null) { return Integer.parseInt(obj.toString()); @@ -381,19 +381,25 @@ public void init(PluginInfo info) { numRecordsToKeep = objToInt(info.initArgs.get("numRecordsToKeep"), 100); maxNumLogsToKeep = objToInt(info.initArgs.get("maxNumLogsToKeep"), 10); - numVersionBuckets = objToInt(info.initArgs.get("numVersionBuckets"), 65536); - if (numVersionBuckets <= 0) - throw new SolrException( - SolrException.ErrorCode.SERVER_ERROR, - "Number of version buckets must be greater than 0!"); + if (info.initArgs.get("numVersionBuckets") != null) { + log.warn("numVersionBuckets is obsolete"); + } + + if (info.initArgs.get("versionBucketLockTimeoutMs") != null) { + log.warn("versionBucketLockTimeoutMs is mostly replaced by docLockTimeoutMs"); + } + int timeoutMs = + objToInt( + info.initArgs.get("docLockTimeoutMs", info.initArgs.get("versionBucketLockTimeoutMs")), + EnvUtils.getPropertyAsLong("solr.update.docLockTimeoutMs", 0L).intValue()); + updateLocks = new UpdateLocks(timeoutMs); log.info( - "Initializing UpdateLog: dataDir={} defaultSyncLevel={} numRecordsToKeep={} maxNumLogsToKeep={} numVersionBuckets={}", + "Initializing UpdateLog: dataDir={} defaultSyncLevel={} numRecordsToKeep={} maxNumLogsToKeep={}", dataDir, defaultSyncLevel, numRecordsToKeep, - maxNumLogsToKeep, - numVersionBuckets); + maxNumLogsToKeep); } private final AtomicBoolean initialized = new AtomicBoolean(); @@ -443,7 +449,6 @@ public void init(UpdateHandler uhandler, SolrCore core) { getTlogDir(), id); } - versionInfo.reload(); core.getCoreMetricManager() .registerMetricProducer(SolrInfoBean.Category.TLOG.toString(), this); @@ -464,7 +469,7 @@ public void init(UpdateHandler uhandler, SolrCore core) { usableForChildDocs = core.getLatestSchema().isUsableForChildDocs(); try { - versionInfo = new VersionInfo(this, numVersionBuckets); + versionInfo = new VersionInfo(this); } catch (SolrException e) { log.error("Unable to use updateLog: ", e); throw new SolrException( @@ -1373,7 +1378,7 @@ public Future recoverFromLog() { ExecutorCompletionService cs = new ExecutorCompletionService<>(recoveryExecutor); LogReplayer replayer = new LogReplayer(recoverLogs, false); - versionInfo.blockUpdates(); + updateLocks.blockUpdates(); try { state = State.REPLAYING; @@ -1383,7 +1388,7 @@ public Future recoverFromLog() { deleteByQueries.clear(); oldDeletes.clear(); } finally { - versionInfo.unblockUpdates(); + updateLocks.unblockUpdates(); } // At this point, we are guaranteed that any new updates coming in will see the state as @@ -1409,11 +1414,11 @@ public Future recoverFromCurrentLog() { ExecutorCompletionService cs = new ExecutorCompletionService<>(recoveryExecutor); LogReplayer replayer = new LogReplayer(Collections.singletonList(tlog), false, true); - versionInfo.blockUpdates(); + updateLocks.blockUpdates(); try { state = State.REPLAYING; } finally { - versionInfo.unblockUpdates(); + updateLocks.unblockUpdates(); } return cs.submit(replayer, recoveryInfo); @@ -1427,7 +1432,7 @@ public Future recoverFromCurrentLog() { * @param cuc any updates that have version larger than the version of cuc will be copied over */ public void copyOverBufferingUpdates(CommitUpdateCommand cuc) { - versionInfo.blockUpdates(); + updateLocks.blockUpdates(); try { synchronized (this) { state = State.ACTIVE; @@ -1440,7 +1445,7 @@ public void copyOverBufferingUpdates(CommitUpdateCommand cuc) { dropBufferTlog(); } } finally { - versionInfo.unblockUpdates(); + updateLocks.unblockUpdates(); } } @@ -1451,7 +1456,7 @@ public void copyOverBufferingUpdates(CommitUpdateCommand cuc) { * @param cuc any updates that have version larger than the version of cuc will be copied over */ public void commitAndSwitchToNewTlog(CommitUpdateCommand cuc) { - versionInfo.blockUpdates(); + updateLocks.blockUpdates(); try { synchronized (this) { if (tlog == null) { @@ -1465,7 +1470,7 @@ public void commitAndSwitchToNewTlog(CommitUpdateCommand cuc) { } } } finally { - versionInfo.unblockUpdates(); + updateLocks.unblockUpdates(); } } @@ -1888,7 +1893,7 @@ public void bufferUpdates() { // block all updates to eliminate race conditions // reading state and acting on it in the distributed update processor - versionInfo.blockUpdates(); + updateLocks.blockUpdates(); try { if (state != State.ACTIVE && state != State.BUFFERING) { // we don't currently have support for handling other states @@ -1906,13 +1911,13 @@ public void bufferUpdates() { state = State.BUFFERING; } finally { - versionInfo.unblockUpdates(); + updateLocks.unblockUpdates(); } } /** Returns true if we were able to drop buffered updates and return to the ACTIVE state */ public boolean dropBufferedUpdates() { - versionInfo.blockUpdates(); + updateLocks.blockUpdates(); try { if (state != State.BUFFERING) return false; @@ -1924,7 +1929,7 @@ public boolean dropBufferedUpdates() { state = State.ACTIVE; } finally { - versionInfo.unblockUpdates(); + updateLocks.unblockUpdates(); } return true; } @@ -1946,7 +1951,7 @@ public Future applyBufferedUpdates() { // block all updates to eliminate race conditions // reading state and acting on it in the update processor - versionInfo.blockUpdates(); + updateLocks.blockUpdates(); try { cancelApplyBufferUpdate = false; if (state != State.BUFFERING) return null; @@ -1962,7 +1967,7 @@ public Future applyBufferedUpdates() { state = State.APPLYING_BUFFERED; } finally { - versionInfo.unblockUpdates(); + updateLocks.unblockUpdates(); } if (ExecutorUtil.isShutdown(recoveryExecutor)) { @@ -2048,7 +2053,7 @@ public void run() { // change the state while updates are still blocked to prevent races state = State.ACTIVE; if (finishing) { - versionInfo.unblockUpdates(); + updateLocks.unblockUpdates(); } // clean up in case we hit some unexpected exception and didn't get @@ -2100,7 +2105,7 @@ public void doReplay(TransactionLog translog) { return proc; }); - OrderedExecutor executor = + OrderedExecutor executor = inSortedOrder ? null : req.getCoreContainer().getReplayUpdatesExecutor(); AtomicInteger pendingTasks = new AtomicInteger(0); AtomicReference exceptionOnExecuteUpdate = new AtomicReference<>(); @@ -2147,7 +2152,7 @@ public void doReplay(TransactionLog translog) { // after we've finished this recovery. // NOTE: our own updates won't be blocked since the thread holding a write lock can // lock a read lock. - versionInfo.blockUpdates(); + updateLocks.blockUpdates(); finishing = true; o = tlogReader.next(); } else { @@ -2309,34 +2314,27 @@ private void waitForAllUpdatesGetExecuted(AtomicInteger pendingTasks) { } } - private Integer getBucketHash(UpdateCommand cmd) { + private BytesRef getIndexedId(UpdateCommand cmd) { if (cmd instanceof AddUpdateCommand) { - BytesRef idBytes = ((AddUpdateCommand) cmd).getIndexedId(); - if (idBytes == null) return null; - return DistributedUpdateProcessor.bucketHash(idBytes); + return ((AddUpdateCommand) cmd).getIndexedId(); } - if (cmd instanceof DeleteUpdateCommand) { - BytesRef idBytes = ((DeleteUpdateCommand) cmd).getIndexedId(); - if (idBytes == null) return null; - return DistributedUpdateProcessor.bucketHash(idBytes); + return ((DeleteUpdateCommand) cmd).getIndexedId(); } - return null; } private void execute( UpdateCommand cmd, - OrderedExecutor executor, + OrderedExecutor executor, AtomicInteger pendingTasks, ThreadLocal procTl, AtomicReference exceptionHolder) { assert cmd instanceof AddUpdateCommand || cmd instanceof DeleteUpdateCommand; if (executor != null) { - // by using the same hash as DUP, independent updates can avoid waiting for same bucket executor.execute( - getBucketHash(cmd), + getIndexedId(cmd), () -> { try { // fail fast diff --git a/solr/core/src/java/org/apache/solr/update/VersionBucket.java b/solr/core/src/java/org/apache/solr/update/VersionBucket.java deleted file mode 100644 index 7724b3a2cb4..00000000000 --- a/solr/core/src/java/org/apache/solr/update/VersionBucket.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.update; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; - -// TODO: make inner? -// TODO: store the highest possible in the index on a commit (but how to not block adds?) -// TODO: could also store highest possible in the transaction log after a commit. -// Or on a new index, just scan "version" for the max? -/** - * The default implementation which uses the intrinsic object monitor. It uses less memory but - * ignores the lockTimeoutMs. - * - * @lucene.internal - */ -public class VersionBucket { - - @FunctionalInterface - public interface CheckedFunction { - R apply() throws IOException; - } - - /** This will run the function with the intrinsic object monitor. */ - public R runWithLock(int lockTimeoutMs, CheckedFunction function) - throws IOException { - synchronized (this) { - return function.apply(); - } - } - - /** Nothing to do for the intrinsic object monitor. */ - public void unlock() {} - - public void signalAll() { - notifyAll(); - } - - public void awaitNanos(long nanosTimeout) { - try { - long millis = TimeUnit.NANOSECONDS.toMillis(nanosTimeout); - if (millis > 0) { - wait(millis); - } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } -} diff --git a/solr/core/src/java/org/apache/solr/update/VersionInfo.java b/solr/core/src/java/org/apache/solr/update/VersionInfo.java index 09a39397197..ad7c427c403 100644 --- a/solr/core/src/java/org/apache/solr/update/VersionInfo.java +++ b/solr/core/src/java/org/apache/solr/update/VersionInfo.java @@ -20,11 +20,8 @@ import java.io.IOException; import java.util.Map; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; -import org.apache.lucene.util.BitUtil; import org.apache.lucene.util.BytesRef; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.SuppressForbidden; @@ -33,18 +30,16 @@ import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.util.RefCounted; +/** + * Related to the {@code _version_} field, in connection with the {@link UpdateLog}. + * + * @lucene.internal + */ public class VersionInfo { - private static final String SYS_PROP_BUCKET_VERSION_LOCK_TIMEOUT_MS = - "bucketVersionLockTimeoutMs"; private final UpdateLog ulog; - private final int numBuckets; - private volatile VersionBucket[] buckets; - private final Object bucketsSync = new Object(); - private final SchemaField versionField; - final ReadWriteLock lock = new ReentrantReadWriteLock(true); - private final int versionBucketLockTimeoutMs; + private final SchemaField versionField; /** * Gets and returns the {@link org.apache.solr.common.params.CommonParams#VERSION_FIELD} from the @@ -82,47 +77,16 @@ public static SchemaField getAndCheckVersionField(IndexSchema schema) throws Sol return sf; } - public VersionInfo(UpdateLog ulog, int nBuckets) { + public VersionInfo(UpdateLog ulog) { this.ulog = ulog; IndexSchema schema = ulog.uhandler.core.getLatestSchema(); versionField = getAndCheckVersionField(schema); - versionBucketLockTimeoutMs = - ulog.uhandler - .core - .getSolrConfig() - .get("updateHandler") - .get("versionBucketLockTimeoutMs") - .intVal( - Integer.parseInt(System.getProperty(SYS_PROP_BUCKET_VERSION_LOCK_TIMEOUT_MS, "0"))); - numBuckets = BitUtil.nextHighestPowerOfTwo(nBuckets); } - public int getVersionBucketLockTimeoutMs() { - return versionBucketLockTimeoutMs; - } - - public void reload() {} - public SchemaField getVersionField() { return versionField; } - public void lockForUpdate() { - lock.readLock().lock(); - } - - public void unlockForUpdate() { - lock.readLock().unlock(); - } - - public void blockUpdates() { - lock.writeLock().lock(); - } - - public void unblockUpdates() { - lock.writeLock().unlock(); - } - /* // todo: initialize... use current time to start? // a clock that increments by 1 for every operation makes it easier to detect missing @@ -172,42 +136,6 @@ public long getNewClock() { } } - public long getOldClock() { - synchronized (clockSync) { - return vclock; - } - } - - public void updateClock(long clock) { - synchronized (clockSync) { - vclock = Math.max(vclock, clock); - } - } - - public VersionBucket bucket(int hash) { - // If this is a user provided hash, it may be poor in the right-hand bits. - // Make sure high bits are moved down, since only the low bits will matter. - // int h = hash + (hash >>> 8) + (hash >>> 16) + (hash >>> 24); - // Assume good hash codes for now. - int slot = hash & (numBuckets - 1); - if (buckets == null) { - synchronized (bucketsSync) { - if (buckets == null) { - buckets = createVersionBuckets(); - } - } - } - return buckets[slot]; - } - - private VersionBucket[] createVersionBuckets() { - VersionBucket[] buckets = new VersionBucket[numBuckets]; - for (int i = 0; i < buckets.length; i++) { - buckets[i] = versionBucketLockTimeoutMs > 0 ? new TimedVersionBucket() : new VersionBucket(); - } - return buckets; - } - public Long lookupVersion(BytesRef idBytes) { return ulog.lookupVersion(idBytes); } diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java index 8588a0a34de..d8c88ac5786 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedUpdateProcessor.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Condition; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CharsRefBuilder; import org.apache.solr.client.solrj.SolrRequest; @@ -43,7 +44,6 @@ import org.apache.solr.common.params.ShardParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.params.UpdateParams; -import org.apache.solr.common.util.Hash; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.TimeSource; import org.apache.solr.handler.component.RealTimeGetComponent; @@ -57,9 +57,9 @@ import org.apache.solr.update.SolrCmdDistributor.Node; import org.apache.solr.update.SolrCmdDistributor.SolrError; import org.apache.solr.update.UpdateCommand; +import org.apache.solr.update.UpdateLocks; import org.apache.solr.update.UpdateLog; import org.apache.solr.update.UpdateShardHandler; -import org.apache.solr.update.VersionBucket; import org.apache.solr.update.VersionInfo; import org.apache.solr.util.TestInjection; import org.apache.solr.util.TimeOut; @@ -129,7 +129,7 @@ public static DistribPhase parseParam(final String param) { private final AtomicUpdateDocumentMerger docMerger; private final UpdateLog ulog; - @VisibleForTesting VersionInfo vinfo; + private final VersionInfo vinfo; private final boolean versionsStored; private boolean returnVersions; @@ -202,6 +202,11 @@ public DistributedUpdateProcessor( // this.rsp = reqInfo != null ? reqInfo.getRsp() : null; } + @VisibleForTesting + protected UpdateLocks getUpdateLocks() { + return ulog.getLocks(); + } + /** * @return the replica type of the collection. */ @@ -277,11 +282,6 @@ private void doLocalDelete(DeleteUpdateCommand cmd) throws IOException { isIndexChanged = true; } - public static int bucketHash(BytesRef idBytes) { - assert idBytes != null; - return Hash.murmurhash3_x86_32(idBytes.bytes, idBytes.offset, idBytes.length, 0); - } - /** * @return whether or not to drop this cmd * @throws IOException If there is a low-level I/O error. @@ -305,10 +305,6 @@ protected boolean versionAdd(AddUpdateCommand cmd) throws IOException { } } - // This is only the hash for the bucket, and must be based only on the uniqueKey (i.e. do not - // use a pluggable hash here) - int bucketHash = bucketHash(idBytes); - // at this point, there is an update we need to try and apply. // we may or may not be the leader. @@ -335,36 +331,41 @@ protected boolean versionAdd(AddUpdateCommand cmd) throws IOException { leaderLogicWithVersionIntegrityCheck(isReplayOrPeersync, isLeader, versionOnUpdate); boolean forwardedFromCollection = cmd.getReq().getParams().get(DISTRIB_FROM_COLLECTION) != null; - VersionBucket bucket = vinfo.bucket(bucketHash); - long dependentVersionFound = -1; // if this is an in-place update, check and wait if we should be waiting for a previous update // (on which this update depends), before entering the synchronized block if (!leaderLogic && cmd.isInPlaceUpdate()) { - dependentVersionFound = - waitForDependentUpdates(cmd, versionOnUpdate, isReplayOrPeersync, bucket); + dependentVersionFound = waitForDependentUpdates(cmd, versionOnUpdate, isReplayOrPeersync); if (dependentVersionFound == -1) { // it means the document has been deleted by now at the leader. drop this update return true; } } - vinfo.lockForUpdate(); - try { - long finalVersionOnUpdate = versionOnUpdate; - return bucket.runWithLock( - vinfo.getVersionBucketLockTimeoutMs(), - () -> - doVersionAdd( + final long finalVersionOnUpdate = versionOnUpdate; + return getUpdateLocks() + .runWithLock( + cmd.getIndexedId(), + (Condition condition) -> { + // just in case anyone is waiting let them know that we have a new update + // we obtain the version when synchronized and then do the add so we can ensure that + // if version1 < version2 then version1 is actually added before version2. + + // even if we don't store the version field, synchronizing + // will enable us to know what version happened first, and thus enable + // realtime-get to work reliably. + // TODO: if versions aren't stored, do we need to set on the cmd anyway for some + // reason? + // there may be other reasons in the future for a version on the commands + condition.signalAll(); + + return doVersionAdd( cmd, finalVersionOnUpdate, isReplayOrPeersync, leaderLogic, - forwardedFromCollection, - bucket)); - } finally { - vinfo.unlockForUpdate(); - } + forwardedFromCollection); + }); } private boolean doVersionAdd( @@ -372,172 +373,157 @@ private boolean doVersionAdd( long versionOnUpdate, boolean isReplayOrPeersync, boolean leaderLogic, - boolean forwardedFromCollection, - VersionBucket bucket) + boolean forwardedFromCollection) throws IOException { - try { - BytesRef idBytes = cmd.getIndexedId(); - bucket.signalAll(); - // just in case anyone is waiting let them know that we have a new update - // we obtain the version when synchronized and then do the add so we can ensure that - // if version1 < version2 then version1 is actually added before version2. - - // even if we don't store the version field, synchronizing on the bucket - // will enable us to know what version happened first, and thus enable - // realtime-get to work reliably. - // TODO: if versions aren't stored, do we need to set on the cmd anyway for some reason? - // there may be other reasons in the future for a version on the commands - - if (versionsStored) { - - if (leaderLogic) { - - if (forwardedFromCollection && ulog.getState() == UpdateLog.State.ACTIVE) { - // forwarded from a collection but we are not buffering so strip original version and - // apply our own - // see SOLR-5308 - if (log.isInfoEnabled()) { - log.info("Removing version field from doc: {}", cmd.getPrintableId()); - } - cmd.solrDoc.remove(CommonParams.VERSION_FIELD); - versionOnUpdate = 0; - } + BytesRef idBytes = cmd.getIndexedId(); - getUpdatedDocument(cmd, versionOnUpdate); + if (versionsStored) { - // leaders can also be in buffering state during "migrate" API call, see SOLR-5308 - if (forwardedFromCollection - && ulog.getState() != UpdateLog.State.ACTIVE - && isReplayOrPeersync == false) { - // we're not in an active state, and this update isn't from a replay, so buffer it. - if (log.isInfoEnabled()) { - log.info( - "Leader logic applied but update log is buffering: {}", cmd.getPrintableId()); - } - cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING); - ulog.add(cmd); - return true; + if (leaderLogic) { + + if (forwardedFromCollection && ulog.getState() == UpdateLog.State.ACTIVE) { + // forwarded from a collection but we are not buffering so strip original version and + // apply our own + // see SOLR-5308 + if (log.isInfoEnabled()) { + log.info("Removing version field from doc: {}", cmd.getPrintableId()); } + cmd.solrDoc.remove(CommonParams.VERSION_FIELD); + versionOnUpdate = 0; + } - if (versionOnUpdate != 0) { - Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId()); - long foundVersion = lastVersion == null ? -1 : lastVersion; - if (versionOnUpdate == foundVersion - || (versionOnUpdate < 0 && foundVersion < 0) - || (versionOnUpdate == 1 && foundVersion > 0)) { - // we're ok if versions match, or if both are negative (all missing docs are equal), - // or if cmd specified it must exist (versionOnUpdate==1) and it does. - } else { - if (cmd.getReq().getParams().getBool(CommonParams.FAIL_ON_VERSION_CONFLICTS, true) - == false) { - return true; - } + getUpdatedDocument(cmd, versionOnUpdate); - throw new SolrException( - ErrorCode.CONFLICT, - "version conflict for " - + cmd.getPrintableId() - + " expected=" - + versionOnUpdate - + " actual=" - + foundVersion); - } + // leaders can also be in buffering state during "migrate" API call, see SOLR-5308 + if (forwardedFromCollection + && ulog.getState() != UpdateLog.State.ACTIVE + && isReplayOrPeersync == false) { + // we're not in an active state, and this update isn't from a replay, so buffer it. + if (log.isInfoEnabled()) { + log.info("Leader logic applied but update log is buffering: {}", cmd.getPrintableId()); } + cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING); + ulog.add(cmd); + return true; + } - long version = vinfo.getNewClock(); - cmd.setVersion(version); - cmd.getSolrInputDocument().setField(CommonParams.VERSION_FIELD, version); - } else { - // The leader forwarded us this update. - cmd.setVersion(versionOnUpdate); + if (versionOnUpdate != 0) { + Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId()); + long foundVersion = lastVersion == null ? -1 : lastVersion; + if (versionOnUpdate == foundVersion + || (versionOnUpdate < 0 && foundVersion < 0) + || (versionOnUpdate == 1 && foundVersion > 0)) { + // we're ok if versions match, or if both are negative (all missing docs are equal), + // or if cmd specified it must exist (versionOnUpdate==1) and it does. + } else { + if (cmd.getReq().getParams().getBool(CommonParams.FAIL_ON_VERSION_CONFLICTS, true) + == false) { + return true; + } - if (shouldBufferUpdate(cmd, isReplayOrPeersync, ulog.getState())) { - // we're not in an active state, and this update isn't from a replay, so buffer it. - cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING); - ulog.add(cmd); - return true; + throw new SolrException( + ErrorCode.CONFLICT, + "version conflict for " + + cmd.getPrintableId() + + " expected=" + + versionOnUpdate + + " actual=" + + foundVersion); } + } - if (cmd.isInPlaceUpdate()) { - long prev = cmd.prevVersion; - Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId()); - if (lastVersion == null || Math.abs(lastVersion) < prev) { - // this was checked for (in waitForDependentUpdates()) before entering the - // synchronized block. So we shouldn't be here, unless what must've happened is: by - // the time synchronization block was entered, the prev update was deleted by DBQ. - // Since now that update is not in index, the vinfo.lookupVersion() is possibly giving - // us a version from the deleted list (which might be older than the prev update!) - UpdateCommand fetchedFromLeader = fetchFullUpdateFromLeader(cmd, versionOnUpdate); - - if (fetchedFromLeader instanceof DeleteUpdateCommand) { - if (log.isInfoEnabled()) { - log.info( - "In-place update of {} failed to find valid lastVersion to apply to, and the document was deleted at the leader subsequently.", - idBytes.utf8ToString()); - } - versionDelete((DeleteUpdateCommand) fetchedFromLeader); - return true; - } else { - assert fetchedFromLeader instanceof AddUpdateCommand; - // Newer document was fetched from the leader. Apply that document instead of this - // current in-place update. - if (log.isInfoEnabled()) { - log.info( - "In-place update of {} failed to find valid lastVersion to apply to, forced to fetch full doc from leader: {}", - idBytes.utf8ToString(), - fetchedFromLeader); - } - // Make this update to become a non-inplace update containing the full document - // obtained from the leader - cmd.solrDoc = ((AddUpdateCommand) fetchedFromLeader).solrDoc; - cmd.prevVersion = -1; - cmd.setVersion((long) cmd.solrDoc.getFieldValue(CommonParams.VERSION_FIELD)); - assert cmd.isInPlaceUpdate() == false; + long version = vinfo.getNewClock(); + cmd.setVersion(version); + cmd.getSolrInputDocument().setField(CommonParams.VERSION_FIELD, version); + } else { + // The leader forwarded us this update. + cmd.setVersion(versionOnUpdate); + + if (shouldBufferUpdate(cmd, isReplayOrPeersync, ulog.getState())) { + // we're not in an active state, and this update isn't from a replay, so buffer it. + cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING); + ulog.add(cmd); + return true; + } + + if (cmd.isInPlaceUpdate()) { + long prev = cmd.prevVersion; + Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId()); + if (lastVersion == null || Math.abs(lastVersion) < prev) { + // this was checked for (in waitForDependentUpdates()) before entering the + // synchronized block. So we shouldn't be here, unless what must've happened is: by + // the time synchronization block was entered, the prev update was deleted by DBQ. + // Since now that update is not in index, the vinfo.lookupVersion() is possibly giving + // us a version from the deleted list (which might be older than the prev update!) + UpdateCommand fetchedFromLeader = fetchFullUpdateFromLeader(cmd, versionOnUpdate); + + if (fetchedFromLeader instanceof DeleteUpdateCommand) { + if (log.isInfoEnabled()) { + log.info( + "In-place update of {} failed to find valid lastVersion to apply to, and the document was deleted at the leader subsequently.", + idBytes.utf8ToString()); } + versionDelete((DeleteUpdateCommand) fetchedFromLeader); + return true; } else { - if (Math.abs(lastVersion) > prev) { - // this means we got a newer full doc update and in that case it makes no sense to - // apply the older inplace update. Drop this update + assert fetchedFromLeader instanceof AddUpdateCommand; + // Newer document was fetched from the leader. Apply that document instead of this + // current in-place update. + if (log.isInfoEnabled()) { log.info( - "Update was applied on version: {}, but last version I have is: {}. Dropping current update", - prev, - lastVersion); - return true; - } else { - // We're good, we should apply this update. + "In-place update of {} failed to find valid lastVersion to apply to, forced to fetch full doc from leader: {}", + idBytes.utf8ToString(), + fetchedFromLeader); } + // Make this update to become a non-inplace update containing the full document + // obtained from the leader + cmd.solrDoc = ((AddUpdateCommand) fetchedFromLeader).solrDoc; + cmd.prevVersion = -1; + cmd.setVersion((long) cmd.solrDoc.getFieldValue(CommonParams.VERSION_FIELD)); + assert cmd.isInPlaceUpdate() == false; } } else { - // if we aren't the leader, then we need to check that updates were not re-ordered - // we need to check the specific version for this id. - Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId()); - if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) { - // This update is a repeat, or was reordered. We need to drop this update. - if (log.isDebugEnabled()) { - log.debug("Dropping add update due to version {}", idBytes.utf8ToString()); - } + if (Math.abs(lastVersion) > prev) { + // this means we got a newer full doc update and in that case it makes no sense to + // apply the older inplace update. Drop this update + log.info( + "Update was applied on version: {}, but last version I have is: {}. Dropping current update", + prev, + lastVersion); return true; + } else { + // We're good, we should apply this update. } } - if (!isSubShardLeader - && replicaType == Replica.Type.TLOG - && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) { - cmd.setFlags(cmd.getFlags() | UpdateCommand.IGNORE_INDEXWRITER); + } else { + // if we aren't the leader, then we need to check that updates were not re-ordered + // we need to check the specific version for this id. + Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId()); + if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) { + // This update is a repeat, or was reordered. We need to drop this update. + if (log.isDebugEnabled()) { + log.debug("Dropping add update due to version {}", idBytes.utf8ToString()); + } + return true; } } + if (!isSubShardLeader + && replicaType == Replica.Type.TLOG + && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) { + cmd.setFlags(cmd.getFlags() | UpdateCommand.IGNORE_INDEXWRITER); + } } + } - SolrInputDocument clonedDoc = shouldCloneCmdDoc() ? cmd.solrDoc.deepCopy() : null; + SolrInputDocument clonedDoc = shouldCloneCmdDoc() ? cmd.solrDoc.deepCopy() : null; - // TODO: possibly set checkDeleteByQueries as a flag on the command? - doLocalAdd(cmd); + // TODO: possibly set checkDeleteByQueries as a flag on the command? + doLocalAdd(cmd); - if (clonedDoc != null) { - cmd.solrDoc = clonedDoc; - } - } finally { - bucket.unlock(); + if (clonedDoc != null) { + cmd.solrDoc = clonedDoc; } + return false; } @@ -573,22 +559,16 @@ boolean shouldBufferUpdate( * has been indexed. */ private long waitForDependentUpdates( - AddUpdateCommand cmd, long versionOnUpdate, boolean isReplayOrPeersync, VersionBucket bucket) - throws IOException { - long lastFoundVersion = 0; + AddUpdateCommand cmd, long versionOnUpdate, boolean isReplayOrPeersync) throws IOException { TimeOut waitTimeout = new TimeOut(5, TimeUnit.SECONDS, TimeSource.NANO_TIME); - vinfo.lockForUpdate(); - try { - lastFoundVersion = - bucket.runWithLock( - vinfo.getVersionBucketLockTimeoutMs(), - () -> - doWaitForDependentUpdates( - cmd, versionOnUpdate, isReplayOrPeersync, bucket, waitTimeout)); - } finally { - vinfo.unlockForUpdate(); - } + long lastFoundVersion = + getUpdateLocks() + .runWithLock( + cmd.getIndexedId(), + (Condition condition) -> + doWaitForDependentUpdates( + cmd, versionOnUpdate, isReplayOrPeersync, condition, waitTimeout)); if (Math.abs(lastFoundVersion) > cmd.prevVersion) { // This must've been the case due to a higher version full update succeeding concurrently, @@ -652,35 +632,34 @@ private long doWaitForDependentUpdates( AddUpdateCommand cmd, long versionOnUpdate, boolean isReplayOrPeersync, - VersionBucket bucket, + Condition condition, TimeOut waitTimeout) { - long lastFoundVersion; - try { - Long lookedUpVersion = vinfo.lookupVersion(cmd.getIndexedId()); - lastFoundVersion = lookedUpVersion == null ? 0L : lookedUpVersion; + Long lookedUpVersion = vinfo.lookupVersion(cmd.getIndexedId()); + long lastFoundVersion = lookedUpVersion == null ? 0L : lookedUpVersion; - if (Math.abs(lastFoundVersion) < cmd.prevVersion) { - if (log.isDebugEnabled()) { - log.debug( - "Re-ordered inplace update. version={}, prevVersion={}, lastVersion={}, replayOrPeerSync={}, id={}", - (cmd.getVersion() == 0 ? versionOnUpdate : cmd.getVersion()), - cmd.prevVersion, - lastFoundVersion, - isReplayOrPeersync, - cmd.getPrintableId()); - } + if (Math.abs(lastFoundVersion) < cmd.prevVersion) { + if (log.isDebugEnabled()) { + log.debug( + "Re-ordered inplace update. version={}, prevVersion={}, lastVersion={}, replayOrPeerSync={}, id={}", + (cmd.getVersion() == 0 ? versionOnUpdate : cmd.getVersion()), + cmd.prevVersion, + lastFoundVersion, + isReplayOrPeersync, + cmd.getPrintableId()); } + } - while (Math.abs(lastFoundVersion) < cmd.prevVersion && !waitTimeout.hasTimedOut()) { - long timeLeftInNanos = waitTimeout.timeLeft(TimeUnit.NANOSECONDS); - if (timeLeftInNanos > 0) { // 0 means: wait forever until notified, but we don't want that. - bucket.awaitNanos(timeLeftInNanos); + while (Math.abs(lastFoundVersion) < cmd.prevVersion && !waitTimeout.hasTimedOut()) { + long timeLeftInNanos = waitTimeout.timeLeft(TimeUnit.NANOSECONDS); + if (timeLeftInNanos > 0) { // 0 means: wait forever until notified, but we don't want that. + try { + condition.await(timeLeftInNanos, TimeUnit.NANOSECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); } - lookedUpVersion = vinfo.lookupVersion(cmd.getIndexedId()); - lastFoundVersion = lookedUpVersion == null ? 0L : lookedUpVersion; } - } finally { - bucket.unlock(); + lookedUpVersion = vinfo.lookupVersion(cmd.getIndexedId()); + lastFoundVersion = lookedUpVersion == null ? 0L : lookedUpVersion; } return lastFoundVersion; } @@ -958,7 +937,7 @@ protected void versionDeleteByQuery(DeleteUpdateCommand cmd) throws IOException boolean leaderLogic = leaderLogicWithVersionIntegrityCheck(isReplayOrPeersync, isLeader, versionOnUpdate); - vinfo.blockUpdates(); + getUpdateLocks().blockUpdates(); try { doLocalDeleteByQuery(cmd, versionOnUpdate, isReplayOrPeersync, leaderLogic); @@ -968,7 +947,7 @@ protected void versionDeleteByQuery(DeleteUpdateCommand cmd) throws IOException // IndexReader (so cache misses will see up-to-date data) } finally { - vinfo.unblockUpdates(); + getUpdateLocks().unblockUpdates(); } } @@ -1040,10 +1019,6 @@ protected boolean versionDelete(DeleteUpdateCommand cmd) throws IOException { return false; } - // This is only the hash for the bucket, and must be based only on the uniqueKey (i.e. do not - // use a pluggable hash here) - int bucketHash = bucketHash(idBytes); - // at this point, there is an update we need to try and apply. // we may or may not be the leader. @@ -1062,25 +1037,18 @@ protected boolean versionDelete(DeleteUpdateCommand cmd) throws IOException { leaderLogicWithVersionIntegrityCheck(isReplayOrPeersync, isLeader, versionOnUpdate); boolean forwardedFromCollection = cmd.getReq().getParams().get(DISTRIB_FROM_COLLECTION) != null; - VersionBucket bucket = vinfo.bucket(bucketHash); - - vinfo.lockForUpdate(); - try { - long finalVersionOnUpdate = versionOnUpdate; - return bucket.runWithLock( - vinfo.getVersionBucketLockTimeoutMs(), - () -> - doVersionDelete( - cmd, - finalVersionOnUpdate, - signedVersionOnUpdate, - isReplayOrPeersync, - leaderLogic, - forwardedFromCollection, - bucket)); - } finally { - vinfo.unlockForUpdate(); - } + long finalVersionOnUpdate = versionOnUpdate; + return getUpdateLocks() + .runWithLock( + cmd.getIndexedId(), + (Condition unused) -> + doVersionDelete( + cmd, + finalVersionOnUpdate, + signedVersionOnUpdate, + isReplayOrPeersync, + leaderLogic, + forwardedFromCollection)); } private boolean doVersionDelete( @@ -1089,94 +1057,89 @@ private boolean doVersionDelete( long signedVersionOnUpdate, boolean isReplayOrPeersync, boolean leaderLogic, - boolean forwardedFromCollection, - VersionBucket bucket) + boolean forwardedFromCollection) throws IOException { - try { - BytesRef idBytes = cmd.getIndexedId(); - if (versionsStored) { + BytesRef idBytes = cmd.getIndexedId(); + if (versionsStored) { - if (leaderLogic) { + if (leaderLogic) { - if (forwardedFromCollection && ulog.getState() == UpdateLog.State.ACTIVE) { - // forwarded from a collection but we are not buffering so strip original version and - // apply our own - // see SOLR-5308 - if (log.isInfoEnabled()) { - log.info("Removing version field from doc: {}", cmd.getId()); - } - versionOnUpdate = signedVersionOnUpdate = 0; + if (forwardedFromCollection && ulog.getState() == UpdateLog.State.ACTIVE) { + // forwarded from a collection but we are not buffering so strip original version and + // apply our own + // see SOLR-5308 + if (log.isInfoEnabled()) { + log.info("Removing version field from doc: {}", cmd.getId()); } + versionOnUpdate = signedVersionOnUpdate = 0; + } - // leaders can also be in buffering state during "migrate" API call, see SOLR-5308 - if (forwardedFromCollection - && ulog.getState() != UpdateLog.State.ACTIVE - && !isReplayOrPeersync) { - // we're not in an active state, and this update isn't from a replay, so buffer it. - if (log.isInfoEnabled()) { - log.info("Leader logic applied but update log is buffering: {}", cmd.getId()); - } - cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING); - ulog.delete(cmd); - return true; + // leaders can also be in buffering state during "migrate" API call, see SOLR-5308 + if (forwardedFromCollection + && ulog.getState() != UpdateLog.State.ACTIVE + && !isReplayOrPeersync) { + // we're not in an active state, and this update isn't from a replay, so buffer it. + if (log.isInfoEnabled()) { + log.info("Leader logic applied but update log is buffering: {}", cmd.getId()); } + cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING); + ulog.delete(cmd); + return true; + } - if (signedVersionOnUpdate != 0) { - Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId()); - long foundVersion = lastVersion == null ? -1 : lastVersion; - if ((signedVersionOnUpdate == foundVersion) - || (signedVersionOnUpdate < 0 && foundVersion < 0) - || (signedVersionOnUpdate == 1 && foundVersion > 0)) { - // we're ok if versions match, or if both are negative (all missing docs are equal), - // or if cmd specified it must exist (versionOnUpdate==1) and it does. - } else { - throw new SolrException( - ErrorCode.CONFLICT, - "version conflict for " - + cmd.getId() - + " expected=" - + signedVersionOnUpdate - + " actual=" - + foundVersion); - } + if (signedVersionOnUpdate != 0) { + Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId()); + long foundVersion = lastVersion == null ? -1 : lastVersion; + if ((signedVersionOnUpdate == foundVersion) + || (signedVersionOnUpdate < 0 && foundVersion < 0) + || (signedVersionOnUpdate == 1 && foundVersion > 0)) { + // we're ok if versions match, or if both are negative (all missing docs are equal), + // or if cmd specified it must exist (versionOnUpdate==1) and it does. + } else { + throw new SolrException( + ErrorCode.CONFLICT, + "version conflict for " + + cmd.getId() + + " expected=" + + signedVersionOnUpdate + + " actual=" + + foundVersion); } + } - long version = vinfo.getNewClock(); - cmd.setVersion(-version); - } else { - cmd.setVersion(-versionOnUpdate); + long version = vinfo.getNewClock(); + cmd.setVersion(-version); + } else { + cmd.setVersion(-versionOnUpdate); - if (ulog.getState() != UpdateLog.State.ACTIVE && isReplayOrPeersync == false) { - // we're not in an active state, and this update isn't from a replay, so buffer it. - cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING); - ulog.delete(cmd); - return true; - } + if (ulog.getState() != UpdateLog.State.ACTIVE && isReplayOrPeersync == false) { + // we're not in an active state, and this update isn't from a replay, so buffer it. + cmd.setFlags(cmd.getFlags() | UpdateCommand.BUFFERING); + ulog.delete(cmd); + return true; + } - // if we aren't the leader, then we need to check that updates were not re-ordered - // we need to check the specific version for this id. - Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId()); - if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) { - // This update is a repeat, or was reordered. We need to drop this update. - if (log.isDebugEnabled()) { - log.debug("Dropping delete update due to version {}", idBytes.utf8ToString()); - } - return true; + // if we aren't the leader, then we need to check that updates were not re-ordered + // we need to check the specific version for this id. + Long lastVersion = vinfo.lookupVersion(cmd.getIndexedId()); + if (lastVersion != null && Math.abs(lastVersion) >= versionOnUpdate) { + // This update is a repeat, or was reordered. We need to drop this update. + if (log.isDebugEnabled()) { + log.debug("Dropping delete update due to version {}", idBytes.utf8ToString()); } + return true; + } - if (!isSubShardLeader - && replicaType == Replica.Type.TLOG - && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) { - cmd.setFlags(cmd.getFlags() | UpdateCommand.IGNORE_INDEXWRITER); - } + if (!isSubShardLeader + && replicaType == Replica.Type.TLOG + && (cmd.getFlags() & UpdateCommand.REPLAY) == 0) { + cmd.setFlags(cmd.getFlags() | UpdateCommand.IGNORE_INDEXWRITER); } } - - doLocalDelete(cmd); - return false; - } finally { - bucket.unlock(); } + + doLocalDelete(cmd); + return false; } private static boolean leaderLogicWithVersionIntegrityCheck( @@ -1202,7 +1165,7 @@ protected void doLocalCommit(CommitUpdateCommand cmd) throws IOException { if (vinfo != null) { long commitVersion = vinfo.getNewClock(); cmd.setVersion(commitVersion); - vinfo.lockForUpdate(); + getUpdateLocks().lockForUpdate(); } try { @@ -1221,7 +1184,7 @@ protected void doLocalCommit(CommitUpdateCommand cmd) throws IOException { } finally { if (vinfo != null) { - vinfo.unlockForUpdate(); + getUpdateLocks().unlockForUpdate(); } } } diff --git a/solr/core/src/java/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactory.java index 31894b9fb49..4e55fc3e01b 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactory.java +++ b/solr/core/src/java/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactory.java @@ -191,7 +191,7 @@ boolean isSkipUpdateIfMissing() { return this.skipUpdateIfMissing; } - boolean doesDocumentExist(BytesRef indexedDocId) throws IOException { + boolean doesDocumentExist(BytesRef indexedDocId) { assert null != indexedDocId; // we don't need any fields populated, we just need to know if the doc is in the tlog... @@ -224,6 +224,17 @@ boolean doesDocumentExist(BytesRef indexedDocId) throws IOException { } } + boolean doesChildDocumentExist(AddUpdateCommand cmd) throws IOException { + return RealTimeGetComponent.getInputDocument( + core, + core.getLatestSchema().indexableUniqueKey(cmd.getSelfOrNestedDocIdStr()), + cmd.getIndexedId(), + null, + null, + RealTimeGetComponent.Resolution.DOC) + != null; + } + boolean isLeader(UpdateCommand cmd) { if ((cmd.getFlags() & (UpdateCommand.REPLAY | UpdateCommand.PEER_SYNC)) != 0) { return false; @@ -241,17 +252,10 @@ public void processAdd(AddUpdateCommand cmd) throws IOException { boolean isUpdate = AtomicUpdateDocumentMerger.isAtomicUpdate(cmd); - // boolean existsByLookup = (RealTimeGetComponent.getInputDocument(core, indexedDocId) != - // null); - // if (docExists != existsByLookup) { - // log.error("Found docExists {} but existsByLookup {} for doc {}", docExists, - // existsByLookup, indexedDocId.utf8ToString()); - // } - if (log.isDebugEnabled()) { log.debug( "Document ID {} ... exists already? {} ... isAtomicUpdate? {} ... isLeader? {}", - indexedDocId.utf8ToString(), + cmd.getPrintableId(), doesDocumentExist(indexedDocId), isUpdate, isLeader(cmd)); @@ -259,20 +263,35 @@ public void processAdd(AddUpdateCommand cmd) throws IOException { if (skipInsertIfExists && !isUpdate && isLeader(cmd) && doesDocumentExist(indexedDocId)) { if (log.isDebugEnabled()) { - log.debug("Skipping insert for pre-existing document ID {}", indexedDocId.utf8ToString()); + log.debug("Skipping insert for pre-existing document ID {}", cmd.getPrintableId()); } return; } - if (skipUpdateIfMissing && isUpdate && isLeader(cmd) && !doesDocumentExist(indexedDocId)) { - if (log.isDebugEnabled()) { - log.debug("Skipping update to non-existent document ID {}", indexedDocId.utf8ToString()); + if (skipUpdateIfMissing && isUpdate && isLeader(cmd)) { + if (!cmd.getSelfOrNestedDocIdStr().equals(cmd.getIndexedIdStr())) { + // must be a child document + if (!doesChildDocumentExist(cmd)) { + if (log.isDebugEnabled()) { + log.debug( + "Skipping update to non-existent child document ID {}", + cmd.getSelfOrNestedDocIdStr()); + } + return; + } + } else { + // not a child document + if (!doesDocumentExist(indexedDocId)) { + if (log.isDebugEnabled()) { + log.debug("Skipping update to non-existent document ID {}", cmd.getPrintableId()); + } + return; + } } - return; } if (log.isDebugEnabled()) { - log.debug("Passing on document ID {}", indexedDocId.utf8ToString()); + log.debug("Passing on document ID {}", cmd.getPrintableId()); } super.processAdd(cmd); diff --git a/solr/core/src/java/org/apache/solr/util/OrderedExecutor.java b/solr/core/src/java/org/apache/solr/util/OrderedExecutor.java index 65d2491ed1e..3e3abb4e312 100644 --- a/solr/core/src/java/org/apache/solr/util/OrderedExecutor.java +++ b/solr/core/src/java/org/apache/solr/util/OrderedExecutor.java @@ -25,9 +25,9 @@ import java.util.concurrent.Semaphore; import org.apache.solr.common.util.ExecutorUtil; -public class OrderedExecutor implements Executor { +public class OrderedExecutor implements Executor { private final ExecutorService delegate; - private final SparseStripedLock sparseStripedLock; + private final SparseStripedLock sparseStripedLock; public OrderedExecutor(int numThreads, ExecutorService delegate) { this.delegate = delegate; @@ -51,7 +51,7 @@ public void execute(Runnable runnable) { * @param command the runnable task * @throws RejectedExecutionException if this task cannot be accepted for execution */ - public void execute(Integer lockId, Runnable command) { + public void execute(T lockId, Runnable command) { try { sparseStripedLock.add(lockId); } catch (InterruptedException e) { diff --git a/solr/core/src/java/org/apache/solr/util/TestInjection.java b/solr/core/src/java/org/apache/solr/util/TestInjection.java index 4f1547ccd00..8168f7beafe 100644 --- a/solr/core/src/java/org/apache/solr/util/TestInjection.java +++ b/solr/core/src/java/org/apache/solr/util/TestInjection.java @@ -16,11 +16,18 @@ */ package org.apache.solr.util; +import com.google.common.util.concurrent.AtomicDouble; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.lang.reflect.Method; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; import java.util.Collections; import java.util.HashSet; +import java.util.Locale; import java.util.Random; import java.util.Set; import java.util.Timer; @@ -32,13 +39,13 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.QueryTimeout; import org.apache.solr.common.NonExistentCoreException; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.util.Pair; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; +import org.apache.solr.search.QueryLimit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -154,10 +161,26 @@ static Random random() { // non-private for testing public static volatile AtomicInteger countDocSetDelays = new AtomicInteger(0); - public static volatile QueryTimeout queryTimeout = null; + public static volatile QueryLimit queryTimeout = null; public static volatile boolean failInExecutePlanAction = false; + public static volatile AtomicInteger cpuTimerDelayInjectedNS = null; + + private static final KeyPairGenerator kpg; + + static { + KeyPairGenerator generator; + try { + generator = KeyPairGenerator.getInstance("RSA"); + } catch (NoSuchAlgorithmException e) { + generator = null; + } + kpg = generator; + } + + private static volatile AtomicDouble cpuLoadPerKey = null; + /** * Defaults to false, If set to true, then {@link * #injectSkipIndexWriterCommitOnClose} will return true @@ -531,6 +554,55 @@ public static boolean injectDirectUpdateLatch() { return true; } + public static void measureCpu() { + if (kpg == null || cpuLoadPerKey != null) { + return; + } + long start = System.nanoTime(); + for (int i = 0; i < 100; i++) { + genKeyPairAndDiscard(); + } + // note that this is potentially imprecise because our thread could get paused in the middle of + // this, but + // it should give us some notion + long end = System.nanoTime(); + cpuLoadPerKey = new AtomicDouble((end - start) / 100.0); + log.info("CPU per key = {}", cpuLoadPerKey); + } + + private static void genKeyPairAndDiscard() { + kpg.initialize(1024); + KeyPair kp = kpg.generateKeyPair(); + // avoid this getting optimized away by logging it + if (log.isTraceEnabled()) { + log.trace("{}", kp.getPublic()); + } + } + + private static void wasteCpu(int nanos) { + double wasteMe = nanos; + double loadPerKey = cpuLoadPerKey.get(); + if (loadPerKey > nanos) { + java.text.DecimalFormatSymbols symbols = new DecimalFormatSymbols(Locale.US); + + DecimalFormat formatter = new DecimalFormat("#,###.00", symbols); + // yes this is still wasting formatting when not warn, but not important here. + log.warn( + "Test requests smaller simulated cpu lag than a single keypair generation actual lag is {} ns", + formatter.format(loadPerKey)); + } + do { + genKeyPairAndDiscard(); + } while ((wasteMe = wasteMe - loadPerKey) > 0.0); + } + + public static void injectCpuUseInSearcherCpuLimitCheck() { + if (LUCENE_TEST_CASE == null) return; + if (cpuTimerDelayInjectedNS != null) { + wasteCpu(cpuTimerDelayInjectedNS.get()); + } + } + public static boolean injectReindexFailure() { if (reindexFailure != null) { Random rand = random(); diff --git a/solr/core/src/java/org/apache/solr/util/ThreadCpuTimer.java b/solr/core/src/java/org/apache/solr/util/ThreadCpuTimer.java index 054a5310d14..17d61fe3748 100644 --- a/solr/core/src/java/org/apache/solr/util/ThreadCpuTimer.java +++ b/solr/core/src/java/org/apache/solr/util/ThreadCpuTimer.java @@ -19,24 +19,19 @@ import java.lang.invoke.MethodHandles; import java.lang.management.ManagementFactory; import java.lang.management.ThreadMXBean; +import java.util.Map; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -import net.jcip.annotations.NotThreadSafe; +import java.util.concurrent.atomic.AtomicLong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** * Allows tracking information about the current thread using the JVM's built-in management bean - * {@link java.lang.management.ThreadMXBean}. - * - *

Calling code should create an instance of this class when starting the operation, and then can - * get the {@link #getElapsedCpuMs()} at any time thereafter. - * - *

This class is irrevocably not thread safe. Never allow instances of this class to be exposed - * to more than one thread. Acquiring an external lock will not be sufficient. This class can be - * considered "lock-hostile" due to its caching of timing information for a specific thread. + * {@link java.lang.management.ThreadMXBean}. Methods on this class are safe for use on any thread, + * but will return different values for different threads by design. */ -@NotThreadSafe public class ThreadCpuTimer { private static final long UNSUPPORTED = -1; public static final String CPU_TIME = "cpuTime"; @@ -59,14 +54,14 @@ public class ThreadCpuTimer { } } - private final long startCpuTimeNanos; + private static final ThreadLocal> threadLocalTimer = + ThreadLocal.withInitial(ConcurrentHashMap::new); - /** - * Create an instance to track the current thread's usage of CPU. Usage information can later be - * retrieved by calling {@link #getElapsedCpuMs()}. Timing starts immediately upon construction. - */ - public ThreadCpuTimer() { - this.startCpuTimeNanos = getThreadTotalCpuNs(); + /* no instances shall be created. */ + private ThreadCpuTimer() {} + + public static void beginContext(String context) { + readNSAndReset(context); } public static boolean isSupported() { @@ -74,47 +69,64 @@ public static boolean isSupported() { } /** - * Return CPU time consumed by this thread since the construction of this timer object. + * Get the number of nanoseconds since the last time this thread took a reading + * for the supplied context. * - * @return current value, or {@link #UNSUPPORTED} if not supported. + * @param context An arbitrary name that code can supply to avoid clashing with other usages. + * @return An optional long which may be empty if + * java.lang.management.ManagementFactory#getThreadMXBean() is unsupported or otherwise + * unavailable. */ - public long getElapsedCpuNs() { - return this.startCpuTimeNanos != UNSUPPORTED - ? getThreadTotalCpuNs() - this.startCpuTimeNanos - : UNSUPPORTED; + public static Optional readNSAndReset(String context) { + // simulate heavy query and/or heavy CPU load in tests + TestInjection.injectCpuUseInSearcherCpuLimitCheck(); + if (THREAD_MX_BEAN == null) { + return Optional.empty(); + } else { + AtomicLong threadCpuTime = + threadLocalTimer + .get() + .computeIfAbsent( + context, (ctx) -> new AtomicLong(THREAD_MX_BEAN.getCurrentThreadCpuTime())); + long currentThreadCpuTime = THREAD_MX_BEAN.getCurrentThreadCpuTime(); + long result = currentThreadCpuTime - threadCpuTime.get(); + threadCpuTime.set(currentThreadCpuTime); + return Optional.of(result); + } } /** - * Get the cpu time for the current thread since {@link Thread#start()} without throwing an - * exception. + * Discard any accumulated time for a given context since the last invocation. * - * @see ThreadMXBean#getCurrentThreadCpuTime() for important details - * @return the number of nanoseconds of cpu consumed by this thread since {@code Thread.start()}. + * @param context the context to reset */ - private long getThreadTotalCpuNs() { + public static void reset(String context) { if (THREAD_MX_BEAN != null) { - return THREAD_MX_BEAN.getCurrentThreadCpuTime(); - } else { - return UNSUPPORTED; + threadLocalTimer + .get() + .computeIfAbsent( + context, (ctx) -> new AtomicLong(THREAD_MX_BEAN.getCurrentThreadCpuTime())) + .set(THREAD_MX_BEAN.getCurrentThreadCpuTime()); } } + public static Optional readMSandReset(String context) { + return readNSAndReset(context) + .map((cpuTimeNs) -> TimeUnit.MILLISECONDS.convert(cpuTimeNs, TimeUnit.NANOSECONDS)); + } + /** - * Get the CPU usage information for the current thread since it created this {@link - * ThreadCpuTimer}. The result is undefined if called by any other thread. - * - * @return the thread's cpu since the creation of this {@link ThreadCpuTimer} instance. If the - * VM's cpu tracking is disabled, returned value will be {@link #UNSUPPORTED}. + * Cleanup method. This should be called at the very end of a request thread when it's absolutely + * sure no code will attempt a new reading. */ - public Optional getElapsedCpuMs() { - long cpuTimeNs = getElapsedCpuNs(); - return cpuTimeNs != UNSUPPORTED - ? Optional.of(TimeUnit.MILLISECONDS.convert(cpuTimeNs, TimeUnit.NANOSECONDS)) - : Optional.empty(); + public static void reset() { + threadLocalTimer.get().clear(); } @Override public String toString() { - return getElapsedCpuMs().map(String::valueOf).orElse("UNSUPPORTED"); + return THREAD_MX_BEAN == null + ? "UNSUPPORTED" + : "Timing contexts:" + threadLocalTimer.get().toString(); } } diff --git a/solr/core/src/test-files/prometheus/solr-prometheus-output.txt b/solr/core/src/test-files/prometheus/solr-prometheus-output.txt deleted file mode 100644 index 825381fe4e8..00000000000 --- a/solr/core/src/test-files/prometheus/solr-prometheus-output.txt +++ /dev/null @@ -1,535 +0,0 @@ -# TYPE solr_metrics_jvm_buffers gauge -solr_metrics_jvm_buffers{item="Count",pool="direct"} -solr_metrics_jvm_buffers{item="Count",pool="mapped"} -solr_metrics_jvm_buffers{item="Count",pool="mapped - 'non-volatile memory'"} -# TYPE solr_metrics_jvm_buffers_bytes gauge -solr_metrics_jvm_buffers_bytes{item="MemoryUsed",pool="direct"} -solr_metrics_jvm_buffers_bytes{item="MemoryUsed",pool="mapped"} -solr_metrics_jvm_buffers_bytes{item="MemoryUsed",pool="mapped - 'non-volatile memory'"} -solr_metrics_jvm_buffers_bytes{item="TotalCapacity",pool="direct"} -solr_metrics_jvm_buffers_bytes{item="TotalCapacity",pool="mapped"} -solr_metrics_jvm_buffers_bytes{item="TotalCapacity",pool="mapped - 'non-volatile memory'"} -# TYPE solr_metrics_jvm_gc gauge -solr_metrics_jvm_gc{} -solr_metrics_jvm_gc{} -# TYPE solr_metrics_jvm_gc_seconds gauge -solr_metrics_jvm_gc_seconds{} -solr_metrics_jvm_gc_seconds{} -# TYPE solr_metrics_jvm_heap gauge -solr_metrics_jvm_heap{item="committed",memory="heap"} -solr_metrics_jvm_heap{item="committed",memory="non-heap"} -solr_metrics_jvm_heap{item="committed",memory="total"} -solr_metrics_jvm_heap{item="init",memory="heap"} -solr_metrics_jvm_heap{item="init",memory="non-heap"} -solr_metrics_jvm_heap{item="init",memory="total"} -solr_metrics_jvm_heap{item="max",memory="heap"} -solr_metrics_jvm_heap{item="max",memory="non-heap"} -solr_metrics_jvm_heap{item="max",memory="total"} -solr_metrics_jvm_heap{item="usage",memory="heap"} -solr_metrics_jvm_heap{item="usage",memory="non-heap"} -solr_metrics_jvm_heap{item="used",memory="heap"} -solr_metrics_jvm_heap{item="used",memory="non-heap"} -solr_metrics_jvm_heap{item="used",memory="total"} -# TYPE solr_metrics_jvm_memory_pools_bytes gauge -solr_metrics_jvm_memory_pools_bytes{item="committed",space="CodeCache"} -solr_metrics_jvm_memory_pools_bytes{item="committed",space="Compressed-Class-Space"} -solr_metrics_jvm_memory_pools_bytes{item="committed",space="Metaspace"} -solr_metrics_jvm_memory_pools_bytes{item="committed",space="PS-Eden-Space"} -solr_metrics_jvm_memory_pools_bytes{item="committed",space="PS-Old-Gen"} -solr_metrics_jvm_memory_pools_bytes{item="committed",space="PS-Survivor-Space"} -solr_metrics_jvm_memory_pools_bytes{item="init",space="CodeCache"} -solr_metrics_jvm_memory_pools_bytes{item="init",space="Compressed-Class-Space"} -solr_metrics_jvm_memory_pools_bytes{item="init",space="Metaspace"} -solr_metrics_jvm_memory_pools_bytes{item="init",space="PS-Eden-Space"} -solr_metrics_jvm_memory_pools_bytes{item="init",space="PS-Old-Gen"} -solr_metrics_jvm_memory_pools_bytes{item="init",space="PS-Survivor-Space"} -solr_metrics_jvm_memory_pools_bytes{item="max",space="CodeCache"} -solr_metrics_jvm_memory_pools_bytes{item="max",space="Compressed-Class-Space"} -solr_metrics_jvm_memory_pools_bytes{item="max",space="Metaspace"} -solr_metrics_jvm_memory_pools_bytes{item="max",space="PS-Eden-Space"} -solr_metrics_jvm_memory_pools_bytes{item="max",space="PS-Old-Gen"} -solr_metrics_jvm_memory_pools_bytes{item="max",space="PS-Survivor-Space"} -solr_metrics_jvm_memory_pools_bytes{item="usage",space="CodeCache"} -solr_metrics_jvm_memory_pools_bytes{item="usage",space="Compressed-Class-Space"} -solr_metrics_jvm_memory_pools_bytes{item="usage",space="Metaspace"} -solr_metrics_jvm_memory_pools_bytes{item="usage",space="PS-Eden-Space"} -solr_metrics_jvm_memory_pools_bytes{item="usage",space="PS-Old-Gen"} -solr_metrics_jvm_memory_pools_bytes{item="usage",space="PS-Survivor-Space"} -solr_metrics_jvm_memory_pools_bytes{item="used",space="CodeCache"} -solr_metrics_jvm_memory_pools_bytes{item="used",space="Compressed-Class-Space"} -solr_metrics_jvm_memory_pools_bytes{item="used",space="Metaspace"} -solr_metrics_jvm_memory_pools_bytes{item="used",space="PS-Eden-Space"} -solr_metrics_jvm_memory_pools_bytes{item="used",space="PS-Old-Gen"} -solr_metrics_jvm_memory_pools_bytes{item="used",space="PS-Survivor-Space"} -solr_metrics_jvm_memory_pools_bytes{item="used-after-gc",space="PS-Eden-Space"} -solr_metrics_jvm_memory_pools_bytes{item="used-after-gc",space="PS-Old-Gen"} -solr_metrics_jvm_memory_pools_bytes{item="used-after-gc",space="PS-Survivor-Space"} -# TYPE solr_metrics_jvm_threads gauge -solr_metrics_jvm_threads{item="blocked"} -solr_metrics_jvm_threads{item="count"} -solr_metrics_jvm_threads{item="daemon"} -solr_metrics_jvm_threads{item="deadlock"} -solr_metrics_jvm_threads{item="new"} -solr_metrics_jvm_threads{item="peak"} -solr_metrics_jvm_threads{item="runnable"} -solr_metrics_jvm_threads{item="terminated"} -solr_metrics_jvm_threads{item="timed_waiting"} -solr_metrics_jvm_threads{item="total_started"} -solr_metrics_jvm_threads{item="waiting"} -# TYPE solr_metrics_os gauge -solr_metrics_os{item="availableProcessors"} -solr_metrics_os{item="committedVirtualMemorySize"} -solr_metrics_os{item="cpuLoad"} -solr_metrics_os{item="freeMemorySize"} -solr_metrics_os{item="freePhysicalMemorySize"} -solr_metrics_os{item="freeSwapSpaceSize"} -solr_metrics_os{item="maxFileDescriptorCount"} -solr_metrics_os{item="openFileDescriptorCount"} -solr_metrics_os{item="processCpuLoad"} -solr_metrics_os{item="processCpuTime"} -solr_metrics_os{item="systemCpuLoad"} -solr_metrics_os{item="systemLoadAverage"} -solr_metrics_os{item="totalMemorySize"} -solr_metrics_os{item="totalPhysicalMemorySize"} -solr_metrics_os{item="totalSwapSpaceSize"} -# TYPE solr_metrics_node_connections gauge -solr_metrics_node_connections{category="UPDATE",handler="updateShardHandler",type="availableConnections"} -solr_metrics_node_connections{category="UPDATE",handler="updateShardHandler",type="leasedConnections"} -solr_metrics_node_connections{category="UPDATE",handler="updateShardHandler",type="maxConnections"} -solr_metrics_node_connections{category="UPDATE",handler="updateShardHandler",type="pendingConnections"} -# TYPE solr_metrics_node_core_root_fs_bytes gauge -solr_metrics_node_core_root_fs_bytes{category="CONTAINER",item="totalSpace"} -solr_metrics_node_core_root_fs_bytes{category="CONTAINER",item="usableSpace"} -# TYPE solr_metrics_node_cores gauge -solr_metrics_node_cores{category="CONTAINER",item="lazy"} -solr_metrics_node_cores{category="CONTAINER",item="loaded"} -solr_metrics_node_cores{category="CONTAINER",item="unloaded"} -# TYPE solr_metrics_node_requests_total counter -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/authorization",type="clientErrors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/authorization",type="errors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/authorization",type="requests"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/authorization",type="serverErrors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/authorization",type="timeouts"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/collections",type="clientErrors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/collections",type="errors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/collections",type="requests"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/collections",type="serverErrors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/collections",type="timeouts"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/configs",type="clientErrors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/configs",type="errors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/configs",type="requests"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/configs",type="serverErrors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/configs",type="timeouts"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/cores",type="clientErrors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/cores",type="errors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/cores",type="requests"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/cores",type="serverErrors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/cores",type="timeouts"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/info",type="clientErrors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/info",type="errors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/info",type="requests"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/info",type="serverErrors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/info",type="timeouts"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/metrics",type="clientErrors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/metrics",type="errors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/metrics",type="requests"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/metrics",type="serverErrors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/metrics",type="timeouts"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/zookeeper",type="clientErrors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/zookeeper",type="errors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/zookeeper",type="requests"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/zookeeper",type="serverErrors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/zookeeper",type="timeouts"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/zookeeper/status",type="clientErrors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/zookeeper/status",type="errors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/zookeeper/status",type="requests"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/zookeeper/status",type="serverErrors"} -solr_metrics_node_requests_total{category="ADMIN",handler="/admin/zookeeper/status",type="timeouts"} -# TYPE solr_metrics_node_requests_time_total counter -solr_metrics_node_requests_time_total{category="ADMIN",handler="/admin/authorization"} -solr_metrics_node_requests_time_total{category="ADMIN",handler="/admin/collections"} -solr_metrics_node_requests_time_total{category="ADMIN",handler="/admin/configs"} -solr_metrics_node_requests_time_total{category="ADMIN",handler="/admin/cores"} -solr_metrics_node_requests_time_total{category="ADMIN",handler="/admin/info"} -solr_metrics_node_requests_time_total{category="ADMIN",handler="/admin/metrics"} -solr_metrics_node_requests_time_total{category="ADMIN",handler="/admin/zookeeper"} -solr_metrics_node_requests_time_total{category="ADMIN",handler="/admin/zookeeper/status"} -# TYPE solr_metrics_node_thread_pool_total counter -solr_metrics_node_thread_pool_total{category="ADMIN",executer="parallelCoreAdminExecutor",handler="/admin/cores",task="completed"} -solr_metrics_node_thread_pool_total{category="ADMIN",executer="parallelCoreAdminExecutor",handler="/admin/cores",task="running"} -solr_metrics_node_thread_pool_total{category="ADMIN",executer="parallelCoreAdminExecutor",handler="/admin/cores",task="submitted"} -solr_metrics_node_thread_pool_total{category="ADMIN",executer="parallelCoreExpensiveAdminExecutor",handler="/admin/cores",task="completed"} -solr_metrics_node_thread_pool_total{category="ADMIN",executer="parallelCoreExpensiveAdminExecutor",handler="/admin/cores",task="running"} -solr_metrics_node_thread_pool_total{category="ADMIN",executer="parallelCoreExpensiveAdminExecutor",handler="/admin/cores",task="submitted"} -solr_metrics_node_thread_pool_total{category="QUERY",executer="httpShardExecutor",handler="httpShardHandler",task="completed"} -solr_metrics_node_thread_pool_total{category="QUERY",executer="httpShardExecutor",handler="httpShardHandler",task="running"} -solr_metrics_node_thread_pool_total{category="QUERY",executer="httpShardExecutor",handler="httpShardHandler",task="submitted"} -solr_metrics_node_thread_pool_total{category="UPDATE",executer="recoveryExecutor",handler="updateShardHandler",task="completed"} -solr_metrics_node_thread_pool_total{category="UPDATE",executer="recoveryExecutor",handler="updateShardHandler",task="running"} -solr_metrics_node_thread_pool_total{category="UPDATE",executer="recoveryExecutor",handler="updateShardHandler",task="submitted"} -solr_metrics_node_thread_pool_total{category="UPDATE",executer="updateOnlyExecutor",handler="updateShardHandler",task="completed"} -solr_metrics_node_thread_pool_total{category="UPDATE",executer="updateOnlyExecutor",handler="updateShardHandler",task="running"} -solr_metrics_node_thread_pool_total{category="UPDATE",executer="updateOnlyExecutor",handler="updateShardHandler",task="submitted"} -solr_metrics_node_thread_pool_total{category="CONTAINER",executer="coreContainerWorkExecutor",task="completed"} -solr_metrics_node_thread_pool_total{category="CONTAINER",executer="coreContainerWorkExecutor",task="running"} -solr_metrics_node_thread_pool_total{category="CONTAINER",executer="coreContainerWorkExecutor",task="submitted"} -solr_metrics_node_thread_pool_total{category="CONTAINER",executer="coreLoadExecutor",task="completed"} -solr_metrics_node_thread_pool_total{category="CONTAINER",executer="coreLoadExecutor",task="running"} -solr_metrics_node_thread_pool_total{category="CONTAINER",executer="coreLoadExecutor",task="submitted"} -# TYPE solr_metrics_core_average_request_time gauge -solr_metrics_core_average_request_time{category="ADMIN",core="collection1",handler="/admin/file"} -solr_metrics_core_average_request_time{category="ADMIN",core="collection1",handler="/admin/luke"} -solr_metrics_core_average_request_time{category="ADMIN",core="collection1",handler="/admin/mbeans"} -solr_metrics_core_average_request_time{category="ADMIN",core="collection1",handler="/admin/ping"} -solr_metrics_core_average_request_time{category="ADMIN",core="collection1",handler="/admin/plugins"} -solr_metrics_core_average_request_time{category="ADMIN",core="collection1",handler="/admin/segments"} -solr_metrics_core_average_request_time{category="ADMIN",core="collection1",handler="/admin/system"} -solr_metrics_core_average_request_time{category="ADMIN",core="collection1",handler="/config"} -solr_metrics_core_average_request_time{category="ADMIN",core="collection1",handler="/schema"} -solr_metrics_core_average_request_time{category="ADMIN",core="collection1",handler="/tasks/cancel"} -solr_metrics_core_average_request_time{category="ADMIN",core="collection1",handler="/tasks/list"} -solr_metrics_core_average_request_time{category="QUERY",core="collection1",handler="/debug/dump"} -solr_metrics_core_average_request_time{category="QUERY",core="collection1",handler="/export"} -solr_metrics_core_average_request_time{category="QUERY",core="collection1",handler="/export[shard]"} -solr_metrics_core_average_request_time{category="QUERY",core="collection1",handler="/get"} -solr_metrics_core_average_request_time{category="QUERY",core="collection1",handler="/get[shard]"} -solr_metrics_core_average_request_time{category="QUERY",core="collection1",handler="/graph"} -solr_metrics_core_average_request_time{category="QUERY",core="collection1",handler="/query"} -solr_metrics_core_average_request_time{category="QUERY",core="collection1",handler="/query[shard]"} -solr_metrics_core_average_request_time{category="QUERY",core="collection1",handler="/select"} -solr_metrics_core_average_request_time{category="QUERY",core="collection1",handler="/select[shard]"} -solr_metrics_core_average_request_time{category="QUERY",core="collection1",handler="/stream"} -solr_metrics_core_average_request_time{category="QUERY",core="collection1",handler="/terms"} -solr_metrics_core_average_request_time{category="QUERY",core="collection1",handler="/terms[shard]"} -solr_metrics_core_average_request_time{category="REPLICATION",core="collection1",handler="/replication"} -solr_metrics_core_average_request_time{category="UPDATE",core="collection1",handler="/update"} -solr_metrics_core_average_request_time{category="UPDATE",core="collection1",handler="/update/cbor"} -solr_metrics_core_average_request_time{category="UPDATE",core="collection1",handler="/update/csv"} -solr_metrics_core_average_request_time{category="UPDATE",core="collection1",handler="/update/json"} -solr_metrics_core_average_request_time{category="UPDATE",core="collection1",handler="/update/json/docs"} -solr_metrics_core_average_request_time{category="UPDATE",core="collection1",handler="update"} -# TYPE solr_metrics_core_average_searcher_warmup_time gauge -solr_metrics_core_average_searcher_warmup_time{core="collection1",type="time"} -solr_metrics_core_average_searcher_warmup_time{core="collection1",type="warmup"} -# TYPE solr_metrics_core_cache gauge -solr_metrics_core_cache{cacheType="documentCache",core="collection1",item="cumulative_evictions"} -solr_metrics_core_cache{cacheType="documentCache",core="collection1",item="cumulative_hitratio"} -solr_metrics_core_cache{cacheType="documentCache",core="collection1",item="cumulative_hits"} -solr_metrics_core_cache{cacheType="documentCache",core="collection1",item="cumulative_inserts"} -solr_metrics_core_cache{cacheType="documentCache",core="collection1",item="cumulative_lookups"} -solr_metrics_core_cache{cacheType="documentCache",core="collection1",item="evictions"} -solr_metrics_core_cache{cacheType="documentCache",core="collection1",item="hitratio"} -solr_metrics_core_cache{cacheType="documentCache",core="collection1",item="hits"} -solr_metrics_core_cache{cacheType="documentCache",core="collection1",item="inserts"} -solr_metrics_core_cache{cacheType="documentCache",core="collection1",item="lookups"} -solr_metrics_core_cache{cacheType="documentCache",core="collection1",item="maxRamMB"} -solr_metrics_core_cache{cacheType="documentCache",core="collection1",item="ramBytesUsed"} -solr_metrics_core_cache{cacheType="documentCache",core="collection1",item="size"} -solr_metrics_core_cache{cacheType="documentCache",core="collection1",item="warmupTime"} -solr_metrics_core_cache{cacheType="fieldCache",core="collection1",item="entries_count"} -solr_metrics_core_cache{cacheType="fieldValueCache",core="collection1",item="cumulative_evictions"} -solr_metrics_core_cache{cacheType="fieldValueCache",core="collection1",item="cumulative_hitratio"} -solr_metrics_core_cache{cacheType="fieldValueCache",core="collection1",item="cumulative_hits"} -solr_metrics_core_cache{cacheType="fieldValueCache",core="collection1",item="cumulative_inserts"} -solr_metrics_core_cache{cacheType="fieldValueCache",core="collection1",item="cumulative_lookups"} -solr_metrics_core_cache{cacheType="fieldValueCache",core="collection1",item="evictions"} -solr_metrics_core_cache{cacheType="fieldValueCache",core="collection1",item="hitratio"} -solr_metrics_core_cache{cacheType="fieldValueCache",core="collection1",item="hits"} -solr_metrics_core_cache{cacheType="fieldValueCache",core="collection1",item="inserts"} -solr_metrics_core_cache{cacheType="fieldValueCache",core="collection1",item="lookups"} -solr_metrics_core_cache{cacheType="fieldValueCache",core="collection1",item="maxRamMB"} -solr_metrics_core_cache{cacheType="fieldValueCache",core="collection1",item="ramBytesUsed"} -solr_metrics_core_cache{cacheType="fieldValueCache",core="collection1",item="size"} -solr_metrics_core_cache{cacheType="fieldValueCache",core="collection1",item="warmupTime"} -solr_metrics_core_cache{cacheType="filterCache",core="collection1",item="cumulative_evictions"} -solr_metrics_core_cache{cacheType="filterCache",core="collection1",item="cumulative_hitratio"} -solr_metrics_core_cache{cacheType="filterCache",core="collection1",item="cumulative_hits"} -solr_metrics_core_cache{cacheType="filterCache",core="collection1",item="cumulative_inserts"} -solr_metrics_core_cache{cacheType="filterCache",core="collection1",item="cumulative_lookups"} -solr_metrics_core_cache{cacheType="filterCache",core="collection1",item="evictions"} -solr_metrics_core_cache{cacheType="filterCache",core="collection1",item="hitratio"} -solr_metrics_core_cache{cacheType="filterCache",core="collection1",item="hits"} -solr_metrics_core_cache{cacheType="filterCache",core="collection1",item="inserts"} -solr_metrics_core_cache{cacheType="filterCache",core="collection1",item="lookups"} -solr_metrics_core_cache{cacheType="filterCache",core="collection1",item="maxRamMB"} -solr_metrics_core_cache{cacheType="filterCache",core="collection1",item="ramBytesUsed"} -solr_metrics_core_cache{cacheType="filterCache",core="collection1",item="size"} -solr_metrics_core_cache{cacheType="filterCache",core="collection1",item="warmupTime"} -solr_metrics_core_cache{cacheType="perSegFilter",core="collection1",item="cumulative_evictions"} -solr_metrics_core_cache{cacheType="perSegFilter",core="collection1",item="cumulative_hitratio"} -solr_metrics_core_cache{cacheType="perSegFilter",core="collection1",item="cumulative_hits"} -solr_metrics_core_cache{cacheType="perSegFilter",core="collection1",item="cumulative_inserts"} -solr_metrics_core_cache{cacheType="perSegFilter",core="collection1",item="cumulative_lookups"} -solr_metrics_core_cache{cacheType="perSegFilter",core="collection1",item="evictions"} -solr_metrics_core_cache{cacheType="perSegFilter",core="collection1",item="hitratio"} -solr_metrics_core_cache{cacheType="perSegFilter",core="collection1",item="hits"} -solr_metrics_core_cache{cacheType="perSegFilter",core="collection1",item="inserts"} -solr_metrics_core_cache{cacheType="perSegFilter",core="collection1",item="lookups"} -solr_metrics_core_cache{cacheType="perSegFilter",core="collection1",item="maxRamMB"} -solr_metrics_core_cache{cacheType="perSegFilter",core="collection1",item="ramBytesUsed"} -solr_metrics_core_cache{cacheType="perSegFilter",core="collection1",item="size"} -solr_metrics_core_cache{cacheType="perSegFilter",core="collection1",item="warmupTime"} -solr_metrics_core_cache{cacheType="queryResultCache",core="collection1",item="cumulative_evictions"} -solr_metrics_core_cache{cacheType="queryResultCache",core="collection1",item="cumulative_hitratio"} -solr_metrics_core_cache{cacheType="queryResultCache",core="collection1",item="cumulative_hits"} -solr_metrics_core_cache{cacheType="queryResultCache",core="collection1",item="cumulative_inserts"} -solr_metrics_core_cache{cacheType="queryResultCache",core="collection1",item="cumulative_lookups"} -solr_metrics_core_cache{cacheType="queryResultCache",core="collection1",item="evictions"} -solr_metrics_core_cache{cacheType="queryResultCache",core="collection1",item="hitratio"} -solr_metrics_core_cache{cacheType="queryResultCache",core="collection1",item="hits"} -solr_metrics_core_cache{cacheType="queryResultCache",core="collection1",item="inserts"} -solr_metrics_core_cache{cacheType="queryResultCache",core="collection1",item="lookups"} -solr_metrics_core_cache{cacheType="queryResultCache",core="collection1",item="maxRamMB"} -solr_metrics_core_cache{cacheType="queryResultCache",core="collection1",item="ramBytesUsed"} -solr_metrics_core_cache{cacheType="queryResultCache",core="collection1",item="size"} -solr_metrics_core_cache{cacheType="queryResultCache",core="collection1",item="warmupTime"} -solr_metrics_core_cache{cacheType="statsCache",core="collection1",item="lookups"} -solr_metrics_core_cache{cacheType="statsCache",core="collection1",item="mergeToGlobalStats"} -solr_metrics_core_cache{cacheType="statsCache",core="collection1",item="missingGlobalFieldStats"} -solr_metrics_core_cache{cacheType="statsCache",core="collection1",item="missingGlobalTermStats"} -solr_metrics_core_cache{cacheType="statsCache",core="collection1",item="receiveGlobalStats"} -solr_metrics_core_cache{cacheType="statsCache",core="collection1",item="retrieveStats"} -solr_metrics_core_cache{cacheType="statsCache",core="collection1",item="returnLocalStats"} -solr_metrics_core_cache{cacheType="statsCache",core="collection1",item="sendGlobalStats"} -solr_metrics_core_cache{cacheType="statsCache",core="collection1",item="useCachedGlobalStats"} -solr_metrics_core_cache{core="collection1",item="hits",type="liveDocsCache"} -solr_metrics_core_cache{core="collection1",item="inserts",type="liveDocsCache"} -solr_metrics_core_cache{core="collection1",item="naiveHits",type="liveDocsCache"} -# TYPE solr_metrics_core_highlighter_requests_total counter -solr_metrics_core_highlighter_requests_total{core="collection1",item="breakIterator",type="SolrBoundaryScanner"} -solr_metrics_core_highlighter_requests_total{core="collection1",item="colored",type="SolrFragmentsBuilder"} -solr_metrics_core_highlighter_requests_total{core="collection1",item="default",type="SolrBoundaryScanner"} -solr_metrics_core_highlighter_requests_total{core="collection1",item="default",type="SolrEncoder"} -solr_metrics_core_highlighter_requests_total{core="collection1",item="default",type="SolrFormatter"} -solr_metrics_core_highlighter_requests_total{core="collection1",item="default",type="SolrFragListBuilder"} -solr_metrics_core_highlighter_requests_total{core="collection1",item="default",type="SolrFragmenter"} -solr_metrics_core_highlighter_requests_total{core="collection1",item="default",type="SolrFragmentsBuilder"} -solr_metrics_core_highlighter_requests_total{core="collection1",item="gap",type="SolrFragmenter"} -solr_metrics_core_highlighter_requests_total{core="collection1",item="html",type="SolrEncoder"} -solr_metrics_core_highlighter_requests_total{core="collection1",item="html",type="SolrFormatter"} -solr_metrics_core_highlighter_requests_total{core="collection1",item="regex",type="SolrFragmenter"} -solr_metrics_core_highlighter_requests_total{core="collection1",item="simple",type="SolrFragListBuilder"} -solr_metrics_core_highlighter_requests_total{core="collection1",item="single",type="SolrFragListBuilder"} -solr_metrics_core_highlighter_requests_total{core="collection1",item="weighted",type="SolrFragListBuilder"} -# TYPE solr_metrics_core_index_size_bytes gauge -solr_metrics_core_index_size_bytes{core="collection1"} -# TYPE solr_metrics_core_requests_total counter -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/file",type="clientErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/file",type="errors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/file",type="requests"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/file",type="serverErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/file",type="timeouts"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/luke",type="clientErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/luke",type="errors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/luke",type="requests"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/luke",type="serverErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/luke",type="timeouts"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/mbeans",type="clientErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/mbeans",type="errors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/mbeans",type="requests"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/mbeans",type="serverErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/mbeans",type="timeouts"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/ping",type="clientErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/ping",type="errors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/ping",type="requests"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/ping",type="serverErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/ping",type="timeouts"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/plugins",type="clientErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/plugins",type="errors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/plugins",type="requests"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/plugins",type="serverErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/plugins",type="timeouts"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/segments",type="clientErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/segments",type="errors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/segments",type="requests"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/segments",type="serverErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/segments",type="timeouts"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/system",type="clientErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/system",type="errors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/system",type="requests"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/system",type="serverErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/admin/system",type="timeouts"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/config",type="clientErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/config",type="errors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/config",type="requests"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/config",type="serverErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/config",type="timeouts"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/schema",type="clientErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/schema",type="errors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/schema",type="requests"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/schema",type="serverErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/schema",type="timeouts"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/tasks/cancel",type="clientErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/tasks/cancel",type="errors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/tasks/cancel",type="requests"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/tasks/cancel",type="serverErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/tasks/cancel",type="timeouts"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/tasks/list",type="clientErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/tasks/list",type="errors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/tasks/list",type="requests"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/tasks/list",type="serverErrors"} -solr_metrics_core_requests_total{category="ADMIN",core="collection1",handler="/tasks/list",type="timeouts"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/debug/dump",type="clientErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/debug/dump",type="errors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/debug/dump",type="requests"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/debug/dump",type="serverErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/debug/dump",type="timeouts"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/export",type="clientErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/export",type="errors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/export",type="requests"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/export",type="serverErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/export",type="timeouts"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/export[shard]",type="clientErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/export[shard]",type="errors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/export[shard]",type="requests"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/export[shard]",type="serverErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/export[shard]",type="timeouts"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/get",type="clientErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/get",type="errors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/get",type="requests"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/get",type="serverErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/get",type="timeouts"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/get[shard]",type="clientErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/get[shard]",type="errors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/get[shard]",type="requests"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/get[shard]",type="serverErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/get[shard]",type="timeouts"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/graph",type="clientErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/graph",type="errors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/graph",type="requests"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/graph",type="serverErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/graph",type="timeouts"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/query",type="clientErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/query",type="errors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/query",type="requests"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/query",type="serverErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/query",type="timeouts"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/query[shard]",type="clientErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/query[shard]",type="errors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/query[shard]",type="requests"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/query[shard]",type="serverErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/query[shard]",type="timeouts"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/select",type="clientErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/select",type="errors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/select",type="requests"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/select",type="serverErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/select",type="timeouts"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/select[shard]",type="clientErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/select[shard]",type="errors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/select[shard]",type="requests"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/select[shard]",type="serverErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/select[shard]",type="timeouts"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/stream",type="clientErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/stream",type="errors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/stream",type="requests"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/stream",type="serverErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/stream",type="timeouts"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/terms",type="clientErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/terms",type="errors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/terms",type="requests"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/terms",type="serverErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/terms",type="timeouts"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/terms[shard]",type="clientErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/terms[shard]",type="errors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/terms[shard]",type="requests"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/terms[shard]",type="serverErrors"} -solr_metrics_core_requests_total{category="QUERY",core="collection1",handler="/terms[shard]",type="timeouts"} -solr_metrics_core_requests_total{category="REPLICATION",core="collection1",handler="/replication",type="clientErrors"} -solr_metrics_core_requests_total{category="REPLICATION",core="collection1",handler="/replication",type="errors"} -solr_metrics_core_requests_total{category="REPLICATION",core="collection1",handler="/replication",type="requests"} -solr_metrics_core_requests_total{category="REPLICATION",core="collection1",handler="/replication",type="serverErrors"} -solr_metrics_core_requests_total{category="REPLICATION",core="collection1",handler="/replication",type="timeouts"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update",type="clientErrors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update",type="errors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update",type="requests"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update",type="serverErrors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update",type="timeouts"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/cbor",type="clientErrors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/cbor",type="errors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/cbor",type="requests"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/cbor",type="serverErrors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/cbor",type="timeouts"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/csv",type="clientErrors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/csv",type="errors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/csv",type="requests"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/csv",type="serverErrors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/csv",type="timeouts"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/json",type="clientErrors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/json",type="errors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/json",type="requests"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/json",type="serverErrors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/json",type="timeouts"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/json/docs",type="clientErrors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/json/docs",type="errors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/json/docs",type="requests"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/json/docs",type="serverErrors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="/update/json/docs",type="timeouts"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="update",type="clientErrors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="update",type="errors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="update",type="requests"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="update",type="serverErrors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="update",type="timeouts"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="updateHandler",type="commits"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="updateHandler",type="cumulativeAdds"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="updateHandler",type="cumulativeDeletesById"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="updateHandler",type="cumulativeDeletesByQuery"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="updateHandler",type="cumulativeErrors"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="updateHandler",type="expungeDeletes"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="updateHandler",type="merges"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="updateHandler",type="optimizes"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="updateHandler",type="rollbacks"} -solr_metrics_core_requests_total{category="UPDATE",core="collection1",handler="updateHandler",type="splits"} -# TYPE solr_metrics_core_requests_time_total counter -solr_metrics_core_requests_time_total{category="ADMIN",core="collection1",handler="/admin/file"} -solr_metrics_core_requests_time_total{category="ADMIN",core="collection1",handler="/admin/luke"} -solr_metrics_core_requests_time_total{category="ADMIN",core="collection1",handler="/admin/mbeans"} -solr_metrics_core_requests_time_total{category="ADMIN",core="collection1",handler="/admin/ping"} -solr_metrics_core_requests_time_total{category="ADMIN",core="collection1",handler="/admin/plugins"} -solr_metrics_core_requests_time_total{category="ADMIN",core="collection1",handler="/admin/segments"} -solr_metrics_core_requests_time_total{category="ADMIN",core="collection1",handler="/admin/system"} -solr_metrics_core_requests_time_total{category="ADMIN",core="collection1",handler="/config"} -solr_metrics_core_requests_time_total{category="ADMIN",core="collection1",handler="/schema"} -solr_metrics_core_requests_time_total{category="ADMIN",core="collection1",handler="/tasks/cancel"} -solr_metrics_core_requests_time_total{category="ADMIN",core="collection1",handler="/tasks/list"} -solr_metrics_core_requests_time_total{category="QUERY",core="collection1",handler="/debug/dump"} -solr_metrics_core_requests_time_total{category="QUERY",core="collection1",handler="/export"} -solr_metrics_core_requests_time_total{category="QUERY",core="collection1",handler="/export[shard]"} -solr_metrics_core_requests_time_total{category="QUERY",core="collection1",handler="/get"} -solr_metrics_core_requests_time_total{category="QUERY",core="collection1",handler="/get[shard]"} -solr_metrics_core_requests_time_total{category="QUERY",core="collection1",handler="/graph"} -solr_metrics_core_requests_time_total{category="QUERY",core="collection1",handler="/query"} -solr_metrics_core_requests_time_total{category="QUERY",core="collection1",handler="/query[shard]"} -solr_metrics_core_requests_time_total{category="QUERY",core="collection1",handler="/select"} -solr_metrics_core_requests_time_total{category="QUERY",core="collection1",handler="/select[shard]"} -solr_metrics_core_requests_time_total{category="QUERY",core="collection1",handler="/stream"} -solr_metrics_core_requests_time_total{category="QUERY",core="collection1",handler="/terms"} -solr_metrics_core_requests_time_total{category="QUERY",core="collection1",handler="/terms[shard]"} -solr_metrics_core_requests_time_total{category="REPLICATION",core="collection1",handler="/replication"} -solr_metrics_core_requests_time_total{category="UPDATE",core="collection1",handler="/update"} -solr_metrics_core_requests_time_total{category="UPDATE",core="collection1",handler="/update/cbor"} -solr_metrics_core_requests_time_total{category="UPDATE",core="collection1",handler="/update/csv"} -solr_metrics_core_requests_time_total{category="UPDATE",core="collection1",handler="/update/json"} -solr_metrics_core_requests_time_total{category="UPDATE",core="collection1",handler="/update/json/docs"} -solr_metrics_core_requests_time_total{category="UPDATE",core="collection1",handler="update"} -# TYPE solr_metrics_core_searcher_documents gauge -solr_metrics_core_searcher_documents{core="collection1",type="deletedDocs"} -solr_metrics_core_searcher_documents{core="collection1",type="fullSortCount"} -solr_metrics_core_searcher_documents{core="collection1",type="indexCommitSize"} -solr_metrics_core_searcher_documents{core="collection1",type="indexVersion"} -solr_metrics_core_searcher_documents{core="collection1",type="maxDoc"} -solr_metrics_core_searcher_documents{core="collection1",type="numDocs"} -solr_metrics_core_searcher_documents{core="collection1",type="skipSortCount"} -solr_metrics_core_searcher_documents{core="collection1",type="warmupTime"} -# TYPE solr_metrics_core_tlog_total counter -solr_metrics_core_tlog_total{core="collection1",item="applyingBuffered"} -solr_metrics_core_tlog_total{core="collection1",item="copyOverOldUpdates"} -solr_metrics_core_tlog_total{core="collection1",item="replay"} -# TYPE solr_metrics_core_update_handler gauge -solr_metrics_core_update_handler{category="REPLICATION",core="collection1",handler="/replication",type="generation"} -solr_metrics_core_update_handler{category="UPDATE",core="collection1",handler="updateHandler",type="adds"} -solr_metrics_core_update_handler{category="UPDATE",core="collection1",handler="updateHandler",type="autoCommits"} -solr_metrics_core_update_handler{category="UPDATE",core="collection1",handler="updateHandler",type="deletesById"} -solr_metrics_core_update_handler{category="UPDATE",core="collection1",handler="updateHandler",type="deletesByQuery"} -solr_metrics_core_update_handler{category="UPDATE",core="collection1",handler="updateHandler",type="docsPending"} -solr_metrics_core_update_handler{category="UPDATE",core="collection1",handler="updateHandler",type="errors"} -solr_metrics_core_update_handler{category="UPDATE",core="collection1",handler="updateHandler",type="softAutoCommits"} diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-error-solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/bad-error-solrconfig.xml index c8bb8cf66e3..af4edbddcaf 100644 --- a/solr/core/src/test-files/solr/collection1/conf/bad-error-solrconfig.xml +++ b/solr/core/src/test-files/solr/collection1/conf/bad-error-solrconfig.xml @@ -19,7 +19,7 @@ - + ${tests.luceneMatchVersion:LATEST} diff --git a/solr/core/src/test-files/solr/collection1/conf/bad-mpf-solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/bad-mpf-solrconfig.xml index 19d786056ad..bfaf24a0cfa 100644 --- a/solr/core/src/test-files/solr/collection1/conf/bad-mpf-solrconfig.xml +++ b/solr/core/src/test-files/solr/collection1/conf/bad-mpf-solrconfig.xml @@ -19,7 +19,7 @@ - + ${tests.luceneMatchVersion:LATEST} diff --git a/solr/core/src/test-files/solr/collection1/conf/bad_solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/bad_solrconfig.xml index e24df5846fd..70edc2d4723 100644 --- a/solr/core/src/test-files/solr/collection1/conf/bad_solrconfig.xml +++ b/solr/core/src/test-files/solr/collection1/conf/bad_solrconfig.xml @@ -20,7 +20,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${unset.sys.property} diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml b/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml index 7d5b3f09051..51c2b26a832 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml @@ -33,7 +33,7 @@ - + id diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-collate.xml b/solr/core/src/test-files/solr/collection1/conf/schema-collate.xml index c9587d75f7d..4a4dbf42a01 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-collate.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-collate.xml @@ -39,12 +39,12 @@ - - - - - - + + + + + + id diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml b/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml index db038b62787..1b36f23d497 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-custom-field.xml @@ -17,8 +17,8 @@ --> - - + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml b/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml index 84d628f7da5..90caa68a3c8 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-distributed-missing-sort.xml @@ -17,30 +17,30 @@ --> - - + + - - + + - - + + - - + + - - + + - - + + + sortMissingLast="true" uninvertible="true"/> + sortMissingFirst="true" uninvertible="true"/> @@ -70,12 +70,12 @@ - + - - + + - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml b/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml index ddb42f36e8f..74f377c6640 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-docValues.xml @@ -38,14 +38,14 @@ These are provided more for backward compatability, allowing one to create a schema that matches an existing lucene index. --> - - - - + + + + - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-docValuesFaceting.xml b/solr/core/src/test-files/solr/collection1/conf/schema-docValuesFaceting.xml index 8db18dc2996..67e6789f610 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-docValuesFaceting.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-docValuesFaceting.xml @@ -17,11 +17,11 @@ --> - - - - - + + + + + @@ -30,41 +30,41 @@ - + - + - + - + - + - + - + - + - + - + - + - + - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-docValuesJoin.xml b/solr/core/src/test-files/solr/collection1/conf/schema-docValuesJoin.xml index 646956e43a6..0fccb86224f 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-docValuesJoin.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-docValuesJoin.xml @@ -28,7 +28,7 @@ 1.4: default auto-phrase (QueryParser feature) to off 1.5: omitNorms defaults to true for primitive field types (int, float, boolean, string...) 1.6: useDocValuesAsStored defaults to true. - 1.7: docValues defaults to true. + 1.7: docValues defaults to true, uninvertible defaults to false. --> @@ -65,33 +65,33 @@ - + - + - + - + - + - + - - + + - - + + - + - + id diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-enums.xml b/solr/core/src/test-files/solr/collection1/conf/schema-enums.xml index 40a6b4441ae..c9067dff946 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-enums.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-enums.xml @@ -20,13 +20,13 @@ - + - - + + - + id @@ -44,6 +44,6 @@ but you can always add new values to the end. --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-id-and-version-fields-only.xml b/solr/core/src/test-files/solr/collection1/conf/schema-id-and-version-fields-only.xml index a98b16a6b84..a88da298762 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-id-and-version-fields-only.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-id-and-version-fields-only.xml @@ -17,7 +17,7 @@ --> - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-multiword-synonyms.xml b/solr/core/src/test-files/solr/collection1/conf/schema-multiword-synonyms.xml index 92989ce2b64..f6cfe7f8b94 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-multiword-synonyms.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-multiword-synonyms.xml @@ -19,8 +19,8 @@ - - + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-numeric.xml b/solr/core/src/test-files/solr/collection1/conf/schema-numeric.xml index 78b13270b80..3a487dfea02 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-numeric.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-numeric.xml @@ -29,37 +29,37 @@ + positionIncrementGap="0" uninvertible="true"/> + positionIncrementGap="0" uninvertible="true"/> + positionIncrementGap="0" uninvertible="true"/> + positionIncrementGap="0" uninvertible="true"/> + positionIncrementGap="0" uninvertible="true"/> + positionIncrementGap="0" uninvertible="true"/> + positionIncrementGap="0" uninvertible="true"/> + positionIncrementGap="0" uninvertible="true"/> + positionIncrementGap="0" uninvertible="true"/> + positionIncrementGap="0" uninvertible="true"/> + positionIncrementGap="0" uninvertible="true"/> + positionIncrementGap="0" uninvertible="true"/> + positionIncrementGap="0" uninvertible="true"/> + positionIncrementGap="0" uninvertible="true"/> + positionIncrementGap="0" uninvertible="true"/> diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-pseudo-fields.xml b/solr/core/src/test-files/solr/collection1/conf/schema-pseudo-fields.xml index 19bedf66555..a5d87f03c86 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-pseudo-fields.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-pseudo-fields.xml @@ -37,8 +37,8 @@ id - - + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml b/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml index 2f5e04790fd..517182771bd 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-rest.xml @@ -30,17 +30,17 @@ 1.4: default auto-phrase (QueryParser feature) to off 1.5: omitNorms defaults to true for primitive field types (int, float, boolean, string...) 1.6: useDocValuesAsStored defaults to true. - 1.7: docValues defaults to true. + 1.7: docValues defaults to true, uninvertible defaults to false. --> - - - - + + + + - - - - + + + + @@ -97,8 +97,8 @@ - - + + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-sorting-text.xml b/solr/core/src/test-files/solr/collection1/conf/schema-sorting-text.xml index cc814ab58ca..fdc675e2d07 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-sorting-text.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-sorting-text.xml @@ -46,7 +46,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml b/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml index bfee166fb0d..7ec98bde0f7 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema-spatial.xml @@ -18,12 +18,12 @@ - - - + + + - + diff --git a/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml b/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml index 100abb67981..7d1e09faf28 100644 --- a/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml +++ b/solr/core/src/test-files/solr/collection1/conf/schema_latest.xml @@ -33,7 +33,7 @@ 1.5: omitNorms defaults to true for primitive field types (int, float, boolean, string...) 1.6: useDocValuesAsStored defaults to true. - 1.7: docValues defaults to true. + 1.7: docValues defaults to true, uninvertible defaults to false. --> - - - - - + + + + + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml index 34ee4f7d038..d95c8a740c3 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-add-schema-fields-update-processor-chains.xml @@ -24,7 +24,7 @@ ${tests.luceneMatchVersion:LATEST} - + true diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-analytics-query.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-analytics-query.xml index cda45dfe364..b6d4a1cb673 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-analytics-query.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-analytics-query.xml @@ -38,7 +38,7 @@ - + 1000000 2000000 3000000 diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-basic.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-basic.xml index d98ecacf454..d28e6a6d052 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-basic.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-basic.xml @@ -23,7 +23,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-cache-enable-disable.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-cache-enable-disable.xml index d51f8050c56..96e0b6d754c 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-cache-enable-disable.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-cache-enable-disable.xml @@ -21,7 +21,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-classification.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-classification.xml index 3370600574d..038813e8dd3 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-classification.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-classification.xml @@ -24,7 +24,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-collapseqparser.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-collapseqparser.xml index 8d242f46a54..d104e8d80af 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-collapseqparser.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-collapseqparser.xml @@ -38,7 +38,7 @@ - + 1000000 2000000 3000000 diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-components-name.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-components-name.xml index 6a14a8cc91c..0ca0e186db6 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-components-name.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-components-name.xml @@ -29,7 +29,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-concurrentmergescheduler.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-concurrentmergescheduler.xml index 140c4cfeedc..3c7c7959ed0 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-concurrentmergescheduler.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-concurrentmergescheduler.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-coreproperties.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-coreproperties.xml index fc707d461fd..13bada25dd7 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-coreproperties.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-coreproperties.xml @@ -21,7 +21,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-delaying-component.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-delaying-component.xml index cfb812fa25b..7cabbd19c6f 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-delaying-component.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-delaying-component.xml @@ -21,7 +21,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy1.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy1.xml index 424783beef5..2bd4d26b150 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy1.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy1.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy2.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy2.xml index bb4d3f85ba0..8a57b7fb8e1 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy2.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-delpolicy2.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml index b23a7dc9ff6..9b56a39a52f 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-distrib-update-processor-chains.xml @@ -20,7 +20,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${solr.hdfs.blockcache.enabled:true} ${solr.hdfs.blockcache.blocksperbank:1024} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml index d814a538a50..63bb8f84ffa 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-elevate.xml @@ -28,7 +28,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-externalversionconstraint.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-externalversionconstraint.xml index 0e785ce0adc..5e366166ea1 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-externalversionconstraint.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-externalversionconstraint.xml @@ -21,7 +21,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${solr.hdfs.blockcache.enabled:true} ${solr.hdfs.blockcache.blocksperbank:1024} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower-auth.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower-auth.xml index 1635cfb099b..236a0997b5a 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower-auth.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower-auth.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${solr.data.dir:} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower.xml index fc0298ef5fd..f009ac9e59c 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${solr.data.dir:} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower1.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower1.xml index 71f4157ccb2..9ae3600b200 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower1.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-follower1.xml @@ -20,7 +20,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-functionquery.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-functionquery.xml index ef0c39ae8e4..2086a6c4ba9 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-functionquery.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-functionquery.xml @@ -25,7 +25,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-headers.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-headers.xml index 328fc9bd56a..695cd36e2a4 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-headers.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-headers.xml @@ -21,7 +21,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-highlight.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-highlight.xml index c714a4148d7..7688b3e0a7e 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-highlight.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-highlight.xml @@ -25,7 +25,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-implicitproperties.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-implicitproperties.xml index b9a72c5634a..072cae1d869 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-implicitproperties.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-implicitproperties.xml @@ -26,7 +26,7 @@ ${solr.data.dir:} + class="${solr.directoryFactory:solr.MockDirectoryFactory}"/> diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader.xml index 89c06ad3a54..ea2e8e332da 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${solr.data.dir:} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader1-keepOneBackup.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader1-keepOneBackup.xml index 36065b43ae7..40e93b8e5c5 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader1-keepOneBackup.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader1-keepOneBackup.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader1.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader1.xml index 4e4e3999224..e4afd9b9f8f 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader1.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader1.xml @@ -20,7 +20,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader2.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader2.xml index f6c2a77dde3..933337f0a8c 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader2.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader2.xml @@ -20,7 +20,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader3.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader3.xml index 49d1ed31b5a..b46fb393cf3 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader3.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-leader3.xml @@ -20,7 +20,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-logmergepolicyfactory.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-logmergepolicyfactory.xml index 539fd5c4d5c..528ea879568 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-logmergepolicyfactory.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-logmergepolicyfactory.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-lucene-codec.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-lucene-codec.xml index 1ee427dc40d..9c131263850 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-lucene-codec.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-lucene-codec.xml @@ -29,7 +29,7 @@ ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-managed-schema-test.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-managed-schema-test.xml index 666132f8e74..e3f4612e7e1 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-managed-schema-test.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-managed-schema-test.xml @@ -22,6 +22,6 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-defaults.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-defaults.xml index 3e0cf1927b6..76684eaebe8 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-defaults.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-defaults.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-legacy.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-legacy.xml index b67d6645f31..d02b9b01827 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-legacy.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicy-legacy.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicyfactory-nocfs.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicyfactory-nocfs.xml index b93fabd7c3a..f9276ba9298 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicyfactory-nocfs.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-mergepolicyfactory-nocfs.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml index 9fa236dda0b..0d32632fc94 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-minhash.xml @@ -38,7 +38,7 @@ - + 1000000 2000000 3000000 diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache.xml index fb891f822fd..f4ffe8a8502 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-nocache.xml @@ -24,7 +24,7 @@ - + ${solr.data.dir:} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-nomergepolicyfactory.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-nomergepolicyfactory.xml index 62fb05b03c4..6e5545be7c4 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-nomergepolicyfactory.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-nomergepolicyfactory.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-noopregen.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-noopregen.xml index e2a20a3379c..1799508b115 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-noopregen.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-noopregen.xml @@ -22,7 +22,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-parallel-commit.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-parallel-commit.xml index 3e619948e18..662089c170a 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-parallel-commit.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-parallel-commit.xml @@ -24,7 +24,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-paramset.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-paramset.xml index a9e71f63ebc..1b41d2b949a 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-paramset.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-paramset.xml @@ -23,7 +23,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-parsing-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-parsing-update-processor-chains.xml index 43f2d285a9f..0416d3e796e 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-parsing-update-processor-chains.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-parsing-update-processor-chains.xml @@ -24,7 +24,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml index 0f79d7c4572..65f1e9f3a4a 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-phrasesuggest.xml @@ -22,7 +22,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-plugcollector.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-plugcollector.xml index 845998ec2f4..63f2ce29b7e 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-plugcollector.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-plugcollector.xml @@ -38,7 +38,7 @@ - + 1000000 2000000 3000000 diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-pluggable-circuitbreaker.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-pluggable-circuitbreaker.xml index 52956f60824..4c8d441b2d3 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-pluggable-circuitbreaker.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-pluggable-circuitbreaker.xml @@ -21,7 +21,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-query-parser-init.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-query-parser-init.xml index eb538fa620b..0dd520baca9 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-query-parser-init.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-query-parser-init.xml @@ -24,7 +24,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender-noquery.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender-noquery.xml index 9d4f83d68c3..e6e94a2aa67 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender-noquery.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender-noquery.xml @@ -23,7 +23,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender.xml index 1404f8abb4f..f8bc376ad29 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-querysender.xml @@ -23,7 +23,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-repeater.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-repeater.xml index f5571f9827f..b426ac82f93 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-repeater.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-repeater.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${solr.data.dir:} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-replication-legacy.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-replication-legacy.xml index 43c42ffb089..ddd116be38a 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-replication-legacy.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-replication-legacy.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${solr.data.dir:} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-response-log-component.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-response-log-component.xml index 643d9a62fc7..f3bb105bd25 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-response-log-component.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-response-log-component.xml @@ -31,7 +31,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-searcher-listeners1.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-searcher-listeners1.xml index c71f8baf9d4..ebceb148579 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-searcher-listeners1.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-searcher-listeners1.xml @@ -30,7 +30,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingmergepolicyfactory.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingmergepolicyfactory.xml index 5920348551a..5d1e7242e70 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingmergepolicyfactory.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingmergepolicyfactory.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingresponse.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingresponse.xml index 1bb3c97a225..ee27bec6144 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingresponse.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-sortingresponse.xml @@ -23,7 +23,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-spatial.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-spatial.xml index 889d1c24968..f6cca32f8a4 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-spatial.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-spatial.xml @@ -22,7 +22,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml index 0253d91b804..8c7387da9c0 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellcheckcomponent.xml @@ -44,7 +44,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellchecker.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellchecker.xml index a876fb6f27c..5349edc7c8c 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellchecker.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-spellchecker.xml @@ -21,7 +21,7 @@ - + ${tests.luceneMatchVersion:LATEST} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml index c97ce085660..de0b3569270 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tagger.xml @@ -26,7 +26,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml index 00d49db3285..1020db8319e 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-test-misc.xml @@ -25,7 +25,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-testxmlparser.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-testxmlparser.xml index 401710e8d32..f9f8427707b 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-testxmlparser.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-testxmlparser.xml @@ -22,7 +22,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tieredmergepolicyfactory.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tieredmergepolicyfactory.xml index 3494db00d9c..5a73fc9a152 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tieredmergepolicyfactory.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tieredmergepolicyfactory.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml index 5203031c456..aff143c616f 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-tlog.xml @@ -21,7 +21,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${solr.hdfs.blockcache.enabled:true} ${solr.hdfs.blockcache.blocksperbank:1024} diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-transformers.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-transformers.xml index ef38e8099e9..b4e4846426d 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-transformers.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-transformers.xml @@ -23,7 +23,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml index e22ad695859..4e6d7bdd370 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-update-processor-chains.xml @@ -27,7 +27,7 @@ - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-warmer-randommergepolicyfactory.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-warmer-randommergepolicyfactory.xml index 21101b1d9e6..3c6056a3fc1 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-warmer-randommergepolicyfactory.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-warmer-randommergepolicyfactory.xml @@ -22,7 +22,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-xinclude.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-xinclude.xml index 17df214cb71..48f6676645f 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig-xinclude.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-xinclude.xml @@ -21,7 +21,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml index 5632e36cb0f..2c06096732c 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig.xml @@ -38,7 +38,7 @@ - + 1000000 2000000 3000000 diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig_SimpleTextCodec.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig_SimpleTextCodec.xml index 7b0c3e368ba..a971665edee 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig_SimpleTextCodec.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig_SimpleTextCodec.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec.xml index ad080961d9f..991d2a50f61 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec2.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec2.xml index c4a8ae73dfe..94f934dbcbe 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec2.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig_codec2.xml @@ -19,7 +19,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig_perf.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig_perf.xml index f2b5ef8a381..0c6b57c4e77 100644 --- a/solr/core/src/test-files/solr/collection1/conf/solrconfig_perf.xml +++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig_perf.xml @@ -28,7 +28,7 @@ - + diff --git a/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml index 0372460c2b2..115fea2368b 100644 --- a/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/configsets/_default/conf/solrconfig.xml @@ -30,7 +30,6 @@ ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} diff --git a/solr/core/src/test-files/solr/configsets/bad-mergepolicy/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/bad-mergepolicy/conf/solrconfig.xml index 3ef080dcca3..c286b018cef 100644 --- a/solr/core/src/test-files/solr/configsets/bad-mergepolicy/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/configsets/bad-mergepolicy/conf/solrconfig.xml @@ -19,7 +19,7 @@ - + ${tests.luceneMatchVersion:LATEST} diff --git a/solr/core/src/test-files/solr/configsets/cloud-dynamic/conf/schema.xml b/solr/core/src/test-files/solr/configsets/cloud-dynamic/conf/schema.xml index 40241c3bda5..58e6405959a 100644 --- a/solr/core/src/test-files/solr/configsets/cloud-dynamic/conf/schema.xml +++ b/solr/core/src/test-files/solr/configsets/cloud-dynamic/conf/schema.xml @@ -30,17 +30,17 @@ 1.4: default auto-phrase (QueryParser feature) to off 1.5: omitNorms defaults to true for primitive field types (int, float, boolean, string...) 1.6: useDocValuesAsStored defaults to true. - 1.7: docValues defaults to true. + 1.7: docValues defaults to true, uninvertible defaults to false. --> - - - - + + + + - - - - + + + + @@ -96,8 +96,8 @@ - - + + diff --git a/solr/core/src/test-files/solr/configsets/cloud-managed/conf/managed-schema.xml b/solr/core/src/test-files/solr/configsets/cloud-managed/conf/managed-schema.xml index 2714a21b5de..d457870208a 100644 --- a/solr/core/src/test-files/solr/configsets/cloud-managed/conf/managed-schema.xml +++ b/solr/core/src/test-files/solr/configsets/cloud-managed/conf/managed-schema.xml @@ -17,12 +17,12 @@ --> - - + + - + diff --git a/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml index b539459ad5e..fc23706512e 100644 --- a/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml +++ b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/schema.xml @@ -17,8 +17,8 @@ --> - - + + diff --git a/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml index 853ba656241..baef69fde90 100644 --- a/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/configsets/cloud-minimal/conf/solrconfig.xml @@ -24,7 +24,7 @@ ${solr.data.dir:} + class="${solr.directoryFactory:solr.MockDirectoryFactory}"/> ${tests.luceneMatchVersion:LATEST} diff --git a/solr/core/src/test-files/solr/configsets/dedup/conf/schema.xml b/solr/core/src/test-files/solr/configsets/dedup/conf/schema.xml index fadaa72eac9..2da9becb844 100644 --- a/solr/core/src/test-files/solr/configsets/dedup/conf/schema.xml +++ b/solr/core/src/test-files/solr/configsets/dedup/conf/schema.xml @@ -18,7 +18,7 @@ - + diff --git a/solr/core/src/test-files/solr/configsets/different-stopwords/collectionA/conf/schema.xml b/solr/core/src/test-files/solr/configsets/different-stopwords/collectionA/conf/schema.xml index 1875c733f96..0d788eb038a 100644 --- a/solr/core/src/test-files/solr/configsets/different-stopwords/collectionA/conf/schema.xml +++ b/solr/core/src/test-files/solr/configsets/different-stopwords/collectionA/conf/schema.xml @@ -30,7 +30,7 @@ 1.4: default auto-phrase (QueryParser feature) to off 1.5: omitNorms defaults to true for primitive field types (int, float, boolean, string...) 1.6: useDocValuesAsStored defaults to true. - 1.7: docValues defaults to true. + 1.7: docValues defaults to true, uninvertible defaults to false. --> diff --git a/solr/core/src/test-files/solr/configsets/different-stopwords/collectionB/conf/schema.xml b/solr/core/src/test-files/solr/configsets/different-stopwords/collectionB/conf/schema.xml index 98fc0a8c5c9..37692c15943 100644 --- a/solr/core/src/test-files/solr/configsets/different-stopwords/collectionB/conf/schema.xml +++ b/solr/core/src/test-files/solr/configsets/different-stopwords/collectionB/conf/schema.xml @@ -30,7 +30,7 @@ 1.4: default auto-phrase (QueryParser feature) to off 1.5: omitNorms defaults to true for primitive field types (int, float, boolean, string...) 1.6: useDocValuesAsStored defaults to true. - 1.7: docValues defaults to true. + 1.7: docValues defaults to true, uninvertible defaults to false. --> diff --git a/solr/core/src/test-files/solr/configsets/doc-expiry/conf/schema.xml b/solr/core/src/test-files/solr/configsets/doc-expiry/conf/schema.xml index 62d52896700..7dd56672a61 100644 --- a/solr/core/src/test-files/solr/configsets/doc-expiry/conf/schema.xml +++ b/solr/core/src/test-files/solr/configsets/doc-expiry/conf/schema.xml @@ -30,17 +30,17 @@ 1.4: default auto-phrase (QueryParser feature) to off 1.5: omitNorms defaults to true for primitive field types (int, float, boolean, string...) 1.6: useDocValuesAsStored defaults to true. - 1.7: docValues defaults to true. + 1.7: docValues defaults to true, uninvertible defaults to false. --> - - - - + + + + - - - - + + + + @@ -157,7 +157,7 @@ - + @@ -177,16 +177,16 @@ - + - + - + - + @@ -221,10 +221,10 @@ - + - + diff --git a/solr/core/src/test-files/solr/configsets/exitable-directory/conf/schema.xml b/solr/core/src/test-files/solr/configsets/exitable-directory/conf/schema.xml index 4ade7650b81..e9090724541 100644 --- a/solr/core/src/test-files/solr/configsets/exitable-directory/conf/schema.xml +++ b/solr/core/src/test-files/solr/configsets/exitable-directory/conf/schema.xml @@ -16,11 +16,11 @@ limitations under the License. --> - + - - + + diff --git a/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml index 515ad64d7f5..7e1f3434e96 100644 --- a/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml @@ -22,7 +22,7 @@ ${tests.luceneMatchVersion:LATEST} - + ${solr.hdfs.blockcache.enabled:true} ${solr.hdfs.blockcache.blocksperbank:1024} diff --git a/solr/core/src/test-files/solr/configsets/minimal/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/minimal/conf/solrconfig.xml index 346b0448318..4e90243daf6 100644 --- a/solr/core/src/test-files/solr/configsets/minimal/conf/solrconfig.xml +++ b/solr/core/src/test-files/solr/configsets/minimal/conf/solrconfig.xml @@ -24,7 +24,7 @@ ${solr.data.dir:} + class="${solr.directoryFactory:solr.MockDirectoryFactory}"/> ${tests.luceneMatchVersion:LATEST} diff --git a/solr/core/src/test/org/apache/solr/cli/AuthToolTest.java b/solr/core/src/test/org/apache/solr/cli/AuthToolTest.java index 60f5271c12f..8c32db28026 100644 --- a/solr/core/src/test/org/apache/solr/cli/AuthToolTest.java +++ b/solr/core/src/test/org/apache/solr/cli/AuthToolTest.java @@ -70,7 +70,7 @@ public void testEnableAuth() throws Exception { dir.toAbsolutePath().toString(), "--solr-include-file", solrIncludeFile.toAbsolutePath().toString(), - "-credentials", + "--credentials", "solr:solr", "--block-unknown", "true" diff --git a/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java b/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java index 333b541f7e6..838c3857ce9 100644 --- a/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java +++ b/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java @@ -49,7 +49,7 @@ public void testCreateCollectionWithBasicAuth() throws Exception { "cloud-minimal", "-z", cluster.getZkClient().getZkServerAddress(), - "-credentials", + "--credentials", SecurityJson.USER_PASS, "-verbose" }; diff --git a/solr/core/src/test/org/apache/solr/cli/DeleteToolTest.java b/solr/core/src/test/org/apache/solr/cli/DeleteToolTest.java index 535ffc75573..86459b1f71a 100644 --- a/solr/core/src/test/org/apache/solr/cli/DeleteToolTest.java +++ b/solr/core/src/test/org/apache/solr/cli/DeleteToolTest.java @@ -64,7 +64,7 @@ public void testDeleteCollectionWithBasicAuth() throws Exception { "false", "-z", cluster.getZkClient().getZkServerAddress(), - "-credentials", + "--credentials", SecurityJson.USER_PASS, "-verbose" }; diff --git a/solr/core/src/test/org/apache/solr/cli/PostToolTest.java b/solr/core/src/test/org/apache/solr/cli/PostToolTest.java index 1f507b281b5..6f7d167e106 100644 --- a/solr/core/src/test/org/apache/solr/cli/PostToolTest.java +++ b/solr/core/src/test/org/apache/solr/cli/PostToolTest.java @@ -127,7 +127,7 @@ public void testRunWithCollectionParam() throws Exception { fw.flush(); String[] args = { - "post", "-c", collection, "-credentials", SecurityJson.USER_PASS, jsonDoc.getAbsolutePath() + "post", "-c", collection, "--credentials", SecurityJson.USER_PASS, jsonDoc.getAbsolutePath() }; assertEquals(0, runTool(args)); @@ -167,7 +167,7 @@ public void testRunCsvWithCustomSeparatorParam() throws Exception { "post", "-c", collection, - "-credentials", + "--credentials", SecurityJson.USER_PASS, "--params", "\"separator=%09&header=false&fieldnames=id,title_s\"", diff --git a/solr/core/src/test/org/apache/solr/cli/SolrCLIZkToolsTest.java b/solr/core/src/test/org/apache/solr/cli/SolrCLIZkToolsTest.java index 3da316a1702..2470d01dbf8 100644 --- a/solr/core/src/test/org/apache/solr/cli/SolrCLIZkToolsTest.java +++ b/solr/core/src/test/org/apache/solr/cli/SolrCLIZkToolsTest.java @@ -768,7 +768,8 @@ public void testRm() throws Exception { int res = tool.runTool(SolrCLI.processCommandLineArgs(tool, args)); assertTrue( - "Should have failed to remove node with children unless -recurse is set to true", res != 0); + "Should have failed to remove node with children unless --recurse is set to true", + res != 0); // Are we sure all the znodes are still there? verifyZkLocalPathsMatch(srcPathCheck, "/configs/rm1"); @@ -783,7 +784,7 @@ public void testRm() throws Exception { res = tool.runTool(SolrCLI.processCommandLineArgs(tool, args)); assertTrue( - "Should have failed to remove node with children if -recurse is set to false", res != 0); + "Should have failed to remove node with children if --recurse is set to false", res != 0); args = new String[] { diff --git a/solr/core/src/test/org/apache/solr/cli/TestExportTool.java b/solr/core/src/test/org/apache/solr/cli/TestExportTool.java index 1223925058b..357fd26d690 100644 --- a/solr/core/src/test/org/apache/solr/cli/TestExportTool.java +++ b/solr/core/src/test/org/apache/solr/cli/TestExportTool.java @@ -245,7 +245,7 @@ public void testWithBasicAuth() throws Exception { "export", "-url", cluster.getJettySolrRunner(0).getBaseUrl() + "/" + COLLECTION_NAME, - "-credentials", + "--credentials", SecurityJson.USER_PASS, "-out", outFile.getAbsolutePath(), diff --git a/solr/core/src/test/org/apache/solr/cli/ZkSubcommandsTest.java b/solr/core/src/test/org/apache/solr/cli/ZkSubcommandsTest.java index 8185c5b1457..250cee1a955 100644 --- a/solr/core/src/test/org/apache/solr/cli/ZkSubcommandsTest.java +++ b/solr/core/src/test/org/apache/solr/cli/ZkSubcommandsTest.java @@ -635,14 +635,14 @@ public void testSetClusterProperty() throws Exception { // add property urlScheme=http String[] args = new String[] { - "cluster", "-property", "urlScheme", "-value", "http", "-z", zkServer.getZkAddress() + "cluster", "--property", "urlScheme", "--value", "http", "-z", zkServer.getZkAddress() }; ClusterTool tool = new ClusterTool(); assertEquals(0, runTool(args, tool)); assertEquals("http", properties.getClusterProperty("urlScheme", "none")); - args = new String[] {"cluster", "-property", "urlScheme", "-z", zkServer.getZkAddress()}; + args = new String[] {"cluster", "--property", "urlScheme", "-z", zkServer.getZkAddress()}; assertEquals(0, runTool(args, tool)); assertNull(properties.getClusterProperty("urlScheme", (String) null)); } diff --git a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java index 6fc0f697105..a90e059e03f 100644 --- a/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java +++ b/solr/core/src/test/org/apache/solr/handler/admin/MetricsHandlerTest.java @@ -40,7 +40,7 @@ import org.apache.solr.metrics.MetricsMap; import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.metrics.SolrMetricsContext; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrRequestHandler; import org.apache.solr.response.SolrQueryResponse; @@ -67,7 +67,7 @@ public static void beforeClass() throws Exception { c = h.getCoreContainer().getMetricManager().counter(null, "solr.jetty", "solrtest_foo:bar"); c.inc(3); - // Manually register for Prometheus exporter tests + // Manually register for Prometheus Formatter tests registerGauge("solr.jvm", "gc.G1-Old-Generation.count"); registerGauge("solr.jvm", "gc.G1-Old-Generation.time"); registerGauge("solr.jvm", "memory.heap.committed"); @@ -725,9 +725,10 @@ public void testPrometheusMetricsCore() throws Exception { NamedList values = resp.getValues(); assertNotNull(values.get("metrics")); values = (NamedList) values.get("metrics"); - SolrPrometheusExporter exporter = (SolrPrometheusExporter) values.get("solr.core.collection1"); - assertNotNull(exporter); - MetricSnapshots actualSnapshots = exporter.collect(); + SolrPrometheusFormatter formatter = + (SolrPrometheusFormatter) values.get("solr.core.collection1"); + assertNotNull(formatter); + MetricSnapshots actualSnapshots = formatter.collect(); assertNotNull(actualSnapshots); MetricSnapshot actualSnapshot = @@ -824,9 +825,9 @@ public void testPrometheusMetricsNode() throws Exception { NamedList values = resp.getValues(); assertNotNull(values.get("metrics")); values = (NamedList) values.get("metrics"); - SolrPrometheusExporter exporter = (SolrPrometheusExporter) values.get("solr.node"); - assertNotNull(exporter); - MetricSnapshots actualSnapshots = exporter.collect(); + SolrPrometheusFormatter formatter = (SolrPrometheusFormatter) values.get("solr.node"); + assertNotNull(formatter); + MetricSnapshots actualSnapshots = formatter.collect(); assertNotNull(actualSnapshots); MetricSnapshot actualSnapshot = @@ -908,9 +909,9 @@ public void testPrometheusMetricsJvm() throws Exception { NamedList values = resp.getValues(); assertNotNull(values.get("metrics")); values = (NamedList) values.get("metrics"); - SolrPrometheusExporter exporter = (SolrPrometheusExporter) values.get("solr.jvm"); - assertNotNull(exporter); - MetricSnapshots actualSnapshots = exporter.collect(); + SolrPrometheusFormatter formatter = (SolrPrometheusFormatter) values.get("solr.jvm"); + assertNotNull(formatter); + MetricSnapshots actualSnapshots = formatter.collect(); assertNotNull(actualSnapshots); MetricSnapshot actualSnapshot = getMetricSnapshot(actualSnapshots, "solr_metrics_jvm_threads"); @@ -962,9 +963,9 @@ public void testPrometheusMetricsJetty() throws Exception { NamedList values = resp.getValues(); assertNotNull(values.get("metrics")); values = (NamedList) values.get("metrics"); - SolrPrometheusExporter exporter = (SolrPrometheusExporter) values.get("solr.jetty"); - assertNotNull(exporter); - MetricSnapshots actualSnapshots = exporter.collect(); + SolrPrometheusFormatter formatter = (SolrPrometheusFormatter) values.get("solr.jetty"); + assertNotNull(formatter); + MetricSnapshots actualSnapshots = formatter.collect(); assertNotNull(actualSnapshots); MetricSnapshot actualSnapshot = @@ -1008,9 +1009,10 @@ public void testPrometheusMetricsFilter() throws Exception { NamedList values = resp.getValues(); assertNotNull(values.get("metrics")); values = (NamedList) values.get("metrics"); - SolrPrometheusExporter exporter = (SolrPrometheusExporter) values.get("solr.core.collection1"); - assertNotNull(exporter); - MetricSnapshots actualSnapshots = exporter.collect(); + SolrPrometheusFormatter formatter = + (SolrPrometheusFormatter) values.get("solr.core.collection1"); + assertNotNull(formatter); + MetricSnapshots actualSnapshots = formatter.collect(); assertNotNull(actualSnapshots); actualSnapshots.forEach( diff --git a/solr/core/src/test/org/apache/solr/handler/component/DebugComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/DebugComponentTest.java index dcc97e7fe2a..9778a4bb078 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/DebugComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/DebugComponentTest.java @@ -297,4 +297,14 @@ private void addRequestId(ResponseBuilder rb, String requestId) { params.add(CommonParams.REQUEST_ID, requestId); rb.req.setParams(params); } + + @Test + public void testDistributedStageResolution() throws IOException { + final DebugComponent debugComponent = new DebugComponent(); + assertEquals( + "PARSE_QUERY", debugComponent.getDistributedStageName(ResponseBuilder.STAGE_PARSE_QUERY)); + assertEquals("DONE", debugComponent.getDistributedStageName(ResponseBuilder.STAGE_DONE)); + assertEquals("STAGE_400", debugComponent.getDistributedStageName(400)); + assertEquals("STAGE_10000", debugComponent.getDistributedStageName(10000)); + } } diff --git a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java index 90a7685d83e..f059acb8a6c 100644 --- a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java +++ b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java @@ -839,6 +839,7 @@ public void testSchemaDiffEndpoint() throws Exception { SimpleOrderedMap idFieldMapUpdated = idFieldMap.clone(); idFieldMapUpdated.setVal(idFieldMapUpdated.indexOf("docValues", 0), Boolean.FALSE); idFieldMapUpdated.setVal(idFieldMapUpdated.indexOf("useDocValuesAsStored", 0), Boolean.FALSE); + idFieldMapUpdated.setVal(idFieldMapUpdated.indexOf("uninvertible", 0), Boolean.TRUE); idFieldMapUpdated.setVal( idFieldMapUpdated.indexOf("omitTermFreqAndPositions", 0), Boolean.FALSE); @@ -892,14 +893,23 @@ public void testSchemaDiffEndpoint() throws Exception { assertEquals( Arrays.asList( Map.of( - "omitTermFreqAndPositions", true, "useDocValuesAsStored", true, "docValues", true), + "omitTermFreqAndPositions", + true, + "useDocValuesAsStored", + true, + "docValues", + true, + "uninvertible", + false), Map.of( "omitTermFreqAndPositions", false, "useDocValuesAsStored", false, "docValues", - false)), + false, + "uninvertible", + true)), mapDiff.get("id")); assertNotNull(fieldsDiff.get("added")); Map fieldsAdded = (Map) fieldsDiff.get("added"); diff --git a/solr/core/src/test/org/apache/solr/metrics/SolrCoreMetricTest.java b/solr/core/src/test/org/apache/solr/metrics/SolrCoreMetricTest.java index 3a8b6e58cd7..19a9fce2cbe 100644 --- a/solr/core/src/test/org/apache/solr/metrics/SolrCoreMetricTest.java +++ b/solr/core/src/test/org/apache/solr/metrics/SolrCoreMetricTest.java @@ -19,7 +19,7 @@ import com.codahale.metrics.Metric; import io.prometheus.metrics.model.snapshots.Labels; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; import org.apache.solr.metrics.prometheus.core.SolrCoreMetric; import org.junit.Test; @@ -76,6 +76,6 @@ public SolrCoreMetric parseLabels() { } @Override - public void toPrometheus(SolrPrometheusExporter exporter) {} + public void toPrometheus(SolrPrometheusFormatter formatter) {} } } diff --git a/solr/core/src/test/org/apache/solr/metrics/SolrPrometheusExporterTest.java b/solr/core/src/test/org/apache/solr/metrics/SolrPrometheusFormatterTest.java similarity index 72% rename from solr/core/src/test/org/apache/solr/metrics/SolrPrometheusExporterTest.java rename to solr/core/src/test/org/apache/solr/metrics/SolrPrometheusFormatterTest.java index 445e02f326f..186c83c8a91 100644 --- a/solr/core/src/test/org/apache/solr/metrics/SolrPrometheusExporterTest.java +++ b/solr/core/src/test/org/apache/solr/metrics/SolrPrometheusFormatterTest.java @@ -31,47 +31,47 @@ import java.util.concurrent.TimeUnit; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.metrics.prometheus.SolrMetric; -import org.apache.solr.metrics.prometheus.SolrPrometheusExporter; +import org.apache.solr.metrics.prometheus.SolrPrometheusFormatter; import org.junit.Test; -public class SolrPrometheusExporterTest extends SolrTestCaseJ4 { +public class SolrPrometheusFormatterTest extends SolrTestCaseJ4 { @Test public void testExportMeter() { - TestSolrPrometheusExporter testExporter = new TestSolrPrometheusExporter(); + TestSolrPrometheusFormatter testFormatter = new TestSolrPrometheusFormatter(); String expectedMetricName = "test_metric"; Meter metric = new Meter(); metric.mark(123); Labels expectedLabels = Labels.of("test", "test-value"); - testExporter.exportMeter(expectedMetricName, metric, expectedLabels); - assertTrue(testExporter.getMetricCounters().containsKey(expectedMetricName)); + testFormatter.exportMeter(expectedMetricName, metric, expectedLabels); + assertTrue(testFormatter.getMetricCounters().containsKey(expectedMetricName)); CounterSnapshot.CounterDataPointSnapshot actual = - testExporter.getMetricCounters().get("test_metric").get(0); + testFormatter.getMetricCounters().get("test_metric").get(0); assertEquals(123.0, actual.getValue(), 0); assertEquals(expectedLabels, actual.getLabels()); } @Test public void testExportCounter() { - TestSolrPrometheusExporter testExporter = new TestSolrPrometheusExporter(); + TestSolrPrometheusFormatter testFormatter = new TestSolrPrometheusFormatter(); String expectedMetricName = "test_metric"; Counter metric = new Counter(); metric.inc(123); Labels expectedLabels = Labels.of("test", "test-value"); - testExporter.exportCounter(expectedMetricName, metric, expectedLabels); - assertTrue(testExporter.getMetricCounters().containsKey(expectedMetricName)); + testFormatter.exportCounter(expectedMetricName, metric, expectedLabels); + assertTrue(testFormatter.getMetricCounters().containsKey(expectedMetricName)); CounterSnapshot.CounterDataPointSnapshot actual = - testExporter.getMetricCounters().get("test_metric").get(0); + testFormatter.getMetricCounters().get("test_metric").get(0); assertEquals(123.0, actual.getValue(), 0); assertEquals(expectedLabels, actual.getLabels()); } @Test public void testExportTimer() throws InterruptedException { - TestSolrPrometheusExporter testExporter = new TestSolrPrometheusExporter(); + TestSolrPrometheusFormatter testFormatter = new TestSolrPrometheusFormatter(); String expectedMetricName = "test_metric"; Timer metric = new Timer(); Timer.Context context = metric.time(); @@ -79,18 +79,18 @@ public void testExportTimer() throws InterruptedException { context.stop(); Labels expectedLabels = Labels.of("test", "test-value"); - testExporter.exportTimer(expectedMetricName, metric, expectedLabels); - assertTrue(testExporter.getMetricGauges().containsKey(expectedMetricName)); + testFormatter.exportTimer(expectedMetricName, metric, expectedLabels); + assertTrue(testFormatter.getMetricGauges().containsKey(expectedMetricName)); GaugeSnapshot.GaugeDataPointSnapshot actual = - testExporter.getMetricGauges().get("test_metric").get(0); + testFormatter.getMetricGauges().get("test_metric").get(0); assertEquals(5000000000L, actual.getValue(), 500000000L); assertEquals(expectedLabels, actual.getLabels()); } @Test public void testExportGaugeNumber() throws InterruptedException { - TestSolrPrometheusExporter testExporter = new TestSolrPrometheusExporter(); + TestSolrPrometheusFormatter testFormatter = new TestSolrPrometheusFormatter(); String expectedMetricName = "test_metric"; Gauge metric = new SettableGauge<>() { @@ -104,18 +104,18 @@ public Number getValue() { }; Labels expectedLabels = Labels.of("test", "test-value"); - testExporter.exportGauge(expectedMetricName, metric, expectedLabels); - assertTrue(testExporter.getMetricGauges().containsKey(expectedMetricName)); + testFormatter.exportGauge(expectedMetricName, metric, expectedLabels); + assertTrue(testFormatter.getMetricGauges().containsKey(expectedMetricName)); GaugeSnapshot.GaugeDataPointSnapshot actual = - testExporter.getMetricGauges().get("test_metric").get(0); + testFormatter.getMetricGauges().get("test_metric").get(0); assertEquals(123.0, actual.getValue(), 0); assertEquals(expectedLabels, actual.getLabels()); } @Test public void testExportGaugeMap() throws InterruptedException { - TestSolrPrometheusExporter testExporter = new TestSolrPrometheusExporter(); + TestSolrPrometheusFormatter testFormatter = new TestSolrPrometheusFormatter(); String expectedMetricName = "test_metric"; Gauge> metric = new SettableGauge<>() { @@ -131,17 +131,17 @@ public Map getValue() { }; Labels labels = Labels.of("test", "test-value"); - testExporter.exportGauge(expectedMetricName, metric, labels); - assertTrue(testExporter.getMetricGauges().containsKey(expectedMetricName)); + testFormatter.exportGauge(expectedMetricName, metric, labels); + assertTrue(testFormatter.getMetricGauges().containsKey(expectedMetricName)); GaugeSnapshot.GaugeDataPointSnapshot actual = - testExporter.getMetricGauges().get("test_metric").get(0); + testFormatter.getMetricGauges().get("test_metric").get(0); assertEquals(123.0, actual.getValue(), 0); Labels expectedLabels = Labels.of("test", "test-value", "item", "test-item"); assertEquals(expectedLabels, actual.getLabels()); } - static class TestSolrPrometheusExporter extends SolrPrometheusExporter { + static class TestSolrPrometheusFormatter extends SolrPrometheusFormatter { @Override public void exportDropwizardMetric(Metric dropwizardMetric, String metricName) {} diff --git a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java index c183a2610f6..0e25201cce0 100644 --- a/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java +++ b/solr/core/src/test/org/apache/solr/response/TestPrometheusResponseWriter.java @@ -16,9 +16,15 @@ */ package org.apache.solr.response; +import com.codahale.metrics.Counter; import com.codahale.metrics.Gauge; +import com.codahale.metrics.Meter; import com.codahale.metrics.SettableGauge; -import java.nio.file.Files; +import com.codahale.metrics.SharedMetricRegistries; +import java.lang.invoke.MethodHandles; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrClient; @@ -26,65 +32,126 @@ import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; -import org.apache.solr.embedded.JettySolrRunner; import org.apache.solr.metrics.SolrMetricManager; import org.apache.solr.util.ExternalPaths; import org.apache.solr.util.SolrJettyTestRule; +import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.ClassRule; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** Tests the {@link PrometheusResponseWriter} behavior */ -@LuceneTestCase.AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/SOLR-17368") public class TestPrometheusResponseWriter extends SolrTestCaseJ4 { - @ClassRule public static SolrJettyTestRule solrClientTestRule = new SolrJettyTestRule(); - public static JettySolrRunner jetty; + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + public static final List VALID_PROMETHEUS_VALUES = Arrays.asList("NaN", "+Inf", "-Inf"); @BeforeClass public static void beforeClass() throws Exception { + SharedMetricRegistries.clear(); + solrClientTestRule.startSolr(LuceneTestCase.createTempDir()); - jetty = solrClientTestRule.getJetty(); solrClientTestRule.newCollection().withConfigSet(ExternalPaths.DEFAULT_CONFIGSET).create(); - jetty.getCoreContainer().waitForLoadingCoresToFinish(30000); - // Manually register metrics not initializing from JettyTestRule - registerGauge( - jetty.getCoreContainer().getMetricManager(), - "solr.jvm", - "buffers.mapped - 'non-volatile memory'.Count"); - registerGauge( - jetty.getCoreContainer().getMetricManager(), - "solr.jvm", - "buffers.mapped - 'non-volatile memory'.MemoryUsed"); - registerGauge( - jetty.getCoreContainer().getMetricManager(), - "solr.jvm", - "buffers.mapped - 'non-volatile memory'.TotalCapacity"); - registerGauge(jetty.getCoreContainer().getMetricManager(), "solr.jvm", "os.cpuLoad"); - registerGauge(jetty.getCoreContainer().getMetricManager(), "solr.jvm", "os.freeMemorySize"); - registerGauge(jetty.getCoreContainer().getMetricManager(), "solr.jvm", "os.totalMemorySize"); + var cc = solrClientTestRule.getCoreContainer(); + cc.waitForLoadingCoresToFinish(30000); + + SolrMetricManager manager = cc.getMetricManager(); + Counter c = manager.counter(null, "solr.core.collection1", "QUERY./dummy/metrics.requests"); + c.inc(10); + c = manager.counter(null, "solr.node", "ADMIN./dummy/metrics.requests"); + c.inc(20); + Meter m = manager.meter(null, "solr.jetty", "dummyMetrics.2xx-responses"); + m.mark(30); + registerGauge(manager, "solr.jvm", "gc.dummyMetrics.count"); + } + + @AfterClass + public static void clearMetricsRegistries() { + SharedMetricRegistries.clear(); } @Test - public void testPrometheusOutput() throws Exception { + public void testPrometheusStructureOutput() throws Exception { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("qt", "/admin/metrics"); + params.set("wt", "prometheus"); + QueryRequest req = new QueryRequest(params); + req.setResponseParser(new NoOpResponseParser("prometheus")); + + try (SolrClient adminClient = getHttpSolrClient(solrClientTestRule.getBaseUrl())) { + NamedList res = adminClient.request(req); + assertNotNull("null response from server", res); + String output = (String) res.get("response"); + List filteredResponse = + output.lines().filter(line -> !line.startsWith("#")).collect(Collectors.toList()); + filteredResponse.forEach( + (actualMetric) -> { + String actualValue; + if (actualMetric.contains("}")) { + actualValue = actualMetric.substring(actualMetric.lastIndexOf("} ") + 1); + } else { + actualValue = actualMetric.split(" ")[1]; + } + assertTrue( + "All metrics should start with 'solr_metrics_'", + actualMetric.startsWith("solr_metrics_")); + try { + Float.parseFloat(actualValue); + } catch (NumberFormatException e) { + log.warn("Prometheus value not a parsable float"); + assertTrue(VALID_PROMETHEUS_VALUES.contains(actualValue)); + } + }); + } + } + + public void testPrometheusDummyOutput() throws Exception { + String expectedCore = + "solr_metrics_core_requests_total{category=\"QUERY\",core=\"collection1\",handler=\"/dummy/metrics\",type=\"requests\"} 10.0"; + String expectedNode = + "solr_metrics_node_requests_total{category=\"ADMIN\",handler=\"/dummy/metrics\",type=\"requests\"} 20.0"; + String expectedJetty = "solr_metrics_jetty_response_total{status=\"2xx\"} 30.0"; + String expectedJvm = "solr_metrics_jvm_gc{item=\"dummyMetrics\"} 0.0"; + ModifiableSolrParams params = new ModifiableSolrParams(); params.set("qt", "/admin/metrics"); params.set("wt", "prometheus"); QueryRequest req = new QueryRequest(params); req.setResponseParser(new NoOpResponseParser("prometheus")); - try (SolrClient adminClient = getHttpSolrClient(jetty.getBaseUrl().toString()); ) { + + try (SolrClient adminClient = getHttpSolrClient(solrClientTestRule.getBaseUrl())) { NamedList res = adminClient.request(req); assertNotNull("null response from server", res); - String actual = (String) res.get("response"); - String expectedOutput = - Files.readString(getFile("prometheus/solr-prometheus-output.txt").toPath()); - // Expression to strip out ending numeric values and JVM item tag as we only want to test for - // Prometheus metric names - actual = - actual.replaceAll( - "(?<=}).*|(?<=solr_metrics_jvm_gc\\{)(.*)(?=})|(?<=solr_metrics_jvm_gc_seconds\\{)(.*)(?=})", - ""); - assertEquals(expectedOutput, actual); + String output = (String) res.get("response"); + assertEquals( + expectedCore, + output + .lines() + .filter(line -> line.contains(expectedCore)) + .collect(Collectors.toList()) + .get(0)); + assertEquals( + expectedNode, + output + .lines() + .filter(line -> line.contains(expectedNode)) + .collect(Collectors.toList()) + .get(0)); + assertEquals( + expectedJetty, + output + .lines() + .filter(line -> line.contains(expectedJetty)) + .collect(Collectors.toList()) + .get(0)); + assertEquals( + expectedJvm, + output + .lines() + .filter(line -> line.contains(expectedJvm)) + .collect(Collectors.toList()) + .get(0)); } } diff --git a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldResource.java b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldResource.java index f019e73aef2..5eb97247a39 100644 --- a/solr/core/src/test/org/apache/solr/rest/schema/TestFieldResource.java +++ b/solr/core/src/test/org/apache/solr/rest/schema/TestFieldResource.java @@ -30,7 +30,7 @@ public void testGetField() { "/response/lst[@name='field']/str[@name='type'] = 'text'", "/response/lst[@name='field']/bool[@name='indexed'] = 'true'", "/response/lst[@name='field']/bool[@name='stored'] = 'true'", - "/response/lst[@name='field']/bool[@name='uninvertible'] = 'true'", + "/response/lst[@name='field']/bool[@name='uninvertible'] = 'false'", "/response/lst[@name='field']/bool[@name='docValues'] = 'false'", "/response/lst[@name='field']/bool[@name='termVectors'] = 'true'", "/response/lst[@name='field']/bool[@name='termPositions'] = 'true'", @@ -64,7 +64,7 @@ public void testJsonGetField() throws Exception { "/field/type=='text'", "/field/indexed==true", "/field/stored==true", - "/field/uninvertible==true", + "/field/uninvertible==false", "/field/docValues==false", "/field/termVectors==true", "/field/termPositions==true", diff --git a/solr/core/src/test/org/apache/solr/schema/SchemaVersionSpecificBehaviorTest.java b/solr/core/src/test/org/apache/solr/schema/SchemaVersionSpecificBehaviorTest.java index 0ddbc7a28a6..41d64fb3cb6 100644 --- a/solr/core/src/test/org/apache/solr/schema/SchemaVersionSpecificBehaviorTest.java +++ b/solr/core/src/test/org/apache/solr/schema/SchemaVersionSpecificBehaviorTest.java @@ -76,9 +76,11 @@ public void testVersionBehavior() throws Exception { ((v < 1.7F || f.contains("text")) ? false : true), field.hasDocValues()); - // uninvertable defaults to true (for now) - assertTrue( - f + " field's type has wrong uninvertable for ver=" + ver, field.isUninvertible()); + // 1.7: uninvertible defaults to false + assertEquals( + f + " field's type has wrong uninvertible for ver=" + ver, + (v < 1.7F ? true : false), + field.isUninvertible()); } // regardless of version, explicit multiValued values on field or type diff --git a/solr/core/src/test/org/apache/solr/search/CallerSpecificQueryLimit.java b/solr/core/src/test/org/apache/solr/search/CallerSpecificQueryLimit.java index 0d53b8653db..96a6bd5ee93 100644 --- a/solr/core/src/test/org/apache/solr/search/CallerSpecificQueryLimit.java +++ b/solr/core/src/test/org/apache/solr/search/CallerSpecificQueryLimit.java @@ -22,7 +22,6 @@ import java.util.Map; import java.util.Optional; import java.util.Set; -import org.apache.lucene.index.QueryTimeout; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -32,7 +31,7 @@ * and optionally a method name, e.g. MoreLikeThisComponent or * ClusteringComponent.finishStage. */ -public class CallerSpecificQueryLimit implements QueryTimeout { +public class CallerSpecificQueryLimit implements QueryLimit { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); final StackWalker stackWalker = @@ -102,4 +101,9 @@ public boolean shouldExit() { } return matchingExpr.isPresent(); } + + @Override + public Object currentValue() { + return "This class just for testing, not a real limit"; + } } diff --git a/solr/core/src/test/org/apache/solr/search/RankQueryTest.java b/solr/core/src/test/org/apache/solr/search/RankQueryTest.java index 8bcdea1d50c..f9c3a8fa043 100644 --- a/solr/core/src/test/org/apache/solr/search/RankQueryTest.java +++ b/solr/core/src/test/org/apache/solr/search/RankQueryTest.java @@ -16,8 +16,19 @@ */ package org.apache.solr.search; +import java.io.IOException; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.TopDocsCollector; +import org.apache.lucene.util.PriorityQueue; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.handler.component.MergeStrategy; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -106,4 +117,64 @@ public void testPluggableCollector() { "//result/doc[2]/str[@name='id'][.='6']", "//result/doc[1]/str[@name='id'][.='5']"); } + + // The following static classes are intended to ensure that support of covariant + // ScoreDocs is supported in rank queries. MyRankQuery will fail to compile + // if covariant ScoreDocs are not supported because it returns a TopDocsCollector + // for MyScoreDoc (a subtype of ScoreDoc). + static class MyScoreDoc extends ScoreDoc { + public int someOtherField; + + public MyScoreDoc(int doc, float score, int shardIndex, int someOtherField) { + super(doc, score, shardIndex); + this.someOtherField = someOtherField; + } + } + + static class MyTopDocsCollector extends TopDocsCollector { + public MyTopDocsCollector(PriorityQueue pq) { + super(pq); + } + + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE; + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { + return null; + } + } + + static class MyRankQuery extends RankQuery { + @Override + public TopDocsCollector getTopDocsCollector( + int len, QueryCommand cmd, IndexSearcher searcher) throws IOException { + return new MyTopDocsCollector(null); + } + + @Override + public MergeStrategy getMergeStrategy() { + return null; + } + + @Override + public RankQuery wrap(Query mainQuery) { + return this; + } + + @Override + public void visit(QueryVisitor visitor) {} + + @Override + public int hashCode() { + return 1; + } + + @Override + public boolean equals(Object obj) { + return true; + } + } } diff --git a/solr/core/src/test/org/apache/solr/search/RankQueryTestPlugin.java b/solr/core/src/test/org/apache/solr/search/RankQueryTestPlugin.java index b11728e0283..c82661200b3 100644 --- a/solr/core/src/test/org/apache/solr/search/RankQueryTestPlugin.java +++ b/solr/core/src/test/org/apache/solr/search/RankQueryTestPlugin.java @@ -140,7 +140,7 @@ public TestRankQuery(int collector, int mergeStrategy) { } @Override - public TopDocsCollector getTopDocsCollector( + public TopDocsCollector getTopDocsCollector( int len, QueryCommand cmd, IndexSearcher searcher) { if (collector == 0) return new TestCollector(null); else return new TestCollector1(null); diff --git a/solr/core/src/test/org/apache/solr/search/SolrIndexSearcherTest.java b/solr/core/src/test/org/apache/solr/search/SolrIndexSearcherTest.java index be7f458587b..49fcc21e45e 100644 --- a/solr/core/src/test/org/apache/solr/search/SolrIndexSearcherTest.java +++ b/solr/core/src/test/org/apache/solr/search/SolrIndexSearcherTest.java @@ -327,7 +327,7 @@ public String toString(String field) { } @Override - public TopDocsCollector getTopDocsCollector( + public TopDocsCollector getTopDocsCollector( int len, QueryCommand cmd, IndexSearcher searcher) throws IOException { return new ReRankCollector( len, diff --git a/solr/core/src/test/org/apache/solr/search/TestCoordinatorRole.java b/solr/core/src/test/org/apache/solr/search/TestCoordinatorRole.java index 70835d7e884..f75448e8b82 100644 --- a/solr/core/src/test/org/apache/solr/search/TestCoordinatorRole.java +++ b/solr/core/src/test/org/apache/solr/search/TestCoordinatorRole.java @@ -863,4 +863,67 @@ public void testMoveReplica() throws Exception { cluster.shutdown(); } } + + public void testCoreReload() throws Exception { + final int DATA_NODE_COUNT = 1; + MiniSolrCloudCluster cluster = + configureCluster(DATA_NODE_COUNT) + .addConfig("conf1", configset("cloud-minimal")) + .configure(); + + List dataNodes = + cluster.getJettySolrRunners().stream() + .map(JettySolrRunner::getNodeName) + .collect(Collectors.toUnmodifiableList()); + + try { + CollectionAdminRequest.createCollection("c1", "conf1", 1, 1).process(cluster.getSolrClient()); + cluster.waitForActiveCollection("c1", 1, 1); + + System.setProperty(NodeRoles.NODE_ROLES_PROP, "coordinator:on"); + JettySolrRunner coordinatorJetty; + try { + coordinatorJetty = cluster.startJettySolrRunner(); + } finally { + System.clearProperty(NodeRoles.NODE_ROLES_PROP); + } + + try (HttpSolrClient coordinatorClient = + new HttpSolrClient.Builder(coordinatorJetty.getBaseUrl().toString()).build()) { + HttpResponse response = + coordinatorClient + .getHttpClient() + .execute( + new HttpGet( + coordinatorJetty.getBaseUrl() + + "/c1/select?q:*:*")); // make a call so the synthetic core would be + // created + assertEquals(200, response.getStatusLine().getStatusCode()); + // conf1 has no cache-control + assertNull(response.getFirstHeader("cache-control")); + + // now update conf1 + cluster.uploadConfigSet(configset("cache-control"), "conf1"); + + response = + coordinatorClient + .getHttpClient() + .execute( + new HttpGet( + coordinatorJetty.getBaseUrl() + + "/admin/cores?core=.sys.COORDINATOR-COLL-conf1_core&action=reload")); + assertEquals(200, response.getStatusLine().getStatusCode()); + + response = + coordinatorClient + .getHttpClient() + .execute(new HttpGet(coordinatorJetty.getBaseUrl() + "/c1/select?q:*:*")); + assertEquals(200, response.getStatusLine().getStatusCode()); + // now the response should show cache-control + assertTrue(response.getFirstHeader("cache-control").getValue().contains("max-age=30")); + } + } finally { + cluster.shutdown(); + } + } } diff --git a/solr/core/src/test/org/apache/solr/search/TestCpuAllowedLimit.java b/solr/core/src/test/org/apache/solr/search/TestCpuAllowedLimit.java index ea02e448da8..1fd6c87616d 100644 --- a/solr/core/src/test/org/apache/solr/search/TestCpuAllowedLimit.java +++ b/solr/core/src/test/org/apache/solr/search/TestCpuAllowedLimit.java @@ -20,12 +20,16 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.cloud.CloudUtil; import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.index.NoMergePolicyFactory; +import org.apache.solr.util.TestInjection; import org.apache.solr.util.ThreadCpuTimer; +import org.junit.AfterClass; import org.junit.Assume; import org.junit.BeforeClass; import org.junit.Test; @@ -61,7 +65,12 @@ private static Path createConfigSet() throws Exception { } @BeforeClass - public static void setup() throws Exception { + public static void setupClass() throws Exception { + // Using NoMergePolicy and 100 commits we should get 100 segments (across all shards). + // At this point of writing MAX_SEGMENTS_PER_SLICE in lucene is 5, so we should be + // ensured that any multithreaded testing will create 20 executable tasks for the + // executor that was provided to index-searcher. + systemSetPropertySolrTestsMergePolicyFactory(NoMergePolicyFactory.class.getName()); System.setProperty(ThreadCpuTimer.ENABLE_CPU_TIME, "true"); Path configset = createConfigSet(); configureCluster(1).addConfig("conf", configset).configure(); @@ -73,8 +82,14 @@ public static void setup() throws Exception { cluster.getOpenOverseer().getSolrCloudManager(), "active", COLLECTION, clusterShape(3, 6)); for (int j = 0; j < 100; j++) { solrClient.add(COLLECTION, sdoc("id", "id-" + j, "val_i", j % 5)); + solrClient.commit(COLLECTION); // need to commit every doc to create many segments. } - solrClient.commit(COLLECTION); + } + + @AfterClass + public static void tearDownClass() { + TestInjection.cpuTimerDelayInjectedNS = null; + systemClearPropertySolrTestsMergePolicyFactory(); } @Test @@ -131,7 +146,9 @@ public void testDistribLimit() throws Exception { Number qtime = (Number) rsp.getHeader().get("QTime"); assertTrue("QTime expected " + qtime + " >> " + sleepMs, qtime.longValue() > sleepMs); assertNull("should not have partial results", rsp.getHeader().get("partialResults")); - + TestInjection.measureCpu(); + // 25 ms per 5 segments ~175ms each shard + TestInjection.cpuTimerDelayInjectedNS = new AtomicInteger(25_000_000); // timeAllowed set, should return partial results log.info("--- timeAllowed, partial results ---"); rsp = @@ -146,6 +163,28 @@ public void testDistribLimit() throws Exception { String.valueOf(sleepMs), "stages", "prepare,process", + "multiThreaded", + "false", + "timeAllowed", + "500")); + // System.err.println("rsp=" + rsp.jsonStr()); + assertNotNull("should have partial results", rsp.getHeader().get("partialResults")); + + log.info("--- timeAllowed, partial results, multithreading ---"); + rsp = + solrClient.query( + COLLECTION, + params( + "q", + "id:*", + "sort", + "id asc", + ExpensiveSearchComponent.SLEEP_MS_PARAM, + String.valueOf(sleepMs), + "stages", + "prepare,process", + "multiThreaded", + "true", "timeAllowed", "500")); // System.err.println("rsp=" + rsp.jsonStr()); @@ -161,15 +200,12 @@ public void testDistribLimit() throws Exception { "id:*", "sort", "id desc", - ExpensiveSearchComponent.CPU_LOAD_COUNT_PARAM, - "1", "stages", "prepare,process", "cpuAllowed", - "1000")); + "10000")); // System.err.println("rsp=" + rsp.jsonStr()); assertNull("should have full results", rsp.getHeader().get("partialResults")); - // cpuAllowed set, should return partial results log.info("--- cpuAllowed 1, partial results ---"); rsp = @@ -180,19 +216,17 @@ public void testDistribLimit() throws Exception { "id:*", "sort", "id desc", - ExpensiveSearchComponent.CPU_LOAD_COUNT_PARAM, - "10", "stages", "prepare,process", "cpuAllowed", - "50", + "100", "multiThreaded", "false")); // System.err.println("rsp=" + rsp.jsonStr()); assertNotNull("should have partial results", rsp.getHeader().get("partialResults")); // cpuAllowed set, should return partial results - log.info("--- cpuAllowed 2, partial results ---"); + log.info("--- cpuAllowed 2, partial results, multi-threaded ---"); rsp = solrClient.query( COLLECTION, @@ -201,14 +235,12 @@ public void testDistribLimit() throws Exception { "id:*", "sort", "id desc", - ExpensiveSearchComponent.CPU_LOAD_COUNT_PARAM, - "10", "stages", "prepare,process", "cpuAllowed", - "50", + "100", "multiThreaded", - "false")); + "true")); // System.err.println("rsp=" + rsp.jsonStr()); assertNotNull("should have partial results", rsp.getHeader().get("partialResults")); } diff --git a/solr/core/src/test/org/apache/solr/search/TestFiltering.java b/solr/core/src/test/org/apache/solr/search/TestFiltering.java index e9194e170f9..5d9ad6bb9da 100644 --- a/solr/core/src/test/org/apache/solr/search/TestFiltering.java +++ b/solr/core/src/test/org/apache/solr/search/TestFiltering.java @@ -22,7 +22,6 @@ import java.util.Locale; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Query; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.FixedBitSet; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrInputDocument; @@ -93,14 +92,14 @@ public void testLiveDocsSharing() throws Exception { QueryResult res = new QueryResult(); searcher.search(res, cmd); set = res.getDocSet(); - assertEffectivelySame(set.getFixedBitSet(), live.getFixedBitSet()); + assertSame(set, live); cmd.setQuery(QParser.getParser(qstr + " OR id:0", null, req).getQuery()); cmd.setFilterList(QParser.getParser(qstr + " OR id:1", null, req).getQuery()); res = new QueryResult(); searcher.search(res, cmd); set = res.getDocSet(); - assertEffectivelySame(set.getFixedBitSet(), live.getFixedBitSet()); + assertSame(set, live); } } finally { @@ -108,19 +107,6 @@ public void testLiveDocsSharing() throws Exception { } } - /** If the a XOR b == 0, then both a & b are effectively the same bitset */ - private void assertEffectivelySame(FixedBitSet a, FixedBitSet b) { - FixedBitSet xor = a.clone(); - xor.xor(b); - assertEquals(new FixedBitSet(xor.length()), xor); - } - - private String bitsString(Bits bits) { - StringBuilder s = new StringBuilder(); - for (int i = 0; i < bits.length(); i++) s.append(bits.get(i) ? 1 : 0); - return s.toString(); - } - public void testCaching() throws Exception { clearIndex(); assertU(adoc("id", "4", "val_i", "1")); diff --git a/solr/core/src/test/org/apache/solr/update/processor/DistributedUpdateProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/DistributedUpdateProcessorTest.java index dc430437121..0014260001a 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/DistributedUpdateProcessorTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/DistributedUpdateProcessorTest.java @@ -18,8 +18,6 @@ package org.apache.solr.update.processor; import static org.hamcrest.CoreMatchers.is; -import static org.mockito.ArgumentMatchers.anyInt; -import static org.mockito.Mockito.doReturn; import java.io.IOException; import java.util.ArrayList; @@ -39,25 +37,18 @@ import org.apache.solr.update.AddUpdateCommand; import org.apache.solr.update.DeleteUpdateCommand; import org.apache.solr.update.SolrCmdDistributor; -import org.apache.solr.update.TimedVersionBucket; +import org.apache.solr.update.UpdateLocks; import org.apache.solr.update.UpdateLog; -import org.apache.solr.update.VersionInfo; import org.junit.AfterClass; import org.junit.BeforeClass; -import org.junit.Rule; import org.junit.Test; -import org.mockito.Mockito; -import org.mockito.junit.MockitoJUnit; -import org.mockito.junit.MockitoRule; public class DistributedUpdateProcessorTest extends SolrTestCaseJ4 { - @Rule public MockitoRule rule = MockitoJUnit.rule(); private static ExecutorService executor; @BeforeClass public static void beforeClass() throws Exception { - assumeWorkingMockito(); executor = ExecutorUtil.newMDCAwareCachedThreadPool(getClassName()); System.setProperty("enable.update.log", "true"); initCore( @@ -66,10 +57,8 @@ public static void beforeClass() throws Exception { } @AfterClass - public static void AfterClass() { - if (null != executor) { // may not have been initialized due to lack of mockito - executor.shutdown(); - } + public static void afterClass() { + ExecutorUtil.shutdownAndAwaitTermination(executor); System.clearProperty("enable.update.log"); } @@ -164,36 +153,12 @@ public void testStatusCodeOnDistribError_NotSolrException() { */ private int runCommands( int threads, - int versionBucketLockTimeoutMs, + int docLockTimeoutMs, SolrQueryRequest req, Function function) throws IOException { - try (DistributedUpdateProcessor processor = - new DistributedUpdateProcessor(req, null, null, null)) { - if (versionBucketLockTimeoutMs > 0) { - // use TimedVersionBucket with versionBucketLockTimeoutMs - VersionInfo vinfo = Mockito.spy(processor.vinfo); - processor.vinfo = vinfo; - - doReturn( - new TimedVersionBucket() { - /** simulate the case: it takes 5 seconds to add the doc */ - @Override - protected boolean tryLock(int lockTimeoutMs) { - boolean locked = super.tryLock(versionBucketLockTimeoutMs); - if (locked) { - try { - Thread.sleep(5000); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - } - return locked; - } - }) - .when(vinfo) - .bucket(anyInt()); - } + + try (DistributedUpdateProcessor processor = newDurp(req, docLockTimeoutMs)) { CountDownLatch latch = new CountDownLatch(1); Collection> futures = new ArrayList<>(); for (int t = 0; t < threads; ++t) { @@ -220,4 +185,45 @@ protected boolean tryLock(int lockTimeoutMs) { return succeeded; } } + + private static DistributedUpdateProcessor newDurp(SolrQueryRequest req, long lockTimeoutMs) { + if (lockTimeoutMs <= 0) { + // default + return new DistributedUpdateProcessor(req, null, null); + } + // customize UpdateLocks with the provided timeout. And simulate docs taking longer to index + final var sleepMs = lockTimeoutMs + 1000; + assert sleepMs > lockTimeoutMs; + var sleepUrp = + new UpdateRequestProcessor(null) { + @Override + public void processAdd(AddUpdateCommand cmd) throws IOException { + sleep(); + super.processAdd(cmd); + } + + @Override + public void processDelete(DeleteUpdateCommand cmd) throws IOException { + sleep(); + super.processDelete(cmd); + } + + private void sleep() { + try { + Thread.sleep(sleepMs); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + }; + + return new DistributedUpdateProcessor(req, null, null, sleepUrp) { + UpdateLocks updateLocks = new UpdateLocks(lockTimeoutMs); + + @Override + protected UpdateLocks getUpdateLocks() { + return updateLocks; + } + }; + } } diff --git a/solr/core/src/test/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactoryTest.java b/solr/core/src/test/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactoryTest.java index bea8bd507a4..19752e1ac7f 100644 --- a/solr/core/src/test/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactoryTest.java +++ b/solr/core/src/test/org/apache/solr/update/processor/SkipExistingDocumentsProcessorFactoryTest.java @@ -231,7 +231,7 @@ public void testSkippableInsertIsNotSkippedIfNotLeader() throws IOException { } @Test - public void testSkippableInsertIsNotSkippedIfSkipInsertsFalse() throws IOException { + public void testSkippableInsertIsNotSkippedIfSkipInsertsIsFalse() throws IOException { UpdateRequestProcessor next = Mockito.mock(DistributedUpdateProcessor.class); SkipExistingDocumentsUpdateProcessor processor = Mockito.spy(new SkipExistingDocumentsUpdateProcessor(defaultRequest, next, false, false)); @@ -245,7 +245,7 @@ public void testSkippableInsertIsNotSkippedIfSkipInsertsFalse() throws IOExcepti } @Test - public void testSkippableInsertIsSkippedIfSkipInsertsTrue() throws IOException { + public void testSkippableInsertIsSkippedIfSkipInsertsIsTrue() throws IOException { UpdateRequestProcessor next = Mockito.mock(DistributedUpdateProcessor.class); SkipExistingDocumentsUpdateProcessor processor = Mockito.spy(new SkipExistingDocumentsUpdateProcessor(defaultRequest, next, true, false)); @@ -259,7 +259,7 @@ public void testSkippableInsertIsSkippedIfSkipInsertsTrue() throws IOException { } @Test - public void testNonSkippableInsertIsNotSkippedIfSkipInsertsTrue() throws IOException { + public void testNonSkippableInsertIsNotSkippedIfSkipInsertsIsTrue() throws IOException { UpdateRequestProcessor next = Mockito.mock(DistributedUpdateProcessor.class); SkipExistingDocumentsUpdateProcessor processor = Mockito.spy(new SkipExistingDocumentsUpdateProcessor(defaultRequest, next, true, false)); @@ -287,7 +287,7 @@ public void testSkippableUpdateIsNotSkippedIfNotLeader() throws IOException { } @Test - public void testSkippableUpdateIsNotSkippedIfSkipUpdatesFalse() throws IOException { + public void testSkippableUpdateIsNotSkippedIfSkipUpdatesIsFalse() throws IOException { UpdateRequestProcessor next = Mockito.mock(DistributedUpdateProcessor.class); SkipExistingDocumentsUpdateProcessor processor = Mockito.spy(new SkipExistingDocumentsUpdateProcessor(defaultRequest, next, false, false)); @@ -301,7 +301,7 @@ public void testSkippableUpdateIsNotSkippedIfSkipUpdatesFalse() throws IOExcepti } @Test - public void testSkippableUpdateIsSkippedIfSkipUpdatesTrue() throws IOException { + public void testSkippableUpdateIsSkippedIfSkipUpdatesIsTrue() throws IOException { UpdateRequestProcessor next = Mockito.mock(DistributedUpdateProcessor.class); SkipExistingDocumentsUpdateProcessor processor = Mockito.spy(new SkipExistingDocumentsUpdateProcessor(defaultRequest, next, false, true)); @@ -315,7 +315,23 @@ public void testSkippableUpdateIsSkippedIfSkipUpdatesTrue() throws IOException { } @Test - public void testNonSkippableUpdateIsNotSkippedIfSkipUpdatesTrue() throws IOException { + public void testSkippableChildDocUpdateIsSkippedIfSkipUpdatesIsTrue() throws IOException { + UpdateRequestProcessor next = Mockito.mock(DistributedUpdateProcessor.class); + SkipExistingDocumentsUpdateProcessor processor = + Mockito.spy(new SkipExistingDocumentsUpdateProcessor(defaultRequest, next, false, true)); + + AddUpdateCommand cmd = Mockito.spy(createAtomicUpdateCmd(defaultRequest)); + doReturn(true).when(processor).isLeader(cmd); + doReturn(false).when(processor).doesChildDocumentExist(cmd); + doReturn("123/child1").when(cmd).getSelfOrNestedDocIdStr(); + doReturn("123").when(cmd).getIndexedIdStr(); + + processor.processAdd(cmd); + verify(next, never()).processAdd(cmd); + } + + @Test + public void testNonSkippableUpdateIsNotSkippedIfSkipUpdatesIsTrue() throws IOException { UpdateRequestProcessor next = Mockito.mock(DistributedUpdateProcessor.class); SkipExistingDocumentsUpdateProcessor processor = Mockito.spy(new SkipExistingDocumentsUpdateProcessor(defaultRequest, next, false, true)); @@ -328,6 +344,22 @@ public void testNonSkippableUpdateIsNotSkippedIfSkipUpdatesTrue() throws IOExcep verify(next).processAdd(cmd); } + @Test + public void testNonSkippableChildDocUpdateIsNotSkippedIfSkipUpdatesIsTrue() throws IOException { + UpdateRequestProcessor next = Mockito.mock(DistributedUpdateProcessor.class); + SkipExistingDocumentsUpdateProcessor processor = + Mockito.spy(new SkipExistingDocumentsUpdateProcessor(defaultRequest, next, false, true)); + + AddUpdateCommand cmd = Mockito.spy(createAtomicUpdateCmd(defaultRequest)); + doReturn(true).when(processor).isLeader(cmd); + doReturn(true).when(processor).doesChildDocumentExist(cmd); + doReturn("123/child1").when(cmd).getSelfOrNestedDocIdStr(); + doReturn("123").when(cmd).getIndexedIdStr(); + + processor.processAdd(cmd); + verify(next).processAdd(cmd); + } + private AddUpdateCommand createInsertUpdateCmd(SolrQueryRequest req) { AddUpdateCommand cmd = new AddUpdateCommand(req); cmd.setIndexedId(docId); diff --git a/solr/core/src/test/org/apache/solr/util/OrderedExecutorTest.java b/solr/core/src/test/org/apache/solr/util/OrderedExecutorTest.java index 4b49baa6e01..199062ba1a4 100644 --- a/solr/core/src/test/org/apache/solr/util/OrderedExecutorTest.java +++ b/solr/core/src/test/org/apache/solr/util/OrderedExecutorTest.java @@ -30,6 +30,7 @@ import java.util.concurrent.TimeoutException; import org.apache.solr.SolrTestCase; import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.SolrNamedThreadFactory; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -37,10 +38,19 @@ public class OrderedExecutorTest extends SolrTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private static OrderedExecutor newOrderedExecutor(int numThreads) { + // initialize exactly as done in CoreContainer so we test realistically + return new OrderedExecutor<>( + numThreads, + ExecutorUtil.newMDCAwareCachedThreadPool( + numThreads, // thread count + numThreads, // queue size + new SolrNamedThreadFactory("testOrderedExecutor"))); + } + @Test public void testExecutionInOrder() { - OrderedExecutor orderedExecutor = - new OrderedExecutor(10, ExecutorUtil.newMDCAwareCachedThreadPool("executeInOrderTest")); + var orderedExecutor = newOrderedExecutor(10); IntBox intBox = new IntBox(); for (int i = 0; i < 100; i++) { orderedExecutor.execute(1, () -> intBox.value++); @@ -53,9 +63,7 @@ public void testExecutionInOrder() { public void testLockWhenQueueIsFull() { final ExecutorService controlExecutor = ExecutorUtil.newMDCAwareCachedThreadPool("testLockWhenQueueIsFull_control"); - final OrderedExecutor orderedExecutor = - new OrderedExecutor( - 10, ExecutorUtil.newMDCAwareCachedThreadPool("testLockWhenQueueIsFull_test")); + final var orderedExecutor = newOrderedExecutor(10); try { // AAA and BBB events will both depend on the use of the same lockId @@ -111,9 +119,7 @@ public void testRunInParallel() { final ExecutorService controlExecutor = ExecutorUtil.newMDCAwareCachedThreadPool("testRunInParallel_control"); - final OrderedExecutor orderedExecutor = - new OrderedExecutor( - parallelism, ExecutorUtil.newMDCAwareCachedThreadPool("testRunInParallel_test")); + final var orderedExecutor = newOrderedExecutor(parallelism); try { // distinct lockIds should be able to be used in parallel, up to the size of the executor, @@ -216,8 +222,7 @@ public void testStress() { base.put(i, i); run.put(i, i); } - OrderedExecutor orderedExecutor = - new OrderedExecutor(10, ExecutorUtil.newMDCAwareCachedThreadPool("testStress")); + var orderedExecutor = newOrderedExecutor(10); for (int i = 0; i < 1000; i++) { int key = random().nextInt(N); base.put(key, base.get(key) + 1); @@ -233,8 +238,7 @@ private static class IntBox { @Test public void testMaxSize() throws InterruptedException { - OrderedExecutor orderedExecutor = - new OrderedExecutor(1, ExecutorUtil.newMDCAwareCachedThreadPool("single")); + var orderedExecutor = newOrderedExecutor(1); CountDownLatch isRunning = new CountDownLatch(1); CountDownLatch blockingLatch = new CountDownLatch(1); diff --git a/solr/licenses/argparse4j-0.9.0.jar.sha1 b/solr/licenses/argparse4j-0.9.0.jar.sha1 deleted file mode 100644 index e9537fd93ed..00000000000 --- a/solr/licenses/argparse4j-0.9.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -0485aaa44f9bfdc787da898828cdc71d6577a86d diff --git a/solr/licenses/argparse4j-LICENSE-MIT.txt b/solr/licenses/argparse4j-LICENSE-MIT.txt deleted file mode 100644 index 235421342ef..00000000000 --- a/solr/licenses/argparse4j-LICENSE-MIT.txt +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright (C) 2011-2017 Tatsuhiro Tsujikawa - * - * Permission is hereby granted, free of charge, to any person - * obtaining a copy of this software and associated documentation - * files (the "Software"), to deal in the Software without - * restriction, including without limitation the rights to use, copy, - * modify, merge, publish, distribute, sublicense, and/or sell copies - * of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be - * included in all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ \ No newline at end of file diff --git a/solr/licenses/argparse4j-NOTICE.txt b/solr/licenses/argparse4j-NOTICE.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/solr/licenses/netty-buffer-4.1.111.Final.jar.sha1 b/solr/licenses/netty-buffer-4.1.111.Final.jar.sha1 deleted file mode 100644 index 7502c1d0d11..00000000000 --- a/solr/licenses/netty-buffer-4.1.111.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b54863f578939e135d3b3aea610284ae57c188cf diff --git a/solr/licenses/netty-buffer-4.1.112.Final.jar.sha1 b/solr/licenses/netty-buffer-4.1.112.Final.jar.sha1 new file mode 100644 index 00000000000..0ac205b0823 --- /dev/null +++ b/solr/licenses/netty-buffer-4.1.112.Final.jar.sha1 @@ -0,0 +1 @@ +bdc12df04bb6858890b8aa108060b5b365a26102 diff --git a/solr/licenses/netty-codec-4.1.111.Final.jar.sha1 b/solr/licenses/netty-codec-4.1.111.Final.jar.sha1 deleted file mode 100644 index 96eec7829cc..00000000000 --- a/solr/licenses/netty-codec-4.1.111.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a6762ec00a6d268f9980741f5b755838bcd658bf diff --git a/solr/licenses/netty-codec-4.1.112.Final.jar.sha1 b/solr/licenses/netty-codec-4.1.112.Final.jar.sha1 new file mode 100644 index 00000000000..55360968342 --- /dev/null +++ b/solr/licenses/netty-codec-4.1.112.Final.jar.sha1 @@ -0,0 +1 @@ +c87f2ec3d9a97bd2b793d16817abb2bab93a7fc3 diff --git a/solr/licenses/netty-codec-http-4.1.111.Final.jar.sha1 b/solr/licenses/netty-codec-http-4.1.111.Final.jar.sha1 deleted file mode 100644 index 27b935424aa..00000000000 --- a/solr/licenses/netty-codec-http-4.1.111.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c6ecbc452321e632bf3cea0f9758839b650455c7 diff --git a/solr/licenses/netty-codec-http-4.1.112.Final.jar.sha1 b/solr/licenses/netty-codec-http-4.1.112.Final.jar.sha1 new file mode 100644 index 00000000000..c5a4048f7eb --- /dev/null +++ b/solr/licenses/netty-codec-http-4.1.112.Final.jar.sha1 @@ -0,0 +1 @@ +81af1040bfa977f98dd0e1bd9639513ea862ca04 diff --git a/solr/licenses/netty-codec-http2-4.1.111.Final.jar.sha1 b/solr/licenses/netty-codec-http2-4.1.111.Final.jar.sha1 deleted file mode 100644 index b3c6f6aef26..00000000000 --- a/solr/licenses/netty-codec-http2-4.1.111.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f0cca5df75bfb4f858d0435f601d8b1cae1de054 diff --git a/solr/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 b/solr/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 new file mode 100644 index 00000000000..d8ff515db81 --- /dev/null +++ b/solr/licenses/netty-codec-http2-4.1.112.Final.jar.sha1 @@ -0,0 +1 @@ +7fa28b510f0f16f4d5d7188b86bef59e048f62f9 diff --git a/solr/licenses/netty-codec-socks-4.1.111.Final.jar.sha1 b/solr/licenses/netty-codec-socks-4.1.111.Final.jar.sha1 deleted file mode 100644 index e932d399674..00000000000 --- a/solr/licenses/netty-codec-socks-4.1.111.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ea52ef6617a9b69b0baaebb7f0b80373527f9607 diff --git a/solr/licenses/netty-codec-socks-4.1.112.Final.jar.sha1 b/solr/licenses/netty-codec-socks-4.1.112.Final.jar.sha1 new file mode 100644 index 00000000000..0b3d39a0cee --- /dev/null +++ b/solr/licenses/netty-codec-socks-4.1.112.Final.jar.sha1 @@ -0,0 +1 @@ +9aed7e78c467d06a47a45b5b27466380a6427e2f diff --git a/solr/licenses/netty-common-4.1.111.Final.jar.sha1 b/solr/licenses/netty-common-4.1.111.Final.jar.sha1 deleted file mode 100644 index c91f89a2dbf..00000000000 --- a/solr/licenses/netty-common-4.1.111.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -58210befcb31adbcadd5724966a061444db91863 diff --git a/solr/licenses/netty-common-4.1.112.Final.jar.sha1 b/solr/licenses/netty-common-4.1.112.Final.jar.sha1 new file mode 100644 index 00000000000..1659204d787 --- /dev/null +++ b/solr/licenses/netty-common-4.1.112.Final.jar.sha1 @@ -0,0 +1 @@ +b2798069092a981a832b7510d0462ee9efb7a80e diff --git a/solr/licenses/netty-handler-4.1.111.Final.jar.sha1 b/solr/licenses/netty-handler-4.1.111.Final.jar.sha1 deleted file mode 100644 index 6f52f18a9bf..00000000000 --- a/solr/licenses/netty-handler-4.1.111.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -2bc6a58ad2e9e279634b6e55022e8dcd3c175cc4 diff --git a/solr/licenses/netty-handler-4.1.112.Final.jar.sha1 b/solr/licenses/netty-handler-4.1.112.Final.jar.sha1 new file mode 100644 index 00000000000..43be3612a0d --- /dev/null +++ b/solr/licenses/netty-handler-4.1.112.Final.jar.sha1 @@ -0,0 +1 @@ +3d5e2d5bcc6baeeb8c13a230980c6132a778e036 diff --git a/solr/licenses/netty-handler-proxy-4.1.111.Final.jar.sha1 b/solr/licenses/netty-handler-proxy-4.1.111.Final.jar.sha1 deleted file mode 100644 index ecf253ebef6..00000000000 --- a/solr/licenses/netty-handler-proxy-4.1.111.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -1e459c8630bb7c942b79a97e62dd728798de6a8c diff --git a/solr/licenses/netty-handler-proxy-4.1.112.Final.jar.sha1 b/solr/licenses/netty-handler-proxy-4.1.112.Final.jar.sha1 new file mode 100644 index 00000000000..d85e58a7af2 --- /dev/null +++ b/solr/licenses/netty-handler-proxy-4.1.112.Final.jar.sha1 @@ -0,0 +1 @@ +b23c87a85451b3b0e7c3e8e89698cea6831a8418 diff --git a/solr/licenses/netty-resolver-4.1.111.Final.jar.sha1 b/solr/licenses/netty-resolver-4.1.111.Final.jar.sha1 deleted file mode 100644 index a870fee5fa0..00000000000 --- a/solr/licenses/netty-resolver-4.1.111.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3493179999f211dc49714319f81da2be86523a3b diff --git a/solr/licenses/netty-resolver-4.1.112.Final.jar.sha1 b/solr/licenses/netty-resolver-4.1.112.Final.jar.sha1 new file mode 100644 index 00000000000..95eb0338e52 --- /dev/null +++ b/solr/licenses/netty-resolver-4.1.112.Final.jar.sha1 @@ -0,0 +1 @@ +58a631d9d44c4ed7cc0dcc9cffa6641da9374d72 diff --git a/solr/licenses/netty-transport-4.1.111.Final.jar.sha1 b/solr/licenses/netty-transport-4.1.111.Final.jar.sha1 deleted file mode 100644 index 61738b73629..00000000000 --- a/solr/licenses/netty-transport-4.1.111.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -24e97cf14ea9d80afe4c5ab69066b587fccc154a diff --git a/solr/licenses/netty-transport-4.1.112.Final.jar.sha1 b/solr/licenses/netty-transport-4.1.112.Final.jar.sha1 new file mode 100644 index 00000000000..5edb433d5f4 --- /dev/null +++ b/solr/licenses/netty-transport-4.1.112.Final.jar.sha1 @@ -0,0 +1 @@ +77cd136dd3843f5e7cbcf68c824975d745c49ddb diff --git a/solr/licenses/netty-transport-classes-epoll-4.1.111.Final.jar.sha1 b/solr/licenses/netty-transport-classes-epoll-4.1.111.Final.jar.sha1 deleted file mode 100644 index 8cca8c5497d..00000000000 --- a/solr/licenses/netty-transport-classes-epoll-4.1.111.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -8b97d32eb1489043e478deea99bd93ce487b82f6 diff --git a/solr/licenses/netty-transport-classes-epoll-4.1.112.Final.jar.sha1 b/solr/licenses/netty-transport-classes-epoll-4.1.112.Final.jar.sha1 new file mode 100644 index 00000000000..1364819aa58 --- /dev/null +++ b/solr/licenses/netty-transport-classes-epoll-4.1.112.Final.jar.sha1 @@ -0,0 +1 @@ +67e590356eb53c20aaabd67f61ae66f628e62e3d diff --git a/solr/licenses/netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar.sha1 b/solr/licenses/netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar.sha1 deleted file mode 100644 index 2d45dd3c89a..00000000000 --- a/solr/licenses/netty-transport-native-epoll-4.1.111.Final-linux-x86_64.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e102a0568f4f2cd7d4049c4154f94528cadf646b diff --git a/solr/licenses/netty-transport-native-epoll-4.1.112.Final-linux-x86_64.jar.sha1 b/solr/licenses/netty-transport-native-epoll-4.1.112.Final-linux-x86_64.jar.sha1 new file mode 100644 index 00000000000..9995f7b1048 --- /dev/null +++ b/solr/licenses/netty-transport-native-epoll-4.1.112.Final-linux-x86_64.jar.sha1 @@ -0,0 +1 @@ +caed6f2ae7aebcef50098a64f55918cb8550d2d0 diff --git a/solr/licenses/netty-transport-native-unix-common-4.1.111.Final.jar.sha1 b/solr/licenses/netty-transport-native-unix-common-4.1.111.Final.jar.sha1 deleted file mode 100644 index 689846d2410..00000000000 --- a/solr/licenses/netty-transport-native-unix-common-4.1.111.Final.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -acafc128cddafa021bc0b48b0788eb0e118add5e diff --git a/solr/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 b/solr/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 new file mode 100644 index 00000000000..265f56097e8 --- /dev/null +++ b/solr/licenses/netty-transport-native-unix-common-4.1.112.Final.jar.sha1 @@ -0,0 +1 @@ +b50ff619cdcdc48e748cba3405c9988529f28f60 diff --git a/solr/modules/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java b/solr/modules/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java index 87a09c56212..9d72b88ee38 100644 --- a/solr/modules/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java +++ b/solr/modules/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java @@ -316,7 +316,7 @@ protected void checkSupportsDocValues() { // we support DocValues } @Override - protected boolean doesTypeSupportDocValues() { + protected boolean enableDocValuesByDefault() { return true; } diff --git a/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml index 90c52d71cbe..44ef6530752 100644 --- a/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml +++ b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-icucollate.xml @@ -23,5 +23,5 @@ ${useCompoundFile:false} - + diff --git a/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-opennlp-extract.xml b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-opennlp-extract.xml index 7fd793e94d3..1bf101f0db1 100644 --- a/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-opennlp-extract.xml +++ b/solr/modules/analysis-extras/src/test-files/analysis-extras/solr/collection1/conf/solrconfig-opennlp-extract.xml @@ -22,7 +22,7 @@ - + diff --git a/solr/modules/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml b/solr/modules/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml index 4d268b0efd8..7376c7a07d1 100644 --- a/solr/modules/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml +++ b/solr/modules/clustering/src/test-files/clustering/solr/collection1/conf/solrconfig.xml @@ -25,7 +25,7 @@ If replication is in use, this should match the replication configuration. --> ${solr.data.dir:} - + diff --git a/solr/modules/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml b/solr/modules/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml index b35a27fac39..2c52f4591e8 100644 --- a/solr/modules/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml +++ b/solr/modules/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml @@ -27,7 +27,7 @@ It defaults to "index" if not present, and should probably not be changed if replication is in use. --> ${solr.data.dir:} - + diff --git a/solr/modules/gcs-repository/src/test-files/conf/schema.xml b/solr/modules/gcs-repository/src/test-files/conf/schema.xml index b539459ad5e..a3a7cc465c2 100644 --- a/solr/modules/gcs-repository/src/test-files/conf/schema.xml +++ b/solr/modules/gcs-repository/src/test-files/conf/schema.xml @@ -17,8 +17,8 @@ --> - - + + diff --git a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/util/HdfsRecoverLeaseFileSystemUtils.java b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/util/HdfsRecoverLeaseFileSystemUtils.java index b0bd7960bff..2d5cd6a06ac 100644 --- a/solr/modules/hdfs/src/java/org/apache/solr/hdfs/util/HdfsRecoverLeaseFileSystemUtils.java +++ b/solr/modules/hdfs/src/java/org/apache/solr/hdfs/util/HdfsRecoverLeaseFileSystemUtils.java @@ -54,7 +54,7 @@ public static void recoverFileLease( /* * Run the dfs recover lease. recoverLease is asynchronous. It returns: - * -false when it starts the lease recovery (i.e. lease recovery not *yet* done) + * - false when it starts the lease recovery (i.e. lease recovery not *yet* done) * - true when the lease recovery has succeeded or the file is closed. * But, we have to be careful. Each time we call recoverLease, it starts the recover lease * process over from the beginning. We could put ourselves in a situation where we are diff --git a/solr/modules/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml b/solr/modules/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml index f03387a170b..a951ea4d80d 100644 --- a/solr/modules/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml +++ b/solr/modules/langid/src/test-files/langid/solr/collection1/conf/solrconfig-languageidentifier.xml @@ -31,7 +31,7 @@ - + ${tests.luceneMatchVersion:LATEST} diff --git a/solr/modules/ltr/src/test-files/solr/collection1/conf/schema.xml b/solr/modules/ltr/src/test-files/solr/collection1/conf/schema.xml index 07779ce8a93..56c756f75b5 100644 --- a/solr/modules/ltr/src/test-files/solr/collection1/conf/schema.xml +++ b/solr/modules/ltr/src/test-files/solr/collection1/conf/schema.xml @@ -22,8 +22,8 @@ - - + + @@ -73,11 +73,11 @@ - - - - - + + + + + diff --git a/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml b/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml index 172a0bca6a8..b863d61728c 100644 --- a/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml +++ b/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr.xml @@ -14,7 +14,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} + class="${solr.directoryFactory:solr.MockDirectoryFactory}" /> diff --git a/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr_Th10_10.xml b/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr_Th10_10.xml index 7aca7e10db5..37ae68a2580 100644 --- a/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr_Th10_10.xml +++ b/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-ltr_Th10_10.xml @@ -14,7 +14,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} + class="${solr.directoryFactory:solr.MockDirectoryFactory}" /> diff --git a/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-multiseg.xml b/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-multiseg.xml index c5db8670234..911db9a9f55 100644 --- a/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-multiseg.xml +++ b/solr/modules/ltr/src/test-files/solr/collection1/conf/solrconfig-multiseg.xml @@ -14,7 +14,7 @@ ${tests.luceneMatchVersion:LATEST} ${solr.data.dir:} + class="${solr.directoryFactory:solr.MockDirectoryFactory}" /> diff --git a/solr/modules/s3-repository/src/test-files/conf/schema.xml b/solr/modules/s3-repository/src/test-files/conf/schema.xml index b539459ad5e..a3a7cc465c2 100644 --- a/solr/modules/s3-repository/src/test-files/conf/schema.xml +++ b/solr/modules/s3-repository/src/test-files/conf/schema.xml @@ -17,8 +17,8 @@ --> - - + + diff --git a/solr/modules/scripting/src/test-files/scripting/solr/collection1/conf/solrconfig-script-updateprocessor.xml b/solr/modules/scripting/src/test-files/scripting/solr/collection1/conf/solrconfig-script-updateprocessor.xml index afa5a7c56d7..82ce0331eda 100644 --- a/solr/modules/scripting/src/test-files/scripting/solr/collection1/conf/solrconfig-script-updateprocessor.xml +++ b/solr/modules/scripting/src/test-files/scripting/solr/collection1/conf/solrconfig-script-updateprocessor.xml @@ -25,7 +25,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/modules/scripting/src/test-files/scripting/solr/collection1/conf/stateless-solrconfig-script-updateprocessor.xml b/solr/modules/scripting/src/test-files/scripting/solr/collection1/conf/stateless-solrconfig-script-updateprocessor.xml index 58fbb862860..d8c6fd2bdbb 100644 --- a/solr/modules/scripting/src/test-files/scripting/solr/collection1/conf/stateless-solrconfig-script-updateprocessor.xml +++ b/solr/modules/scripting/src/test-files/scripting/solr/collection1/conf/stateless-solrconfig-script-updateprocessor.xml @@ -25,7 +25,7 @@ ${tests.luceneMatchVersion:LATEST} - + diff --git a/solr/modules/sql/src/test-files/solr/configsets/sql/conf/schema.xml b/solr/modules/sql/src/test-files/solr/configsets/sql/conf/schema.xml index 7c7131ba66b..2311e766632 100644 --- a/solr/modules/sql/src/test-files/solr/configsets/sql/conf/schema.xml +++ b/solr/modules/sql/src/test-files/solr/configsets/sql/conf/schema.xml @@ -43,7 +43,7 @@ 1.4: default auto-phrase (QueryParser feature) to off 1.5: omitNorms defaults to true for primitive field types (int, float, boolean, string...) 1.6: useDocValuesAsStored defaults to true. - 1.7: docValues defaults to true. + 1.7: docValues defaults to true, uninvertible defaults to false. --> diff --git a/solr/packaging/build.gradle b/solr/packaging/build.gradle index bf816677a02..25d17ea1125 100644 --- a/solr/packaging/build.gradle +++ b/solr/packaging/build.gradle @@ -294,7 +294,7 @@ class BatsTask extends Exec { // Note: tests to run must be listed after all other arguments // Additional debugging output: -x, --verbose-run - setArgs(['-T', '--print-output-on-failure'] + (testFiles.empty ? testDir : testFiles)) + setArgs(['-T', '--print-output-on-failure', '--report-formatter', 'junit', '--output', "$project.buildDir/test-output"] + (testFiles.empty ? testDir : testFiles)) super.exec() diff --git a/solr/packaging/test/test_adminconsole_urls.bats b/solr/packaging/test/test_adminconsole_urls.bats index 68fb79df3ed..2ab86af39cd 100644 --- a/solr/packaging/test/test_adminconsole_urls.bats +++ b/solr/packaging/test/test_adminconsole_urls.bats @@ -25,7 +25,7 @@ teardown() { # save a snapshot of SOLR_HOME for failed tests save_home_on_failure - solr stop -all >/dev/null 2>&1 + solr stop --all >/dev/null 2>&1 } @test "assert able to launch solr admin console" { diff --git a/solr/packaging/test/test_basic_auth.bats b/solr/packaging/test/test_basic_auth.bats index 8cd997a08ad..9cdebb7af18 100644 --- a/solr/packaging/test/test_basic_auth.bats +++ b/solr/packaging/test/test_basic_auth.bats @@ -49,15 +49,15 @@ teardown() { assert_output --partial "Created collection 'COLL_NAME'" # Test config - run solr config -u name:password -c COLL_NAME --action set-property -property updateHandler.autoCommit.maxDocs --value 100 --solr-url http://localhost:${SOLR_PORT}/solr + run solr config -u name:password -c COLL_NAME --action set-property --property updateHandler.autoCommit.maxDocs --value 100 --solr-url http://localhost:${SOLR_PORT}/solr assert_output --partial "Successfully set-property updateHandler.autoCommit.maxDocs to 100" # Test api - run solr api -u name:password --solr-url "http://localhost:${SOLR_PORT}/solr/COLL_NAME/select?q=*:*" -verbose + run solr api -u name:password --solr-url "http://localhost:${SOLR_PORT}/solr/COLL_NAME/select?q=*:*" --verbose assert_output --partial '"numFound":0' # Test delete - run solr delete --credentials name:password -c COLL_NAME -z localhost:${ZK_PORT} -verbose + run solr delete --credentials name:password -c COLL_NAME -z localhost:${ZK_PORT} --verbose assert_output --partial "Deleted collection 'COLL_NAME'" refute collection_exists "COLL_NAME" @@ -69,7 +69,7 @@ run solr create -c COLL_NAME assert_output --partial "Created collection 'COLL_NAME'" # Test post - run solr post -u name:password -type application/xml --solr-update-url http://localhost:${SOLR_PORT}/solr/monitors/update ${SOLR_TIP}/example/exampledocs/monitor.xml + run solr post -u name:password -t application/xml --solr-update-url http://localhost:${SOLR_PORT}/solr/monitors/update ${SOLR_TIP}/example/exampledocs/monitor.xml assert_output --partial '1 files indexed.' # Test postlogs diff --git a/solr/packaging/test/test_create_collection.bats b/solr/packaging/test/test_create_collection.bats index 05ec63d7f74..3ffa6bd0ff3 100644 --- a/solr/packaging/test/test_create_collection.bats +++ b/solr/packaging/test/test_create_collection.bats @@ -63,7 +63,7 @@ teardown() { } @test "reject d option with invalid config dir" { - run ! solr create -c COLL_NAME -d /asdf -solrUrl http://localhost:${SOLR_PORT} + run ! solr create -c COLL_NAME -d /asdf --solr-url http://localhost:${SOLR_PORT} assert_output --partial "Specified configuration directory /asdf not found!" } diff --git a/solr/packaging/test/test_extraction.bats b/solr/packaging/test/test_extraction.bats index 7557dc93b94..346f1df431a 100644 --- a/solr/packaging/test/test_extraction.bats +++ b/solr/packaging/test/test_extraction.bats @@ -73,7 +73,7 @@ teardown() { }' "http://localhost:${SOLR_PORT}/solr/content_extraction/config" # We filter to pdf to invoke the Extract handler. - run solr post -filetypes pdf -url http://localhost:${SOLR_PORT}/solr/content_extraction/update ${SOLR_TIP}/example/exampledocs + run solr post --filetypes pdf -url http://localhost:${SOLR_PORT}/solr/content_extraction/update ${SOLR_TIP}/example/exampledocs assert_output --partial '1 files indexed.' refute_output --partial 'ERROR' @@ -99,8 +99,8 @@ teardown() { } }' "http://localhost:${SOLR_PORT}/solr/website_extraction/config" - # Change to -recursive 1 to crawl multiple pages, but may be too slow. - run solr post -mode web -url http://localhost:${SOLR_PORT}/solr/website_extraction/update -recursive 0 -delay 1 https://solr.apache.org/ + # Change to --recursive 1 to crawl multiple pages, but may be too slow. + run solr post --mode web --solr-update-url http://localhost:${SOLR_PORT}/solr/website_extraction/update --recursive 0 --delay 1 https://solr.apache.org/ assert_output --partial 'POSTed web resource https://solr.apache.org (depth: 0)' refute_output --partial 'ERROR' diff --git a/solr/packaging/test/test_packages.bats b/solr/packaging/test/test_packages.bats index 185b3381b83..7de497a8d8c 100644 --- a/solr/packaging/test/test_packages.bats +++ b/solr/packaging/test/test_packages.bats @@ -44,7 +44,7 @@ teardown() { solr create -c foo-1.2 # Deploy package - the package doesn't need to exist before the collection validation kicks in - run solr package deploy PACKAGE_NAME -collections foo-1.2 + run solr package deploy PACKAGE_NAME --collections foo-1.2 # assert_output --partial "Deployment successful" refute_output --partial "Invalid collection" @@ -52,7 +52,7 @@ teardown() { assert_output --partial "Package instance doesn't exist: PACKAGE_NAME:null" # Undeploy package - run solr package undeploy PACKAGE_NAME -collections foo-1.2 + run solr package undeploy PACKAGE_NAME --collections foo-1.2 refute_output --partial "Invalid collection" assert_output --partial "Package PACKAGE_NAME not deployed on collection foo-1.2" } @@ -72,7 +72,7 @@ teardown() { # run solr package install solr-splainer # assert_output --partial "solr-splainer installed." -# run solr package deploy solr-splainer -y -cluster +# run solr package deploy solr-splainer -y --cluster # assert_output --partial "Deployment successful" # run -0 curl --fail http://localhost:${SOLR_PORT}/v2/splainer/index.html diff --git a/solr/packaging/test/test_post.bats b/solr/packaging/test/test_post.bats index bb9b2e6cb1a..38652a8c5da 100644 --- a/solr/packaging/test/test_post.bats +++ b/solr/packaging/test/test_post.bats @@ -162,14 +162,14 @@ teardown() { run solr create -c test_args -d _default assert_output --partial "Created collection 'test_args'" - run solr post --solr-update-url http://localhost:${SOLR_PORT}/solr/test_args/update --mode args -type application/xml --out "*:*" + run solr post --solr-update-url http://localhost:${SOLR_PORT}/solr/test_args/update --mode args --type application/xml --out "*:*" assert_output --partial '0' # confirm default type run solr post --solr-update-url http://localhost:${SOLR_PORT}/solr/test_args/update --mode args --out "{'delete': {'query': '*:*'}}" assert_output --partial '"status":0' - # confirm we don't get back output without -out + # confirm we don't get back output without --out run solr post --solr-update-url http://localhost:${SOLR_PORT}/solr/test_args/update --mode args "{'delete': {'query': '*:*'}}" refute_output --partial '"status":0' diff --git a/solr/packaging/test/test_prometheus.bats b/solr/packaging/test/test_prometheus.bats index 72dbb3b4690..9af9a515922 100644 --- a/solr/packaging/test/test_prometheus.bats +++ b/solr/packaging/test/test_prometheus.bats @@ -25,7 +25,7 @@ teardown() { # save a snapshot of SOLR_HOME for failed tests save_home_on_failure - solr stop -all >/dev/null 2>&1 + solr stop --all >/dev/null 2>&1 shutdown_exporter } diff --git a/solr/packaging/test/test_ssl.bats b/solr/packaging/test/test_ssl.bats index 08b3fc1d786..d114c15e25d 100644 --- a/solr/packaging/test/test_ssl.bats +++ b/solr/packaging/test/test_ssl.bats @@ -604,7 +604,7 @@ teardown() { run solr api --solr-url "https://localhost:${SOLR2_PORT}/solr/test-single-shard/select?q=query4" assert_output --partial '"numFound":0' - run solr post -url https://localhost:${SOLR_PORT}/solr/test/update ${SOLR_TIP}/example/exampledocs/books.csv + run solr post --solr-update-url https://localhost:${SOLR_PORT}/solr/test/update ${SOLR_TIP}/example/exampledocs/books.csv run solr api --solr-url "https://localhost:${SOLR_PORT}/solr/test/select?q=*:*" assert_output --partial '"numFound":10' diff --git a/solr/packaging/test/test_zk.bats b/solr/packaging/test/test_zk.bats index 1c2c2a2bad7..817a33e1054 100644 --- a/solr/packaging/test/test_zk.bats +++ b/solr/packaging/test/test_zk.bats @@ -122,7 +122,7 @@ teardown() { @test "bin/solr zk cp gets 'solrhome' from '--solr-home' command line option" { touch afile.txt - run solr zk cp afile.txt zk:/afile.txt -z localhost:${ZK_PORT} -verbose --solr-home ${SOLR_TIP}/server/solr + run solr zk cp afile.txt zk:/afile.txt -z localhost:${ZK_PORT} --verbose --solr-home ${SOLR_TIP}/server/solr assert_output --partial "Using SolrHome: ${SOLR_TIP}/server/solr" refute_output --partial 'Failed to load solr.xml from ZK or SolrHome' diff --git a/solr/prometheus-exporter/build.gradle b/solr/prometheus-exporter/build.gradle index 4c526f7c896..964c38073f1 100644 --- a/solr/prometheus-exporter/build.gradle +++ b/solr/prometheus-exporter/build.gradle @@ -43,7 +43,7 @@ dependencies { exclude group: "com.fasterxml.jackson.core", module: "jackson-databind" }) implementation libs.fasterxml.jackson.core.databind - implementation libs.argparse4j.argparse4j + implementation libs.commonscli.commonscli implementation(libs.benmanes.caffeine) { transitive = false } implementation libs.slf4j.api implementation libs.commonscodec.commonscodec diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/collector/SchedulerMetricsCollector.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/collector/SchedulerMetricsCollector.java index 8a1a3f60638..26fe8110c7b 100644 --- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/collector/SchedulerMetricsCollector.java +++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/collector/SchedulerMetricsCollector.java @@ -74,7 +74,7 @@ public SchedulerMetricsCollector( } public void start() { - scheduler.scheduleWithFixedDelay(this::collectMetrics, 0, duration, timeUnit); + scheduler.scheduleAtFixedRate(this::collectMetrics, 0, duration, timeUnit); } @SuppressWarnings("try") diff --git a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java index 73aec2c567f..b7918b90901 100644 --- a/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java +++ b/solr/prometheus-exporter/src/java/org/apache/solr/prometheus/exporter/SolrExporter.java @@ -23,10 +23,14 @@ import java.net.InetSocketAddress; import java.nio.file.Paths; import java.util.concurrent.ExecutorService; -import net.sourceforge.argparse4j.ArgumentParsers; -import net.sourceforge.argparse4j.inf.ArgumentParser; -import net.sourceforge.argparse4j.inf.ArgumentParserException; -import net.sourceforge.argparse4j.inf.Namespace; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.DeprecatedAttributes; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; import org.apache.commons.codec.digest.DigestUtils; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.IOUtils; @@ -44,72 +48,14 @@ public class SolrExporter { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static final String[] ARG_PORT_FLAGS = {"-p", "--port"}; - private static final String ARG_PORT_METAVAR = "PORT"; - private static final String ARG_PORT_DEST = "port"; - private static final int ARG_PORT_DEFAULT = 8989; - private static final String ARG_PORT_HELP = - "Specify the solr-exporter HTTP listen port; default is " + ARG_PORT_DEFAULT + "."; - - private static final String[] ARG_BASE_URL_FLAGS = {"-b", "--baseurl"}; - private static final String ARG_BASE_URL_METAVAR = "BASE_URL"; - private static final String ARG_BASE_URL_DEST = "baseUrl"; - private static final String ARG_BASE_URL_DEFAULT = "http://localhost:8983/solr"; - private static final String ARG_BASE_URL_HELP = - "Specify the Solr base URL when connecting to Solr in standalone mode. If omitted both the -b parameter and the -z parameter, connect to http://localhost:8983/solr. For example 'http://localhost:8983/solr'."; - - private static final String[] ARG_ZK_HOST_FLAGS = {"-z", "--zk-host"}; - private static final String ARG_ZK_HOST_METAVAR = "ZK_HOST"; - private static final String ARG_ZK_HOST_DEST = "zkHost"; - private static final String ARG_ZK_HOST_DEFAULT = ""; - private static final String ARG_ZK_HOST_HELP = - "Specify the ZooKeeper connection string when connecting to Solr in SolrCloud mode. If omitted both the -b parameter and the -z parameter, connect to http://localhost:8983/solr. For example 'localhost:2181/solr'."; - - private static final String[] ARG_CLUSTER_ID_FLAGS = {"-i", "--cluster-id"}; - private static final String ARG_CLUSTER_ID_METAVAR = "CLUSTER_ID"; - private static final String ARG_CLUSTER_ID_DEST = "clusterId"; - private static final String ARG_CLUSTER_ID_DEFAULT = ""; - private static final String ARG_CLUSTER_ID_HELP = - "Specify a unique identifier for the cluster, which can be used to select between multiple clusters in Grafana. By default this ID will be equal to a hash of the -b or -z argument"; - - private static final String[] ARG_CONFIG_FLAGS = {"-f", "--config-file"}; - private static final String ARG_CONFIG_METAVAR = "CONFIG"; - private static final String ARG_CONFIG_DEST = "configFile"; - private static final String ARG_CONFIG_DEFAULT = "solr-exporter-config.xml"; - private static final String ARG_CONFIG_HELP = - "Specify the configuration file; the default is " + ARG_CONFIG_DEFAULT + "."; - - private static final String[] ARG_SCRAPE_INTERVAL_FLAGS = {"-s", "--scrape-interval"}; - private static final String ARG_SCRAPE_INTERVAL_METAVAR = "SCRAPE_INTERVAL"; - private static final String ARG_SCRAPE_INTERVAL_DEST = "scrapeInterval"; - private static final int ARG_SCRAPE_INTERVAL_DEFAULT = 60; - private static final String ARG_SCRAPE_INTERVAL_HELP = - "Specify the delay between scraping Solr metrics; the default is " - + ARG_SCRAPE_INTERVAL_DEFAULT - + " seconds."; - - private static final String[] ARG_NUM_THREADS_FLAGS = {"-n", "--num-threads"}; - private static final String ARG_NUM_THREADS_METAVAR = "NUM_THREADS"; - private static final String ARG_NUM_THREADS_DEST = "numThreads"; - private static final Integer ARG_NUM_THREADS_DEFAULT = 1; - private static final String ARG_NUM_THREADS_HELP = - "Specify the number of threads. solr-exporter creates a thread pools for request to Solr. If you need to improve request latency via solr-exporter, you can increase the number of threads; the default is " - + ARG_NUM_THREADS_DEFAULT - + "."; - - private static final String[] ARG_CREDENTIALS_FLAGS = {"-u", "--credentials"}; - private static final String ARG_CREDENTIALS_METAVAR = "CREDENTIALS"; - private static final String ARG_CREDENTIALS_DEST = "credentials"; - private static final String ARG_CREDENTIALS_DEFAULT = ""; - private static final String ARG_CREDENTIALS_HELP = - "Specify the credentials in the format username:password. Example: --credentials solr:SolrRocks"; - - private static final String[] ARG_SSL_FLAGS = {"-ssl", "--ssl-enabled"}; - private static final String ARG_SSL_METAVAR = "SSL_ENABLED"; - private static final String ARG_SSL_DEST = "ssl_enabled"; - private static final boolean ARG_SSL_DEFAULT = false; - private static final String ARG_SSL_HELP = - "Enable TLS connection to Solr. Expects following env variables: SOLR_SSL_KEY_STORE, SOLR_SSL_KEY_STORE_PASSWORD, SOLR_SSL_TRUST_STORE, SOLR_SSL_TRUST_STORE_PASSWORD. Example: --ssl-enabled"; + private static final int DEFAULT_PORT = 8989; + private static final String DEFAULT_BASE_URL = "http://localhost:8983/solr"; + private static final String DEFAULT_ZK_HOST = ""; + private static final String DEFAULT_CLUSTER_ID = ""; + private static final String DEFAULT_CONFIG = "solr-exporter-config.xml"; + private static final int DEFAULT_SCRAPE_INTERVAL = 60; + private static final Integer DEFAULT_NUM_THREADS = 1; + private static final String DEFAULT_CREDENTIALS = ""; public static final CollectorRegistry defaultRegistry = new CollectorRegistry(); @@ -196,116 +142,182 @@ private SolrScraper createScraper( } public static void main(String[] args) { - ArgumentParser parser = - ArgumentParsers.newFor(SolrExporter.class.getSimpleName()) - .build() - .description("Prometheus exporter for Apache Solr."); - - parser - .addArgument(ARG_PORT_FLAGS) - .metavar(ARG_PORT_METAVAR) - .dest(ARG_PORT_DEST) - .type(Integer.class) - .setDefault(ARG_PORT_DEFAULT) - .help(ARG_PORT_HELP); - - parser - .addArgument(ARG_BASE_URL_FLAGS) - .metavar(ARG_BASE_URL_METAVAR) - .dest(ARG_BASE_URL_DEST) - .type(String.class) - .setDefault(ARG_BASE_URL_DEFAULT) - .help(ARG_BASE_URL_HELP); - - parser - .addArgument(ARG_ZK_HOST_FLAGS) - .metavar(ARG_ZK_HOST_METAVAR) - .dest(ARG_ZK_HOST_DEST) - .type(String.class) - .setDefault(ARG_ZK_HOST_DEFAULT) - .help(ARG_ZK_HOST_HELP); - - parser - .addArgument(ARG_CONFIG_FLAGS) - .metavar(ARG_CONFIG_METAVAR) - .dest(ARG_CONFIG_DEST) - .type(String.class) - .setDefault(ARG_CONFIG_DEFAULT) - .help(ARG_CONFIG_HELP); - - parser - .addArgument(ARG_SCRAPE_INTERVAL_FLAGS) - .metavar(ARG_SCRAPE_INTERVAL_METAVAR) - .dest(ARG_SCRAPE_INTERVAL_DEST) - .type(Integer.class) - .setDefault(ARG_SCRAPE_INTERVAL_DEFAULT) - .help(ARG_SCRAPE_INTERVAL_HELP); - - parser - .addArgument(ARG_NUM_THREADS_FLAGS) - .metavar(ARG_NUM_THREADS_METAVAR) - .dest(ARG_NUM_THREADS_DEST) - .type(Integer.class) - .setDefault(ARG_NUM_THREADS_DEFAULT) - .help(ARG_NUM_THREADS_HELP); - - parser - .addArgument(ARG_CLUSTER_ID_FLAGS) - .metavar(ARG_CLUSTER_ID_METAVAR) - .dest(ARG_CLUSTER_ID_DEST) - .type(String.class) - .setDefault(ARG_CLUSTER_ID_DEFAULT) - .help(ARG_CLUSTER_ID_HELP); - - parser - .addArgument(ARG_CREDENTIALS_FLAGS) - .metavar(ARG_CREDENTIALS_METAVAR) - .dest(ARG_CREDENTIALS_DEST) - .type(String.class) - .setDefault(ARG_CREDENTIALS_DEFAULT) - .help(ARG_CREDENTIALS_HELP); - - parser - .addArgument(ARG_SSL_FLAGS) - .metavar(ARG_SSL_METAVAR) - .dest(ARG_SSL_DEST) - .type(Boolean.class) - .setDefault(ARG_SSL_DEFAULT) - .help(ARG_SSL_HELP); + Options mainOptions = new Options(); + Options deprecatedOptions = new Options(); + + Option baseUrlOption = + Option.builder("b") + .longOpt("base-url") + .hasArg() + .argName("BASE_URL") + .type(String.class) + .desc( + "Specify the Solr base URL when connecting to Solr in standalone mode. If omitted both the -b parameter and the -z parameter, connect to http://localhost:8983/solr. For example 'http://localhost:8983/solr'.") + .build(); + mainOptions.addOption(baseUrlOption); + + Option baseUrlDepOption = + Option.builder() + .longOpt("baseUrl") + .hasArg() + .argName("BASE_URL") + .type(String.class) + .deprecated( + DeprecatedAttributes.builder() + .setForRemoval(true) + .setSince("9.7") + .setDescription("Use --base-url instead") + .get()) + .desc( + "Specify the Solr base URL when connecting to Solr in standalone mode. If omitted both the -b parameter and the -z parameter, connect to http://localhost:8983/solr. For example 'http://localhost:8983/solr'.") + .build(); + deprecatedOptions.addOption(baseUrlDepOption); + + Option configOption = + Option.builder("f") + .longOpt("config-file") + .hasArg() + .argName("CONFIG") + .type(String.class) + .desc("Specify the configuration file; the default is " + DEFAULT_CONFIG + ".") + .build(); + mainOptions.addOption(configOption); + + Option helpOption = + Option.builder("h").longOpt("help").desc("Prints this help message.").build(); + mainOptions.addOption(helpOption); + + Option clusterIdOption = + Option.builder("i") + .longOpt("cluster-id") + .hasArg() + .argName("CLUSTER_ID") + .type(String.class) + .desc( + "Specify a unique identifier for the cluster, which can be used to select between multiple clusters in Grafana. By default this ID will be equal to a hash of the -b or -z argument") + .build(); + mainOptions.addOption(clusterIdOption); + + Option numThreadsOption = + Option.builder("n") + .longOpt("num-threads") + .hasArg() + .argName("NUM_THREADS") + .type(Integer.class) + .desc( + "Specify the number of threads. solr-exporter creates a thread pools for request to Solr. If you need to improve request latency via solr-exporter, you can increase the number of threads; the default is " + + DEFAULT_NUM_THREADS + + ".") + .build(); + mainOptions.addOption(numThreadsOption); + + Option portOption = + Option.builder("p") + .longOpt("port") + .hasArg() + .argName("PORT") + .type(Integer.class) + .desc("Specify the solr-exporter HTTP listen port; default is " + DEFAULT_PORT + ".") + .build(); + mainOptions.addOption(portOption); + + Option scrapeIntervalOption = + Option.builder("s") + .longOpt("scrape-interval") + .hasArg() + .argName("SCRAPE_INTERVAL") + .type(Integer.class) + .desc( + "Specify the delay between scraping Solr metrics; the default is " + + DEFAULT_SCRAPE_INTERVAL + + " seconds.") + .build(); + mainOptions.addOption(scrapeIntervalOption); + + Option sslOption = + Option.builder("ssl") + .longOpt("ssl-enabled") + .type(Boolean.class) + .desc( + "Enable TLS connection to Solr. Expects following env variables: SOLR_SSL_KEY_STORE, SOLR_SSL_KEY_STORE_PASSWORD, SOLR_SSL_TRUST_STORE, SOLR_SSL_TRUST_STORE_PASSWORD. Example: --ssl-enabled") + .build(); + mainOptions.addOption(sslOption); + + Option credentialsOption = + Option.builder("u") + .longOpt("credentials") + .hasArg() + .argName("CREDENTIALS") + .type(String.class) + .desc( + "Specify the credentials in the format username:password. Example: --credentials solr:SolrRocks") + .build(); + mainOptions.addOption(credentialsOption); + + Option zkHostOption = + Option.builder("z") + .longOpt("zk-host") + .hasArg() + .argName("ZK_HOST") + .type(String.class) + .desc( + "Specify the ZooKeeper connection string when connecting to Solr in SolrCloud mode. If omitted both the -b parameter and the -z parameter, connect to http://localhost:8983/solr. For example 'localhost:2181/solr'.") + .build(); + mainOptions.addOption(zkHostOption); + + Options options = new Options(); + options.addOptions(mainOptions); + options.addOptions(deprecatedOptions); try { - Namespace res = parser.parseArgs(args); + CommandLineParser parser = new DefaultParser(); + CommandLine commandLine = parser.parse(options, args); + + if (commandLine.hasOption(helpOption)) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp( + "bin/solr-exporter", "Prometheus exporter for Apache Solr.", mainOptions, null, true); + return; + } SolrScrapeConfiguration scrapeConfiguration = null; String defaultClusterId = ""; - if (!res.getString(ARG_ZK_HOST_DEST).isEmpty()) { - defaultClusterId = makeShortHash(res.getString(ARG_ZK_HOST_DEST)); - scrapeConfiguration = SolrScrapeConfiguration.solrCloud(res.getString(ARG_ZK_HOST_DEST)); - } else if (!res.getString(ARG_BASE_URL_DEST).isEmpty()) { - defaultClusterId = makeShortHash(res.getString(ARG_BASE_URL_DEST)); - scrapeConfiguration = SolrScrapeConfiguration.standalone(res.getString(ARG_BASE_URL_DEST)); + if (commandLine.hasOption(zkHostOption)) { + String zkHost = commandLine.getOptionValue(zkHostOption, DEFAULT_ZK_HOST); + defaultClusterId = makeShortHash(zkHost); + scrapeConfiguration = SolrScrapeConfiguration.solrCloud(zkHost); + } else if (commandLine.hasOption(baseUrlOption) || commandLine.hasOption(baseUrlDepOption)) { + String baseUrl = + commandLine.hasOption(baseUrlOption) + ? commandLine.getOptionValue(baseUrlOption) + : commandLine.getOptionValue(baseUrlDepOption, DEFAULT_BASE_URL); + defaultClusterId = makeShortHash(baseUrl); + scrapeConfiguration = SolrScrapeConfiguration.standalone(baseUrl); } if (scrapeConfiguration == null) { - log.error("Must provide either {} or {}", ARG_BASE_URL_FLAGS, ARG_ZK_HOST_FLAGS); + log.error( + "Must provide either --{} or --{}", + baseUrlOption.getLongOpt(), + zkHostOption.getLongOpt()); } - int port = res.getInt(ARG_PORT_DEST); - String clusterId = res.getString(ARG_CLUSTER_ID_DEST); + int port = commandLine.getParsedOptionValue(portOption, DEFAULT_PORT); + String clusterId = commandLine.getOptionValue(clusterIdOption, DEFAULT_CLUSTER_ID); if (StrUtils.isNullOrEmpty(clusterId)) { clusterId = defaultClusterId; } - if (!res.getString(ARG_CREDENTIALS_DEST).isEmpty()) { - String credentials = res.getString(ARG_CREDENTIALS_DEST); + if (commandLine.hasOption(credentialsOption)) { + String credentials = commandLine.getOptionValue(credentialsOption, DEFAULT_CREDENTIALS); if (credentials.indexOf(':') > 0) { String[] credentialsArray = credentials.split(":", 2); scrapeConfiguration.withBasicAuthCredentials(credentialsArray[0], credentialsArray[1]); } } - if (Boolean.TRUE.equals(res.getBoolean(ARG_SSL_DEST))) { + if (commandLine.hasOption(sslOption)) { log.info("SSL ENABLED"); scrapeConfiguration.withSslConfiguration( @@ -318,10 +330,10 @@ public static void main(String[] args) { SolrExporter solrExporter = new SolrExporter( port, - res.getInt(ARG_NUM_THREADS_DEST), - res.getInt(ARG_SCRAPE_INTERVAL_DEST), + commandLine.getParsedOptionValue(numThreadsOption, DEFAULT_NUM_THREADS), + commandLine.getParsedOptionValue(scrapeIntervalOption, DEFAULT_SCRAPE_INTERVAL), scrapeConfiguration, - loadMetricsConfiguration(res.getString(ARG_CONFIG_DEST)), + loadMetricsConfiguration(commandLine.getOptionValue(configOption, DEFAULT_CONFIG)), clusterId); log.info("Starting Solr Prometheus Exporting on port {}", port); @@ -332,8 +344,8 @@ public static void main(String[] args) { scrapeConfiguration); } catch (IOException e) { log.error("Failed to start Solr Prometheus Exporter: ", e); - } catch (ArgumentParserException e) { - parser.handleError(e); + } catch (ParseException e) { + log.error("Failed to parse command line arguments: ", e); } } diff --git a/solr/prometheus-exporter/src/test-files/solr/collection1/conf/managed-schema.xml b/solr/prometheus-exporter/src/test-files/solr/collection1/conf/managed-schema.xml index 76a6f64af34..2c78c1667f0 100644 --- a/solr/prometheus-exporter/src/test-files/solr/collection1/conf/managed-schema.xml +++ b/solr/prometheus-exporter/src/test-files/solr/collection1/conf/managed-schema.xml @@ -55,7 +55,7 @@ 1.5: omitNorms defaults to true for primitive field types (int, float, boolean, string...) 1.6: useDocValuesAsStored defaults to true. - 1.7: docValues defaults to true. + 1.7: docValues defaults to true, uninvertible defaults to false. --> - + - + diff --git a/solr/prometheus-exporter/src/test-files/solr/collection1/conf/solrconfig.xml b/solr/prometheus-exporter/src/test-files/solr/collection1/conf/solrconfig.xml index bbc9b68628b..6ca22087fe4 100644 --- a/solr/prometheus-exporter/src/test-files/solr/collection1/conf/solrconfig.xml +++ b/solr/prometheus-exporter/src/test-files/solr/collection1/conf/solrconfig.xml @@ -35,7 +35,6 @@ ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} diff --git a/solr/server/solr/configsets/_default/conf/managed-schema.xml b/solr/server/solr/configsets/_default/conf/managed-schema.xml index 5250e837ff5..8d7b974bec0 100644 --- a/solr/server/solr/configsets/_default/conf/managed-schema.xml +++ b/solr/server/solr/configsets/_default/conf/managed-schema.xml @@ -55,7 +55,7 @@ 1.5: omitNorms defaults to true for primitive field types (int, float, boolean, string...) 1.6: useDocValuesAsStored defaults to true. - 1.7: docValues defaults to true. + 1.7: docValues defaults to true, uninvertible defaults to false. --> ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} @@ -134,11 +134,11 @@ - + - - + + diff --git a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml index 1cdef7ca55f..c8250755dc7 100644 --- a/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml +++ b/solr/server/solr/configsets/sample_techproducts_configs/conf/solrconfig.xml @@ -268,16 +268,9 @@ is recommended (see below). "dir" - the target directory for transaction logs, defaults to the solr data directory. - "numVersionBuckets" - sets the number of buckets used to keep - track of max version values when checking for re-ordered - updates; increase this value to reduce the cost of - synchronizing access to version buckets during high-volume - indexing, this requires 8 bytes (long) * numVersionBuckets - of heap space per Solr core. --> ${solr.ulog.dir:} - ${solr.ulog.numVersionBuckets:65536} - - + + diff --git a/solr/solrj-streaming/src/test-files/solrj/solr/configsets/streaming/conf/schema.xml b/solr/solrj-streaming/src/test-files/solrj/solr/configsets/streaming/conf/schema.xml index 5784fd0ffe4..5a202baa2b8 100644 --- a/solr/solrj-streaming/src/test-files/solrj/solr/configsets/streaming/conf/schema.xml +++ b/solr/solrj-streaming/src/test-files/solrj/solr/configsets/streaming/conf/schema.xml @@ -43,12 +43,12 @@ - + - - - - + + + + diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/CollectionPropertiesZkStateReader.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/CollectionPropertiesZkStateReader.java new file mode 100644 index 00000000000..93ea4d9cfe3 --- /dev/null +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/CollectionPropertiesZkStateReader.java @@ -0,0 +1,411 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.common.cloud; + +import static java.util.Collections.emptyMap; + +import java.io.Closeable; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.ZkStateReader.CollectionWatch; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.common.util.Utils; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; +import org.apache.zookeeper.data.Stat; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Fetches and manages collection properties from a ZooKeeper ensemble */ +public class CollectionPropertiesZkStateReader implements Closeable { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + private volatile boolean closed = false; + + private final SolrZkClient zkClient; + + /** Collection properties being actively watched */ + private final ConcurrentHashMap watchedCollectionProps = + new ConcurrentHashMap<>(); + + /** + * Manages ZooKeeper watchers for each collection. These watchers monitor changes to the + * properties of the collection in ZooKeeper. When a change is detected in ZooKeeper, the watcher + * triggers an update, which then notifies the relevant "collectionPropsObserver". + */ + private final ConcurrentHashMap collectionPropsWatchers = + new ConcurrentHashMap<>(); + + /** + * Manages a list of observers (listeners) for each collection. These observers need to be + * notified when the properties of the collection change. When a collection's properties change, + * all registered observers for that collection are notified by a "collectionPropWatcher". + */ + private ConcurrentHashMap> + collectionPropsObservers = new ConcurrentHashMap<>(); + + /** Used to submit notifications to Collection Properties watchers in order */ + private final ExecutorService collectionPropsNotifications = + ExecutorUtil.newMDCAwareSingleThreadExecutor( + new SolrNamedThreadFactory("collectionPropsNotifications")); + + private volatile ScheduledThreadPoolExecutor cacheCleanerExecutor = null; + + private final ConcurrentHashMap collectionLocks = new ConcurrentHashMap<>(); + + public CollectionPropertiesZkStateReader(ZkStateReader zkStateReader) { + this.zkClient = zkStateReader.getZkClient(); + } + + /** + * Get and cache collection properties for a given collection. If the collection is watched, or + * still cached simply return it from the cache, otherwise fetch it directly from zookeeper and + * retain the value for at least cacheForMillis milliseconds. Cached properties are watched in + * zookeeper and updated automatically. This version of {@code getCollectionProperties} should be + * used when properties need to be consulted frequently in the absence of an active {@link + * CollectionPropsWatcher}. + * + * @param collection The collection for which properties are desired + * @param cacheForMillis The minimum number of milliseconds to maintain a cache for the specified + * collection's properties. Setting a {@code CollectionPropsWatcher} will override this value + * and retain the cache for the life of the watcher. A lack of changes in zookeeper may allow + * the caching to remain for a greater duration up to the cycle time of {@code CacheCleaner}. + * Passing zero for this value will explicitly remove the cached copy if and only if it is due + * to expire and no watch exists. Any positive value will extend the expiration time if + * required. + * @return a map representing the key/value properties for the collection. + */ + public Map getCollectionProperties(final String collection, long cacheForMillis) { + Watcher watcher = null; // synchronized on the specific collection + if (cacheForMillis > 0) { + watcher = + collectionPropsWatchers.compute( + collection, + (c, w) -> w == null ? new PropsWatcher(c, cacheForMillis) : w.renew(cacheForMillis)); + } + VersionedCollectionProps vprops = watchedCollectionProps.get(collection); + boolean haveUnexpiredProps = vprops != null && vprops.cacheUntilNs > System.nanoTime(); + long untilNs = + System.nanoTime() + TimeUnit.NANOSECONDS.convert(cacheForMillis, TimeUnit.MILLISECONDS); + if (haveUnexpiredProps) { + vprops.cacheUntilNs = Math.max(vprops.cacheUntilNs, untilNs); + return vprops.props; + } + // Synchronize only when properties are expired or not present + synchronized (getCollectionLock(collection)) { + // Re-check inside the synchronized block to avoid race conditions + vprops = watchedCollectionProps.get(collection); + haveUnexpiredProps = vprops != null && vprops.cacheUntilNs > System.nanoTime(); + if (haveUnexpiredProps) { + vprops.cacheUntilNs = Math.max(vprops.cacheUntilNs, untilNs); + return vprops.props; + } + try { + VersionedCollectionProps vcp = fetchCollectionProperties(collection, watcher); + Map properties = vcp.props; + if (cacheForMillis > 0) { + vcp.cacheUntilNs = untilNs; + watchedCollectionProps.put(collection, vcp); + } else { + // we're synchronized on watchedCollectionProps and we can only get here if we have + // found an expired vprops above, so it is safe to remove the cached value and let the + // GC free up some mem a bit sooner. + if (!collectionPropsObservers.containsKey(collection)) { + watchedCollectionProps.remove(collection); + } + } + return properties; + } catch (Exception e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Error reading collection properties", + SolrZkClient.checkInterrupted(e)); + } + } + } + + @Override + public void close() { + this.closed = true; + ExecutorUtil.shutdownAndAwaitTermination(cacheCleanerExecutor); + ExecutorUtil.shutdownAndAwaitTermination(collectionPropsNotifications); + } + + private static class VersionedCollectionProps { + int zkVersion; + Map props; + long cacheUntilNs = 0; + + VersionedCollectionProps(int zkVersion, Map props) { + this.zkVersion = zkVersion; + this.props = props; + } + } + + /** Watches collection properties */ + class PropsWatcher implements Watcher { + private final String coll; + private long watchUntilNs; + + PropsWatcher(String coll) { + this.coll = coll; + watchUntilNs = 0; + } + + PropsWatcher(String coll, long forMillis) { + this.coll = coll; + watchUntilNs = + System.nanoTime() + TimeUnit.NANOSECONDS.convert(forMillis, TimeUnit.MILLISECONDS); + } + + public PropsWatcher renew(long forMillis) { + watchUntilNs = + System.nanoTime() + TimeUnit.NANOSECONDS.convert(forMillis, TimeUnit.MILLISECONDS); + return this; + } + + @Override + public void process(WatchedEvent event) { + // session events are not change events, and do not remove the watcher + if (Event.EventType.None.equals(event.getType())) { + return; + } + + boolean expired = System.nanoTime() > watchUntilNs; + if (!collectionPropsObservers.containsKey(coll) && expired) { + // No one can be notified of the change, we can ignore it and "unset" the watch + log.debug("Ignoring property change for collection {}", coll); + return; + } + + log.info( + "A collection property change: [{}] for collection [{}] has occurred - updating...", + event, + coll); + + refreshAndWatch(true); + } + + /** + * Refresh collection properties from ZK and leave a watch for future changes. Updates the + * properties in watchedCollectionProps with the results of the refresh. Optionally notifies + * watchers + */ + void refreshAndWatch(boolean notifyWatchers) { + try { + synchronized (getCollectionLock(coll)) { + // making decisions based on the result of a get... + VersionedCollectionProps vcp = fetchCollectionProperties(coll, this); + Map properties = vcp.props; + VersionedCollectionProps existingVcp = watchedCollectionProps.get(coll); + if (existingVcp == null + || // never called before, record what we found + vcp.zkVersion > existingVcp.zkVersion + || // newer info we should update + vcp.zkVersion == -1) { // node was deleted start over + watchedCollectionProps.put(coll, vcp); + if (notifyWatchers) { + notifyPropsWatchers(coll, properties); + } + if (vcp.zkVersion == -1 && existingVcp != null) { // Collection DELETE detected + + // We should not be caching a collection that has been deleted. + watchedCollectionProps.remove(coll); + + // core ref counting not relevant here, don't need canRemove(), we just sent + // a notification of an empty set of properties, no reason to watch what doesn't + // exist. + collectionPropsObservers.remove(coll); + + // This is the one time we know it's safe to throw this out. We just failed to set the + // watch due to an NoNodeException, so it isn't held by ZK and can't re-set itself due + // to an update. + collectionPropsWatchers.remove(coll); + } + } + } + } catch (KeeperException.SessionExpiredException + | KeeperException.ConnectionLossException e) { + log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: ", e); + } catch (KeeperException e) { + log.error("Lost collection property watcher for {} due to ZK error", coll, e); + throw new ZooKeeperException( + SolrException.ErrorCode.SERVER_ERROR, "A ZK error has occurred", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.error( + "Lost collection property watcher for {} due to the thread being interrupted", coll, e); + } + } + } + + private Object getCollectionLock(String collection) { + return collectionLocks.computeIfAbsent(collection, k -> new Object()); + } + + public void registerCollectionPropsWatcher( + final String collection, CollectionPropsWatcher propsWatcher) { + AtomicBoolean watchSet = new AtomicBoolean(false); + collectionPropsObservers.compute( + collection, + (k, v) -> { + if (v == null) { + v = new CollectionWatch<>(); + watchSet.set(true); + } + v.stateWatchers.add(propsWatcher); + return v; + }); + + if (watchSet.get()) { + collectionPropsWatchers.computeIfAbsent(collection, PropsWatcher::new).refreshAndWatch(false); + } + } + + protected void refreshCollectionProperties() { + collectionPropsObservers.forEach( + (k, v) -> { + collectionPropsWatchers.computeIfAbsent(k, PropsWatcher::new).refreshAndWatch(true); + }); + } + + public static String getCollectionPropsPath(final String collection) { + return ZkStateReader.COLLECTIONS_ZKNODE + + '/' + + collection + + '/' + + ZkStateReader.COLLECTION_PROPS_ZKNODE; + } + + private VersionedCollectionProps fetchCollectionProperties(String collection, Watcher watcher) + throws KeeperException, InterruptedException { + final String znodePath = getCollectionPropsPath(collection); + // lazy init cache cleaner once we know someone is using collection properties. + if (cacheCleanerExecutor == null) { + synchronized (this) { + if (cacheCleanerExecutor == null) { + cacheCleanerExecutor = new ScheduledThreadPoolExecutor(1); + cacheCleanerExecutor.scheduleAtFixedRate(new CacheCleaner(), 0, 1, TimeUnit.MINUTES); + } + } + } + while (true) { + try { + Stat stat = new Stat(); + byte[] data = zkClient.getData(znodePath, watcher, stat, true); + @SuppressWarnings("unchecked") + Map props = (Map) Utils.fromJSON(data); + return new VersionedCollectionProps(stat.getVersion(), props); + } catch (ClassCastException e) { + throw new SolrException( + SolrException.ErrorCode.SERVER_ERROR, + "Unable to parse collection properties for collection " + collection, + e); + } catch (KeeperException.NoNodeException e) { + if (watcher != null) { + // Leave an exists watch in place in case a collectionprops.json is created later. + Stat exists = zkClient.exists(znodePath, watcher, true); + if (exists != null) { + // Rare race condition, we tried to fetch the data and couldn't find it, then we found + // it exists. Loop and try again. + continue; + } + } + return new VersionedCollectionProps(-1, emptyMap()); + } + } + } + + private void notifyPropsWatchers(String collection, Map properties) { + try { + collectionPropsNotifications.submit(new PropsNotification(collection, properties)); + } catch (RejectedExecutionException e) { + if (!closed) { + log.error("Couldn't run collection properties notifications for {}", collection, e); + } + } + } + + private class PropsNotification implements Runnable { + + private final String collection; + private final Map collectionProperties; + private final List watchers = new ArrayList<>(); + + private PropsNotification(String collection, Map collectionProperties) { + this.collection = collection; + this.collectionProperties = collectionProperties; + // guarantee delivery of notification regardless of what happens to collectionPropsObservers + // while we wait our turn in the executor by capturing the list on creation. + collectionPropsObservers.compute( + collection, + (k, v) -> { + if (v == null) return null; + watchers.addAll(v.stateWatchers); + return v; + }); + } + + @Override + public void run() { + for (CollectionPropsWatcher watcher : watchers) { + if (watcher.onStateChanged(collectionProperties)) { + removeCollectionPropsWatcher(collection, watcher); + } + } + } + } + + private class CacheCleaner implements Runnable { + @Override + public void run() { + watchedCollectionProps + .entrySet() + .removeIf( + entry -> + entry.getValue().cacheUntilNs < System.nanoTime() + && !collectionPropsObservers.containsKey(entry.getKey())); + } + } + + public void removeCollectionPropsWatcher(String collection, CollectionPropsWatcher watcher) { + collectionPropsObservers.compute( + collection, + (k, v) -> { + if (v == null) return null; + v.stateWatchers.remove(watcher); + if (v.canBeRemoved()) { + // don't want this to happen in middle of other blocks that might add it back. + synchronized (getCollectionLock(collection)) { + watchedCollectionProps.remove(collection); + return null; + } + } + return v; + }); + } +} diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java index 7acb40eb3b5..a3fe06f4a11 100644 --- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java @@ -137,7 +137,7 @@ public static void zkTransfer( throw new SolrServerException("One or both of source or destination must specify ZK nodes."); } - // Make sure -recurse is specified if the source has children. + // Make sure --recurse is specified if the source has children. if (recurse == false) { if (srcIsZk) { if (zkClient.getChildren(src, null, true).size() != 0) { diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java index b9882ddcc11..c2f76d62de3 100644 --- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkStateReader.java @@ -37,7 +37,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; @@ -58,7 +57,6 @@ import org.apache.solr.common.util.CommonTestInjection; import org.apache.solr.common.util.ExecutorUtil; import org.apache.solr.common.util.ObjectReleaseTracker; -import org.apache.solr.common.util.SolrNamedThreadFactory; import org.apache.solr.common.util.Utils; import org.apache.zookeeper.KeeperException; import org.apache.zookeeper.KeeperException.NoNodeException; @@ -156,6 +154,7 @@ public class ZkStateReader implements SolrCloseable { public static final String CONTAINER_PLUGINS = "plugin"; public static final String PLACEMENT_PLUGIN = "placement-plugin"; + private final CollectionPropertiesZkStateReader collectionPropertiesZkStateReader; /** A view of the current state of all collections. */ protected volatile ClusterState clusterState; @@ -173,14 +172,6 @@ public class ZkStateReader implements SolrCloseable { private final ConcurrentHashMap lazyCollectionStates = new ConcurrentHashMap<>(); - /** Collection properties being actively watched */ - private final ConcurrentHashMap watchedCollectionProps = - new ConcurrentHashMap<>(); - - /** Watchers of Collection properties */ - private final ConcurrentHashMap collectionPropsWatchers = - new ConcurrentHashMap<>(); - private volatile SortedSet liveNodes = emptySortedSet(); private volatile Map clusterProperties = Collections.emptyMap(); @@ -191,11 +182,6 @@ public class ZkStateReader implements SolrCloseable { */ private DocCollectionWatches collectionWatches = new DocCollectionWatches(); - // named this observers so there's less confusion between CollectionPropsWatcher map and the - // PropsWatcher map. - private ConcurrentHashMap> - collectionPropsObservers = new ConcurrentHashMap<>(); - private Set cloudCollectionsListeners = ConcurrentHashMap.newKeySet(); private final ExecutorService notifications = ExecutorUtil.newMDCAwareCachedThreadPool("watches"); @@ -204,17 +190,9 @@ public class ZkStateReader implements SolrCloseable { private Set clusterPropertiesListeners = ConcurrentHashMap.newKeySet(); - /** Used to submit notifications to Collection Properties watchers in order */ - private final ExecutorService collectionPropsNotifications = - ExecutorUtil.newMDCAwareSingleThreadExecutor( - new SolrNamedThreadFactory("collectionPropsNotifications")); - private static final long LAZY_CACHE_TIME = TimeUnit.NANOSECONDS.convert(STATE_UPDATE_DELAY, TimeUnit.MILLISECONDS); - // only kept to identify if the cleaner has already been started. - private Future collectionPropsCacheCleaner; - /** * Gets the ZkStateReader inside a ZK based SolrClient. * @@ -229,7 +207,7 @@ public static ZkStateReader from(CloudSolrClient solrClient) { } } - private static class CollectionWatch { + protected static class CollectionWatch { int coreRefCount = 0; Set stateWatchers = ConcurrentHashMap.newKeySet(); @@ -417,6 +395,7 @@ public ZkStateReader(SolrZkClient zkClient, Runnable securityNodeListener) { this.zkClient = zkClient; this.closeClient = false; this.securityNodeWatcher = new SecurityNodeWatcher(this, securityNodeListener); + collectionPropertiesZkStateReader = new CollectionPropertiesZkStateReader(this); assert ObjectReleaseTracker.track(this); } @@ -454,7 +433,7 @@ public ZkStateReader( this.zkClient = builder.build(); this.closeClient = true; this.securityNodeWatcher = null; - + collectionPropertiesZkStateReader = new CollectionPropertiesZkStateReader(this); assert ObjectReleaseTracker.track(this); } @@ -592,11 +571,7 @@ public synchronized void createClusterStateWatchersAndUpdate() if (securityNodeWatcher != null) { securityNodeWatcher.register(); } - - collectionPropsObservers.forEach( - (k, v) -> { - collectionPropsWatchers.computeIfAbsent(k, PropsWatcher::new).refreshAndWatch(true); - }); + collectionPropertiesZkStateReader.refreshCollectionProperties(); } catch (KeeperException.NoNodeException nne) { throw new SolrException( ErrorCode.SERVICE_UNAVAILABLE, @@ -896,6 +871,10 @@ public Object getUpdateLock() { return this; } + public SolrZkClient getZKClient() { + return zkClient; + } + @Override public void close() { this.closed = true; @@ -909,7 +888,7 @@ public void close() { }); ExecutorUtil.shutdownAndAwaitTermination(notifications); - ExecutorUtil.shutdownAndAwaitTermination(collectionPropsNotifications); + collectionPropertiesZkStateReader.close(); if (closeClient) { zkClient.close(); } @@ -1190,131 +1169,18 @@ private void loadClusterProperties() { } } - /** - * Get collection properties for a given collection. If the collection is watched, simply return - * it from the cache, otherwise fetch it directly from zookeeper. This is a convenience for {@code - * getCollectionProperties(collection,0)} - * - * @param collection the collection for which properties are desired - * @return a map representing the key/value properties for the collection. - */ + /** Get properties for a specific collection */ public Map getCollectionProperties(final String collection) { - return getCollectionProperties(collection, 0); + return collectionPropertiesZkStateReader.getCollectionProperties(collection, 0); } - /** - * Get and cache collection properties for a given collection. If the collection is watched, or - * still cached simply return it from the cache, otherwise fetch it directly from zookeeper and - * retain the value for at least cacheForMillis milliseconds. Cached properties are watched in - * zookeeper and updated automatically. This version of {@code getCollectionProperties} should be - * used when properties need to be consulted frequently in the absence of an active {@link - * CollectionPropsWatcher}. - * - * @param collection The collection for which properties are desired - * @param cacheForMillis The minimum number of milliseconds to maintain a cache for the specified - * collection's properties. Setting a {@code CollectionPropsWatcher} will override this value - * and retain the cache for the life of the watcher. A lack of changes in zookeeper may allow - * the caching to remain for a greater duration up to the cycle time of {@code CacheCleaner}. - * Passing zero for this value will explicitly remove the cached copy if and only if it is due - * to expire and no watch exists. Any positive value will extend the expiration time if - * required. - * @return a map representing the key/value properties for the collection. - */ + /** Get and cache collection properties for a given collection */ public Map getCollectionProperties(final String collection, long cacheForMillis) { - synchronized (watchedCollectionProps) { // making decisions based on the result of a get... - Watcher watcher = null; - if (cacheForMillis > 0) { - watcher = - collectionPropsWatchers.compute( - collection, - (c, w) -> - w == null ? new PropsWatcher(c, cacheForMillis) : w.renew(cacheForMillis)); - } - VersionedCollectionProps vprops = watchedCollectionProps.get(collection); - boolean haveUnexpiredProps = vprops != null && vprops.cacheUntilNs > System.nanoTime(); - long untilNs = - System.nanoTime() + TimeUnit.NANOSECONDS.convert(cacheForMillis, TimeUnit.MILLISECONDS); - Map properties; - if (haveUnexpiredProps) { - properties = vprops.props; - vprops.cacheUntilNs = Math.max(vprops.cacheUntilNs, untilNs); - } else { - try { - VersionedCollectionProps vcp = fetchCollectionProperties(collection, watcher); - properties = vcp.props; - if (cacheForMillis > 0) { - vcp.cacheUntilNs = untilNs; - watchedCollectionProps.put(collection, vcp); - } else { - // we're synchronized on watchedCollectionProps and we can only get here if we have - // found an expired vprops above, so it is safe to remove the cached value and let the - // GC free up some mem a bit sooner. - if (!collectionPropsObservers.containsKey(collection)) { - watchedCollectionProps.remove(collection); - } - } - } catch (Exception e) { - throw new SolrException( - ErrorCode.SERVER_ERROR, - "Error reading collection properties", - SolrZkClient.checkInterrupted(e)); - } - } - return properties; - } - } - - private static class VersionedCollectionProps { - int zkVersion; - Map props; - long cacheUntilNs = 0; - - VersionedCollectionProps(int zkVersion, Map props) { - this.zkVersion = zkVersion; - this.props = props; - } + return collectionPropertiesZkStateReader.getCollectionProperties(collection, cacheForMillis); } static String getCollectionPropsPath(final String collection) { - return COLLECTIONS_ZKNODE + '/' + collection + '/' + COLLECTION_PROPS_ZKNODE; - } - - private VersionedCollectionProps fetchCollectionProperties(String collection, Watcher watcher) - throws KeeperException, InterruptedException { - final String znodePath = getCollectionPropsPath(collection); - // lazy init cache cleaner once we know someone is using collection properties. - if (collectionPropsCacheCleaner == null) { - synchronized (this) { // There can be only one! :) - if (collectionPropsCacheCleaner == null) { - collectionPropsCacheCleaner = notifications.submit(new CacheCleaner()); - } - } - } - while (true) { - try { - Stat stat = new Stat(); - byte[] data = zkClient.getData(znodePath, watcher, stat, true); - @SuppressWarnings("unchecked") - Map props = (Map) Utils.fromJSON(data); - return new VersionedCollectionProps(stat.getVersion(), props); - } catch (ClassCastException e) { - throw new SolrException( - ErrorCode.SERVER_ERROR, - "Unable to parse collection properties for collection " + collection, - e); - } catch (KeeperException.NoNodeException e) { - if (watcher != null) { - // Leave an exists watch in place in case a collectionprops.json is created later. - Stat exists = zkClient.exists(znodePath, watcher, true); - if (exists != null) { - // Rare race condition, we tried to fetch the data and couldn't find it, then we found - // it exists. Loop and try again. - continue; - } - } - return new VersionedCollectionProps(-1, emptyMap()); - } - } + return CollectionPropertiesZkStateReader.getCollectionPropsPath(collection); } /** @@ -1453,101 +1319,6 @@ private void refreshAndWatchChildren() throws KeeperException, InterruptedExcept } } - /** Watches collection properties */ - class PropsWatcher implements Watcher { - private final String coll; - private long watchUntilNs; - - PropsWatcher(String coll) { - this.coll = coll; - watchUntilNs = 0; - } - - PropsWatcher(String coll, long forMillis) { - this.coll = coll; - watchUntilNs = - System.nanoTime() + TimeUnit.NANOSECONDS.convert(forMillis, TimeUnit.MILLISECONDS); - } - - public PropsWatcher renew(long forMillis) { - watchUntilNs = - System.nanoTime() + TimeUnit.NANOSECONDS.convert(forMillis, TimeUnit.MILLISECONDS); - return this; - } - - @Override - public void process(WatchedEvent event) { - // session events are not change events, and do not remove the watcher - if (EventType.None.equals(event.getType())) { - return; - } - - boolean expired = System.nanoTime() > watchUntilNs; - if (!collectionPropsObservers.containsKey(coll) && expired) { - // No one can be notified of the change, we can ignore it and "unset" the watch - log.debug("Ignoring property change for collection {}", coll); - return; - } - - log.info( - "A collection property change: [{}] for collection [{}] has occurred - updating...", - event, - coll); - - refreshAndWatch(true); - } - - /** - * Refresh collection properties from ZK and leave a watch for future changes. Updates the - * properties in watchedCollectionProps with the results of the refresh. Optionally notifies - * watchers - */ - void refreshAndWatch(boolean notifyWatchers) { - try { - synchronized (watchedCollectionProps) { // making decisions based on the result of a get... - VersionedCollectionProps vcp = fetchCollectionProperties(coll, this); - Map properties = vcp.props; - VersionedCollectionProps existingVcp = watchedCollectionProps.get(coll); - if (existingVcp == null - || // never called before, record what we found - vcp.zkVersion > existingVcp.zkVersion - || // newer info we should update - vcp.zkVersion == -1) { // node was deleted start over - watchedCollectionProps.put(coll, vcp); - if (notifyWatchers) { - notifyPropsWatchers(coll, properties); - } - if (vcp.zkVersion == -1 && existingVcp != null) { // Collection DELETE detected - - // We should not be caching a collection that has been deleted. - watchedCollectionProps.remove(coll); - - // core ref counting not relevant here, don't need canRemove(), we just sent - // a notification of an empty set of properties, no reason to watch what doesn't - // exist. - collectionPropsObservers.remove(coll); - - // This is the one time we know it's safe to throw this out. We just failed to set the - // watch due to an NoNodeException, so it isn't held by ZK and can't re-set itself due - // to an update. - collectionPropsWatchers.remove(coll); - } - } - } - } catch (KeeperException.SessionExpiredException - | KeeperException.ConnectionLossException e) { - log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: ", e); - } catch (KeeperException e) { - log.error("Lost collection property watcher for {} due to ZK error", coll, e); - throw new ZooKeeperException(ErrorCode.SERVER_ERROR, "A ZK error has occurred", e); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - log.error( - "Lost collection property watcher for {} due to the thread being interrupted", coll, e); - } - } - } - /** Watches /collections children . */ class CollectionsChildWatcher implements Watcher { @@ -2069,38 +1840,12 @@ Map getCollectionWatches() { public void registerCollectionPropsWatcher( final String collection, CollectionPropsWatcher propsWatcher) { - AtomicBoolean watchSet = new AtomicBoolean(false); - collectionPropsObservers.compute( - collection, - (k, v) -> { - if (v == null) { - v = new CollectionWatch<>(); - watchSet.set(true); - } - v.stateWatchers.add(propsWatcher); - return v; - }); - if (watchSet.get()) { - collectionPropsWatchers.computeIfAbsent(collection, PropsWatcher::new).refreshAndWatch(false); - } + collectionPropertiesZkStateReader.registerCollectionPropsWatcher(collection, propsWatcher); } public void removeCollectionPropsWatcher(String collection, CollectionPropsWatcher watcher) { - collectionPropsObservers.compute( - collection, - (k, v) -> { - if (v == null) return null; - v.stateWatchers.remove(watcher); - if (v.canBeRemoved()) { - // don't want this to happen in middle of other blocks that might add it back. - synchronized (watchedCollectionProps) { - watchedCollectionProps.remove(collection); - } - return null; - } - return v; - }); + collectionPropertiesZkStateReader.removeCollectionPropsWatcher(collection, watcher); } public static class ConfigData { @@ -2352,66 +2097,6 @@ private boolean setIfNewer(SolrZkClient.NodeData n) { } } - private void notifyPropsWatchers(String collection, Map properties) { - try { - collectionPropsNotifications.submit(new PropsNotification(collection, properties)); - } catch (RejectedExecutionException e) { - if (!closed) { - log.error("Couldn't run collection properties notifications for {}", collection, e); - } - } - } - - private class PropsNotification implements Runnable { - - private final String collection; - private final Map collectionProperties; - private final List watchers = new ArrayList<>(); - - private PropsNotification(String collection, Map collectionProperties) { - this.collection = collection; - this.collectionProperties = collectionProperties; - // guarantee delivery of notification regardless of what happens to collectionPropsObservers - // while we wait our turn in the executor by capturing the list on creation. - collectionPropsObservers.compute( - collection, - (k, v) -> { - if (v == null) return null; - watchers.addAll(v.stateWatchers); - return v; - }); - } - - @Override - public void run() { - for (CollectionPropsWatcher watcher : watchers) { - if (watcher.onStateChanged(collectionProperties)) { - removeCollectionPropsWatcher(collection, watcher); - } - } - } - } - - private class CacheCleaner implements Runnable { - @Override - public void run() { - while (!Thread.interrupted()) { - try { - Thread.sleep(60000); - } catch (InterruptedException e) { - // Executor shutdown will send us an interrupt - break; - } - watchedCollectionProps - .entrySet() - .removeIf( - entry -> - entry.getValue().cacheUntilNs < System.nanoTime() - && !collectionPropsObservers.containsKey(entry.getKey())); - } - } - } - /** * Helper class that acts as both a {@link DocCollectionWatcher} and a {@link LiveNodesListener} * while wraping and delegating to a {@link CollectionStateWatcher} diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DelegatingClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DelegatingClusterStateProvider.java index b8e7322828f..4177f0a017a 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DelegatingClusterStateProvider.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/DelegatingClusterStateProvider.java @@ -78,6 +78,15 @@ public String resolveSimpleAlias(String alias) throws IllegalArgumentException { } } + @Override + public Object getClusterProperty(String propertyName) { + if (delegate != null) { + return delegate.getClusterProperty(propertyName); + } else { + return null; + } + } + @Override public ClusterState getClusterState() { if (delegate != null) { diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java index d5e2d188a75..d9186057ef7 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/BaseHttpClusterStateProvider.java @@ -39,6 +39,7 @@ import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.PerReplicaStates; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.EnvUtils; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.Utils; @@ -55,7 +56,8 @@ public abstract class BaseHttpClusterStateProvider implements ClusterStateProvid volatile Map> aliasProperties; long aliasesTimestamp = 0; - private int cacheTimeout = 5; // the liveNodes and aliases cache will be invalidated after 5 secs + // the liveNodes and aliases cache will be invalidated after 5 secs + private int cacheTimeout = EnvUtils.getPropertyAsInteger("solr.solrj.cache.timeout.sec", 5); public void init(List solrUrls) throws Exception { for (String solrUrl : solrUrls) { @@ -84,6 +86,13 @@ public void init(List solrUrls) throws Exception { /** Create a SolrClient implementation that uses the specified Solr node URL */ protected abstract SolrClient getSolrClient(String baseUrl); + @Override + public DocCollection getCollection(String collection) { + // This change is to prevent BaseHttpCSP make a call to fetch the entire cluster state, as the + // default implementation calls getClusterState().getCollectionOrNull(name) + return getState(collection).get(); + } + @Override public ClusterState.CollectionRef getState(String collection) { for (String nodeName : liveNodes) { @@ -122,15 +131,9 @@ public ClusterState.CollectionRef getState(String collection) { private ClusterState fetchClusterState( SolrClient client, String collection, Map clusterProperties) throws SolrServerException, IOException, NotACollectionException { - ModifiableSolrParams params = new ModifiableSolrParams(); - if (collection != null) { - params.set("collection", collection); - } - params.set("action", "CLUSTERSTATUS"); - params.set("prs", "true"); - QueryRequest request = new QueryRequest(params); - request.setPath("/admin/collections"); - SimpleOrderedMap cluster = (SimpleOrderedMap) client.request(request).get("cluster"); + SimpleOrderedMap cluster = + submitClusterStateRequest(client, collection, ClusterStateRequestType.FETCH_COLLECTION); + Map collectionsMap; if (collection != null) { collectionsMap = @@ -149,10 +152,16 @@ private ClusterState fetchClusterState( } else { znodeVersion = -1; } - Set liveNodes = new HashSet<>((List) (cluster.get("live_nodes"))); - this.liveNodes = liveNodes; - liveNodesTimestamp = System.nanoTime(); - ClusterState cs = new ClusterState(liveNodes, new HashMap<>()); + + ClusterState cs = new ClusterState(this.liveNodes, new HashMap<>()); + List liveNodesList = (List) cluster.get("live_nodes"); + if (liveNodesList != null) { + Set liveNodes = new HashSet<>(liveNodesList); + this.liveNodes = liveNodes; + liveNodesTimestamp = System.nanoTime(); + cs = new ClusterState(liveNodes, new HashMap<>()); + } + for (Map.Entry e : collectionsMap.entrySet()) { @SuppressWarnings("rawtypes") Map m = (Map) e.getValue(); @@ -173,6 +182,30 @@ private ClusterState fetchClusterState( return cs; } + private SimpleOrderedMap submitClusterStateRequest( + SolrClient client, String collection, ClusterStateRequestType requestType) + throws SolrServerException, IOException { + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", "CLUSTERSTATUS"); + + if (requestType == ClusterStateRequestType.FETCH_COLLECTION && collection != null) { + params.set("collection", collection); + } else if (requestType == ClusterStateRequestType.FETCH_LIVE_NODES) { + params.set("liveNodes", "true"); + } else if (requestType == ClusterStateRequestType.FETCH_CLUSTER_PROP) { + params.set("clusterProperties", "true"); + } else if (requestType == ClusterStateRequestType.FETCH_NODE_ROLES) { + params.set("roles", "true"); + } + + params.set("includeAll", "false"); + params.set("prs", "true"); + QueryRequest request = new QueryRequest(params); + request.setPath("/admin/collections"); + return (SimpleOrderedMap) client.request(request).get("cluster"); + } + @SuppressWarnings({"rawtypes", "unchecked"}) private DocCollection fillPrs( int znodeVersion, Map.Entry e, Instant creationTime, Map m) { @@ -228,12 +261,10 @@ > getCacheTimeout()) { } @SuppressWarnings({"rawtypes", "unchecked"}) - private static Set fetchLiveNodes(SolrClient client) throws Exception { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", "CLUSTERSTATUS"); - QueryRequest request = new QueryRequest(params); - request.setPath("/admin/collections"); - NamedList cluster = (SimpleOrderedMap) client.request(request).get("cluster"); + private Set fetchLiveNodes(SolrClient client) throws Exception { + + SimpleOrderedMap cluster = + submitClusterStateRequest(client, null, ClusterStateRequestType.FETCH_LIVE_NODES); return (Set) new HashSet((List) (cluster.get("live_nodes"))); } @@ -335,21 +366,18 @@ public ClusterState getClusterState() { + " solrUrl(s) or zkHost(s)."); } + @SuppressWarnings("unchecked") @Override public Map getClusterProperties() { + // Map clusterPropertiesMap = new HashMap<>(); for (String nodeName : liveNodes) { String baseUrl = Utils.getBaseUrlForNodeName(nodeName, urlScheme); try (SolrClient client = getSolrClient(baseUrl)) { - Map clusterProperties = new HashMap<>(); - fetchClusterState(client, null, clusterProperties); - return clusterProperties; + SimpleOrderedMap cluster = + submitClusterStateRequest(client, null, ClusterStateRequestType.FETCH_CLUSTER_PROP); + return (Map) cluster.get("properties"); } catch (SolrServerException | BaseHttpSolrClient.RemoteSolrException | IOException e) { log.warn("Attempt to fetch cluster state from {} failed.", baseUrl, e); - } catch (NotACollectionException e) { - // not possible! (we passed in null for collection so it can't be an alias) - throw new RuntimeException( - "null should never cause NotACollectionException in " - + "fetchClusterState() Please report this as a bug!"); } } throw new RuntimeException( @@ -385,10 +413,6 @@ public int getCacheTimeout() { return cacheTimeout; } - public void setCacheTimeout(int cacheTimeout) { - this.cacheTimeout = cacheTimeout; - } - // This exception is not meant to escape this class it should be caught and wrapped. private static class NotACollectionException extends Exception {} @@ -399,4 +423,11 @@ public String getQuorumHosts() { } return String.join(",", this.liveNodes); } + + private enum ClusterStateRequestType { + FETCH_LIVE_NODES, + FETCH_CLUSTER_PROP, + FETCH_NODE_ROLES, + FETCH_COLLECTION + } } diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java index 223fef6ab23..3ac950d0417 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/CloudSolrClient.java @@ -362,8 +362,9 @@ private NamedList directUpdate(AbstractUpdateRequest request, String col // Check to see if the collection is an alias. Updates to multi-collection aliases are ok as // long as they are routed aliases - List aliasedCollections = getClusterStateProvider().resolveAlias(collection); - if (getClusterStateProvider().isRoutedAlias(collection) || aliasedCollections.size() == 1) { + List aliasedCollections = + new ArrayList<>(resolveAliases(Collections.singletonList(collection))); + if (aliasedCollections.size() == 1 || getClusterStateProvider().isRoutedAlias(collection)) { collection = aliasedCollections.get(0); // pick 1st (consistent with HttpSolrCall behavior) } else { throw new SolrException( @@ -1149,7 +1150,7 @@ private Set resolveAliases(List inputCollections) { } LinkedHashSet uniqueNames = new LinkedHashSet<>(); // consistent ordering for (String collectionName : inputCollections) { - if (getClusterStateProvider().getState(collectionName) == null) { + if (getDocCollection(collectionName, -1) == null) { // perhaps it's an alias uniqueNames.addAll(getClusterStateProvider().resolveAlias(collectionName)); } else { @@ -1208,15 +1209,6 @@ protected DocCollection getDocCollection(String collection, Integer expectedVers if (expectedVersion <= col.getZNodeVersion() && !cacheEntry.shouldRetry()) return col; } - ClusterState.CollectionRef ref = getCollectionRef(collection); - if (ref == null) { - // no such collection exists - return null; - } - if (!ref.isLazilyLoaded()) { - // it is readily available just return it - return ref.get(); - } Object[] locks = this.locks; int lockId = Math.abs(Hash.murmurhash3_x86_32(collection, 0, collection.length(), 0) % locks.length); @@ -1228,6 +1220,11 @@ protected DocCollection getDocCollection(String collection, Integer expectedVers if (col != null) { if (expectedVersion <= col.getZNodeVersion() && !cacheEntry.shouldRetry()) return col; } + ClusterState.CollectionRef ref = getCollectionRef(collection); + if (ref == null) { + // no such collection exists + return null; + } // We are going to fetch a new version // we MUST try to get a new version DocCollection fetchedCol = ref.get(); // this is a call to ZK diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java index 81bb885c38b..b6afac3114f 100644 --- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java +++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java @@ -91,7 +91,11 @@ default boolean isRoutedAlias(String alias) { .anyMatch(e -> e.getKey().startsWith(CollectionAdminParams.ROUTER_PREFIX)); } - /** Obtain the current cluster state. */ + /** + * Obtain the current cluster state. WARNING: This method is quite expensive as it involves + * fetching remote information. Use with caution and be aware of the potential performance + * implications. + */ ClusterState getClusterState(); default DocCollection getCollection(String name) throws IOException { @@ -108,15 +112,13 @@ default DocCollection getCollection(String name) throws IOException { /** Obtain a cluster property, or the default value if it doesn't exist. */ default T getClusterProperty(String key, T defaultValue) { @SuppressWarnings({"unchecked"}) - T value = (T) getClusterProperties().get(key); + T value = (T) getClusterProperty(key); if (value == null) return defaultValue; return value; } /** Obtain a cluster property, or null if it doesn't exist. */ - default Object getClusterProperty(String propertyName) { - return getClusterProperties().get(propertyName); - } + Object getClusterProperty(String propertyName); /** Get the collection-specific policy */ String getPolicyNameByCollection(String coll); diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java index 9fca76238b3..4e94b1d97c6 100644 --- a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java +++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java @@ -167,6 +167,9 @@ public interface CommonParams { /** Timeout value in milliseconds. If not set, or the value is < 0, there is no timeout. */ String TIME_ALLOWED = "timeAllowed"; + /** Whether or not the search may use the multi-threaded logic */ + String MULTI_THREADED = "multiThreaded"; + /** * Maximum query CPU usage value in milliseconds. If not set, or the value is < 0, there is no * timeout. diff --git a/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java b/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java index a0444ea5a53..b21a968adef 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/ExecutorUtil.java @@ -202,14 +202,15 @@ public static ExecutorService newMDCAwareFixedThreadPool( } public static ExecutorService newMDCAwareFixedThreadPool( - int nThreads, int queueCapacity, ThreadFactory threadFactory) { + int nThreads, int queueCapacity, ThreadFactory threadFactory, Runnable beforeExecute) { return new MDCAwareThreadPoolExecutor( nThreads, nThreads, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>(queueCapacity), - threadFactory); + threadFactory, + beforeExecute); } /** @@ -236,29 +237,48 @@ public static ExecutorService newMDCAwareCachedThreadPool(String name) { return newMDCAwareCachedThreadPool(new SolrNamedThreadFactory(name)); } - /** See {@link java.util.concurrent.Executors#newCachedThreadPool(ThreadFactory)} */ + /** + * Create a new pool of threads, with no limit for the number of threads. The pool has no task + * queue. Each submitted task is executed immediately, either by reusing an existing thread if one + * is available, or by starting a new thread. Unused threads will be closed after 60 seconds. + */ public static ExecutorService newMDCAwareCachedThreadPool(ThreadFactory threadFactory) { return new MDCAwareThreadPoolExecutor( 0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<>(), threadFactory); } + /** + * Create a new pool of threads. Threads are created for new work if there is room to do so up to + * {@code maxThreads}. Beyond that, the queue is used up to {@code queueCapacity}. Beyond that, + * work is rejected with an exception. Unused threads will be closed after 60 seconds. + */ public static ExecutorService newMDCAwareCachedThreadPool( int maxThreads, int queueCapacity, ThreadFactory threadFactory) { - return new MDCAwareThreadPoolExecutor( - 0, - maxThreads, - 60L, - TimeUnit.SECONDS, - new LinkedBlockingQueue<>(queueCapacity), - threadFactory); + // Create an executor with same value of core size and max total size. With an unbounded queue, + // the ThreadPoolExecutor ignores the configured max value and only considers core pool size. + // Since we allow core threads to die when idle for too long, this ends in having a pool with + // lazily-initialized and cached threads. + MDCAwareThreadPoolExecutor executor = + new MDCAwareThreadPoolExecutor( + maxThreads, + maxThreads, + 60L, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(queueCapacity), + threadFactory); + // Allow core threads to die + executor.allowCoreThreadTimeOut(true); + return executor; } @SuppressForbidden(reason = "class customizes ThreadPoolExecutor so it can be used instead") public static class MDCAwareThreadPoolExecutor extends ThreadPoolExecutor { private static final int MAX_THREAD_NAME_LEN = 512; + public static final Runnable NOOP = () -> {}; private final boolean enableSubmitterStackTrace; + private final Runnable beforeExecuteTask; public MDCAwareThreadPoolExecutor( int corePoolSize, @@ -270,6 +290,7 @@ public MDCAwareThreadPoolExecutor( RejectedExecutionHandler handler) { super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler); this.enableSubmitterStackTrace = true; + this.beforeExecuteTask = NOOP; } public MDCAwareThreadPoolExecutor( @@ -280,6 +301,7 @@ public MDCAwareThreadPoolExecutor( BlockingQueue workQueue) { super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue); this.enableSubmitterStackTrace = true; + this.beforeExecuteTask = NOOP; } public MDCAwareThreadPoolExecutor( @@ -289,7 +311,8 @@ public MDCAwareThreadPoolExecutor( TimeUnit unit, BlockingQueue workQueue, ThreadFactory threadFactory) { - this(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, true); + this( + corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, true, NOOP); } public MDCAwareThreadPoolExecutor( @@ -299,9 +322,30 @@ public MDCAwareThreadPoolExecutor( TimeUnit unit, BlockingQueue workQueue, ThreadFactory threadFactory, - boolean enableSubmitterStackTrace) { + Runnable beforeExecuteTask) { + this( + corePoolSize, + maximumPoolSize, + keepAliveTime, + unit, + workQueue, + threadFactory, + true, + beforeExecuteTask); + } + + public MDCAwareThreadPoolExecutor( + int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + BlockingQueue workQueue, + ThreadFactory threadFactory, + boolean enableSubmitterStackTrace, + Runnable beforeExecuteTask) { super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory); this.enableSubmitterStackTrace = enableSubmitterStackTrace; + this.beforeExecuteTask = beforeExecuteTask; } public MDCAwareThreadPoolExecutor( @@ -313,6 +357,37 @@ public MDCAwareThreadPoolExecutor( RejectedExecutionHandler handler) { super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, handler); this.enableSubmitterStackTrace = true; + this.beforeExecuteTask = NOOP; + } + + public MDCAwareThreadPoolExecutor( + int corePoolSize, + int maximumPoolSize, + int keepAliveTime, + TimeUnit timeUnit, + BlockingQueue blockingQueue, + SolrNamedThreadFactory httpShardExecutor, + boolean enableSubmitterStackTrace) { + super( + corePoolSize, maximumPoolSize, keepAliveTime, timeUnit, blockingQueue, httpShardExecutor); + this.enableSubmitterStackTrace = enableSubmitterStackTrace; + this.beforeExecuteTask = NOOP; + } + + public MDCAwareThreadPoolExecutor( + int i, + int maxValue, + long l, + TimeUnit timeUnit, + BlockingQueue es, + SolrNamedThreadFactory testExecutor, + boolean b) { + this(i, maxValue, l, timeUnit, es, testExecutor, b, NOOP); + } + + @Override + protected void beforeExecute(Thread t, Runnable r) { + this.beforeExecuteTask.run(); } @Override diff --git a/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-replication1.xml b/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-replication1.xml index 691ada4ba64..e0d04c62517 100644 --- a/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-replication1.xml +++ b/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-replication1.xml @@ -27,7 +27,7 @@ - + diff --git a/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-sql.xml b/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-sql.xml index 08dd32abf9d..f557a80e64d 100644 --- a/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-sql.xml +++ b/solr/solrj/src/test-files/solrj/solr/collection1/conf/schema-sql.xml @@ -45,12 +45,12 @@ - + - - - - + + + + - - - - - - - - - + + + + + + + + + diff --git a/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig-follower1.xml b/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig-follower1.xml index ab2773d6eb4..43ff0410a8c 100644 --- a/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig-follower1.xml +++ b/solr/solrj/src/test-files/solrj/solr/collection1/conf/solrconfig-follower1.xml @@ -23,7 +23,7 @@ ${useCompoundFile:false} ${solr.data.dir:} - + diff --git a/solr/solrj/src/test-files/solrj/solr/configsets/ml/conf/schema.xml b/solr/solrj/src/test-files/solrj/solr/configsets/ml/conf/schema.xml index 544edf8ae04..be212b86919 100644 --- a/solr/solrj/src/test-files/solrj/solr/configsets/ml/conf/schema.xml +++ b/solr/solrj/src/test-files/solrj/solr/configsets/ml/conf/schema.xml @@ -29,13 +29,13 @@ - - + + - - - - + + + + diff --git a/solr/solrj/src/test-files/solrj/solr/configsets/shared/conf/schema.xml b/solr/solrj/src/test-files/solrj/solr/configsets/shared/conf/schema.xml index 4aad5690101..f12c5e09038 100644 --- a/solr/solrj/src/test-files/solrj/solr/configsets/shared/conf/schema.xml +++ b/solr/solrj/src/test-files/solrj/solr/configsets/shared/conf/schema.xml @@ -42,7 +42,7 @@ - + diff --git a/solr/solrj/src/test-files/solrj/solr/configsets/shared/conf/solrconfig.xml b/solr/solrj/src/test-files/solrj/solr/configsets/shared/conf/solrconfig.xml index 11ea52b8da2..fb89854b08c 100644 --- a/solr/solrj/src/test-files/solrj/solr/configsets/shared/conf/solrconfig.xml +++ b/solr/solrj/src/test-files/solrj/solr/configsets/shared/conf/solrconfig.xml @@ -26,7 +26,7 @@ ${useCompoundFile:false} ${tempDir}/data/${l10n:}-${version:} - + diff --git a/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/schema.xml b/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/schema.xml index 5784fd0ffe4..5a202baa2b8 100644 --- a/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/schema.xml +++ b/solr/solrj/src/test-files/solrj/solr/configsets/streaming/conf/schema.xml @@ -43,12 +43,12 @@ - + - - - - + + + + diff --git a/solr/solrj/src/test-files/solrj/solr/configsets/tracking-updates/conf/schema.xml b/solr/solrj/src/test-files/solrj/solr/configsets/tracking-updates/conf/schema.xml index b539459ad5e..fc23706512e 100644 --- a/solr/solrj/src/test-files/solrj/solr/configsets/tracking-updates/conf/schema.xml +++ b/solr/solrj/src/test-files/solrj/solr/configsets/tracking-updates/conf/schema.xml @@ -17,8 +17,8 @@ --> - - + + diff --git a/solr/solrj/src/test-files/solrj/solr/multicore/core0/conf/schema.xml b/solr/solrj/src/test-files/solrj/solr/multicore/core0/conf/schema.xml index 8b3a3933996..8dddc57a24e 100644 --- a/solr/solrj/src/test-files/solrj/solr/multicore/core0/conf/schema.xml +++ b/solr/solrj/src/test-files/solrj/solr/multicore/core0/conf/schema.xml @@ -19,7 +19,7 @@ - + diff --git a/solr/solrj/src/test-files/solrj/solr/multicore/core1/conf/schema.xml b/solr/solrj/src/test-files/solrj/solr/multicore/core1/conf/schema.xml index fd1cac0374f..079a2683edb 100644 --- a/solr/solrj/src/test-files/solrj/solr/multicore/core1/conf/schema.xml +++ b/solr/solrj/src/test-files/solrj/solr/multicore/core1/conf/schema.xml @@ -19,7 +19,7 @@ - + diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java index 474ebb8a177..2a0693c2360 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java @@ -70,6 +70,9 @@ import org.apache.solr.handler.admin.CollectionsHandler; import org.apache.solr.handler.admin.ConfigSetsHandler; import org.apache.solr.handler.admin.CoreAdminHandler; +import org.apache.solr.servlet.HttpSolrCall; +import org.apache.solr.util.LogLevel; +import org.apache.solr.util.LogListener; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -114,6 +117,7 @@ public static void setupCluster() throws Exception { @AfterClass public static void tearDownAfterClass() throws Exception { + if (httpBasedCloudSolrClient != null) { try { httpBasedCloudSolrClient.close(); @@ -246,6 +250,58 @@ public void testAliasHandling() throws Exception { 2, client.query(null, paramsWithMixedCollectionAndAlias).getResults().getNumFound()); } + @Test + @LogLevel("org.apache.solr.servlet.HttpSolrCall=DEBUG") + public void testHttpCspPerf() throws Exception { + + String collectionName = "HTTPCSPTEST"; + CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(collectionName, 2, 2); + + try (LogListener adminLogs = LogListener.info(HttpSolrCall.class).substring("[admin]"); + CloudSolrClient solrClient = createHttpCSPBasedCloudSolrClient(); ) { + + assertEquals(1, adminLogs.getCount()); + assertTrue( + adminLogs + .pollMessage() + .contains( + "path=/admin/collections params={prs=true&liveNodes=true&action" + + "=CLUSTERSTATUS&includeAll=false")); + + SolrInputDocument doc = new SolrInputDocument("id", "1", "title_s", "my doc"); + solrClient.add(collectionName, doc); + + // getCount seems to return a cumulative count, but add() results in only 1 additional admin + // request to fetch CLUSTERSTATUS for the collection + assertEquals(2, adminLogs.getCount()); + assertTrue( + adminLogs + .pollMessage() + .contains( + "path=/admin/collections " + + "params={prs=true&action=CLUSTERSTATUS&includeAll=false")); + + solrClient.commit(collectionName); + // No additional admin requests sent + assertEquals(2, adminLogs.getCount()); + + for (int i = 0; i < 3; i++) { + assertEquals( + 1, solrClient.query(collectionName, params("q", "*:*")).getResults().getNumFound()); + // No additional admin requests sent + assertEquals(2, adminLogs.getCount()); + } + } + } + + private CloudSolrClient createHttpCSPBasedCloudSolrClient() { + final List solrUrls = new ArrayList<>(); + solrUrls.add(cluster.getJettySolrRunner(0).getBaseUrl().toString()); + return new CloudHttp2SolrClient.Builder(solrUrls).build(); + } + @Test public void testRouting() throws Exception { CollectionAdminRequest.createCollection("routing_collection", "conf", 2, 1) diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClusterStateSSLTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClusterStateSSLTest.java index db385e26b83..1242cde9945 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClusterStateSSLTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/HttpClusterStateSSLTest.java @@ -23,7 +23,6 @@ import java.util.Map; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.util.Utils; @@ -88,7 +87,7 @@ public void testHttpClusterStateWithSSL() throws Exception { new CloudSolrClient.Builder(Collections.singletonList(url0.toExternalForm())).build()) { ClusterStateProvider csp = httpBasedCloudSolrClient.getClusterStateProvider(); assertTrue(csp instanceof Http2ClusterStateProvider); - verifyUrlSchemeInClusterState(csp.getClusterState(), collectionId, expectedReplicas); + verifyUrlSchemeInClusterState(csp.getCollection(collectionId), expectedReplicas); } // http2 @@ -97,20 +96,19 @@ public void testHttpClusterStateWithSSL() throws Exception { .build()) { ClusterStateProvider csp = http2BasedClient.getClusterStateProvider(); assertTrue(csp instanceof Http2ClusterStateProvider); - verifyUrlSchemeInClusterState(csp.getClusterState(), collectionId, expectedReplicas); + verifyUrlSchemeInClusterState(csp.getCollection(collectionId), expectedReplicas); } // Zk cluster state now ClusterStateProvider csp = cluster.getSolrClient().getClusterStateProvider(); assertTrue(csp instanceof ZkClientClusterStateProvider); - verifyUrlSchemeInClusterState(csp.getClusterState(), collectionId, expectedReplicas); + verifyUrlSchemeInClusterState(csp.getCollection(collectionId), expectedReplicas); } private void verifyUrlSchemeInClusterState( - final ClusterState cs, final String collectionId, final int expectedReplicas) { - DocCollection dc = cs.getCollection(collectionId); - assertNotNull(dc); - List replicas = dc.getReplicas(); + final DocCollection collection, final int expectedReplicas) { + assertNotNull(collection); + List replicas = collection.getReplicas(); assertNotNull(replicas); assertEquals(expectedReplicas, replicas.size()); for (Replica r : replicas) { diff --git a/solr/solrj/src/test/org/apache/solr/common/util/ExecutorUtilTest.java b/solr/solrj/src/test/org/apache/solr/common/util/ExecutorUtilTest.java index a9df98a296b..f797ef547a3 100644 --- a/solr/solrj/src/test/org/apache/solr/common/util/ExecutorUtilTest.java +++ b/solr/solrj/src/test/org/apache/solr/common/util/ExecutorUtilTest.java @@ -28,9 +28,12 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import org.apache.lucene.util.NamedThreadFactory; import org.apache.solr.SolrTestCase; +import org.apache.solr.logging.MDCLoggingContext; import org.apache.solr.util.TimeOut; import org.junit.Test; +import org.slf4j.MDC; public class ExecutorUtilTest extends SolrTestCase { @@ -108,6 +111,57 @@ public void testExecutorUtilAwaitsTerminationWhenTaskRespectsInterupt() throws E } } + @Test + public void testCMDCAwareCachedThreadPool() throws Exception { + // 5 threads max, unbounded queue + ExecutorService executor = + ExecutorUtil.newMDCAwareCachedThreadPool( + 5, Integer.MAX_VALUE, new NamedThreadFactory("test")); + + AtomicInteger concurrentTasks = new AtomicInteger(); + AtomicInteger maxConcurrentTasks = new AtomicInteger(); + int taskCount = 5 + random().nextInt(100); + CountDownLatch latch = new CountDownLatch(5); + List> futures = new ArrayList<>(); + + for (int i = 0; i < taskCount; i++) { + String core = "id_" + random().nextLong(); + + Callable task = + () -> { + // ensure we never have too many concurrent tasks + int concurrent = concurrentTasks.incrementAndGet(); + assertTrue(concurrent <= 5); + maxConcurrentTasks.getAndAccumulate(concurrent, Math::max); + + // assert MDC context is copied from the parent thread that submitted the task + assertEquals(core, MDC.get("core")); + + // The first 4 tasks to be executed will wait on the latch, and the 5th will + // release all the threads. + latch.countDown(); + latch.await(1, TimeUnit.SECONDS); + concurrentTasks.decrementAndGet(); + return null; + }; + + MDCLoggingContext.setCoreName(core); + futures.add(executor.submit(task)); + } + + ExecutorUtil.shutdownAndAwaitTermination(executor); + + for (Future future : futures) { + // Throws an exception (and make the test fail) if an assertion failed + // in the subtask + future.get(); + } + + // assert the pool was actually multithreaded. Since we submitted many tasks, + // all the threads should have been started + assertEquals(5, maxConcurrentTasks.get()); + } + private static final class Worker implements Callable { // how we communiate out to our caller private final CountDownLatch taskStartedLatch = new CountDownLatch(1); diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java index 55ad19495e0..fc723f60aa5 100644 --- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java +++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java @@ -769,10 +769,6 @@ public static void initCore() throws Exception { log.info("####initCore"); ignoreException("ignore_exception"); - factoryProp = System.getProperty("solr.directoryFactory"); - if (factoryProp == null) { - System.setProperty("solr.directoryFactory", "solr.RAMDirectoryFactory"); - } // other methods like starting a jetty instance need these too System.setProperty("solr.test.sys.prop1", "propone"); diff --git a/solr/test-framework/src/java/org/apache/solr/schema/SortableBinaryField.java b/solr/test-framework/src/java/org/apache/solr/schema/SortableBinaryField.java index 5c6d31c43ba..28edf725d14 100644 --- a/solr/test-framework/src/java/org/apache/solr/schema/SortableBinaryField.java +++ b/solr/test-framework/src/java/org/apache/solr/schema/SortableBinaryField.java @@ -37,7 +37,7 @@ protected void checkSupportsDocValues() { // we support DocValues } @Override - protected boolean doesTypeSupportDocValues() { + protected boolean enableDocValuesByDefault() { return true; } diff --git a/solr/test-framework/src/test-files/solr/collection1/conf/schema.xml b/solr/test-framework/src/test-files/solr/collection1/conf/schema.xml index f18a9b7490f..e65df97ed61 100644 --- a/solr/test-framework/src/test-files/solr/collection1/conf/schema.xml +++ b/solr/test-framework/src/test-files/solr/collection1/conf/schema.xml @@ -33,16 +33,16 @@ behavior of the fieldType. --> - - - - + + + + - - - - + + + + @@ -112,8 +112,8 @@ - - + + @@ -545,13 +545,13 @@ - + - + @@ -791,7 +791,7 @@ - + - + 1000000 2000000 3000000