Skip to content

Commit

Permalink
SOLR-17381: Make CLUSTERSTATUS configurable (#2599)
Browse files Browse the repository at this point in the history 
Make CLUSTERSTATUS configurable to improve performance by allowing retrieval of specific information with these boolean parameters: liveNodes, clusterProperties, aliases, roles, and includeAll

Updated SolrJ HttpClusterStateProvider (state from Solr nodes) to use these parameters so that only required information is fetched and returned.
  • Loading branch information
@aparnasuresh85
aparnasuresh85 authored Aug 9, 2024
1 parent 5abeb4b commit c7af8b8
Show file tree
Hide file tree
Showing 8 changed files with 219 additions and 113 deletions.
3 changes: 3 additions & 0 deletions solr/CHANGES.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,9 @@ Optimizations
* SOLR-17102: The VersionBucket indexing lock mechanism was replaced with something just as fast yet
that which consumes almost no memory, saving 1MB of memory per SolrCore. (David Smiley)

* SOLR-17381: Make CLUSTERSTATUS request configurable to improve performance by allowing retrieval of specific information,
reducing unnecessary data fetching. (Aparna Suresh, David Smiley)

* SOLR-17396: Reduce thread contention in ZkStateReader.getCollectionProperties(). (Aparna Suresh, David Smiley, Paul McArthur)

Bug Fixes
Expand Down
110 changes: 73 additions & 37 deletions solr/core/src/java/org/apache/solr/handler/admin/ClusterStatus.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,19 +34,24 @@
import org.apache.solr.common.cloud.PerReplicaStates;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.Utils;
import org.apache.zookeeper.KeeperException;

public class ClusterStatus {
private final ZkStateReader zkStateReader;
private final ZkNodeProps message;
private final SolrParams solrParams;
private final String collection; // maybe null

public static final String INCLUDE_ALL = "includeAll";
public static final String LIVENODES_PROP = "liveNodes";
public static final String CLUSTER_PROP = "clusterProperties";
public static final String ALIASES_PROP = "aliases";

/** Shard / collection health state. */
public enum Health {
/** All replicas up, leader exists. */
Expand Down Expand Up @@ -89,16 +94,72 @@ public static Health combine(Collection<Health> states) {
}
}

public ClusterStatus(ZkStateReader zkStateReader, ZkNodeProps props) {
public ClusterStatus(ZkStateReader zkStateReader, SolrParams params) {
this.zkStateReader = zkStateReader;
this.message = props;
collection = props.getStr(ZkStateReader.COLLECTION_PROP);
this.solrParams = params;
collection = params.get(ZkStateReader.COLLECTION_PROP);
}

public void getClusterStatus(NamedList<Object> results)
throws KeeperException, InterruptedException {
NamedList<Object> clusterStatus = new SimpleOrderedMap<>();

boolean includeAll = solrParams.getBool(INCLUDE_ALL, true);
boolean withLiveNodes = solrParams.getBool(LIVENODES_PROP, includeAll);
boolean withClusterProperties = solrParams.getBool(CLUSTER_PROP, includeAll);
boolean withRoles = solrParams.getBool(ZkStateReader.ROLES_PROP, includeAll);
boolean withCollection = includeAll || (collection != null);
boolean withAliases = solrParams.getBool(ALIASES_PROP, includeAll);

List<String> liveNodes = null;
if (withLiveNodes || collection != null) {
liveNodes =
zkStateReader.getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true);
// add live_nodes
if (withLiveNodes) clusterStatus.add("live_nodes", liveNodes);
}

Aliases aliases = null;
if (withCollection || withAliases) {
aliases = zkStateReader.getAliases();
}

if (withCollection) {
assert liveNodes != null;
fetchClusterStatusForCollOrAlias(clusterStatus, liveNodes, aliases);
}

if (withAliases) {
addAliasMap(aliases, clusterStatus);
}

if (withClusterProperties) {
Map<String, Object> clusterProps = zkStateReader.getClusterProperties();
if (clusterProps == null) {
clusterProps = Collections.emptyMap();
}
clusterStatus.add("properties", clusterProps);
}

// add the roles map
if (withRoles) {
Map<?, ?> roles = Collections.emptyMap();
if (zkStateReader.getZkClient().exists(ZkStateReader.ROLES, true)) {
roles =
(Map<?, ?>)
Utils.fromJSON(
zkStateReader.getZkClient().getData(ZkStateReader.ROLES, null, null, true));
}
clusterStatus.add("roles", roles);
}

results.add("cluster", clusterStatus);
}

private void fetchClusterStatusForCollOrAlias(
NamedList<Object> clusterStatus, List<String> liveNodes, Aliases aliases) {

// read aliases
Aliases aliases = zkStateReader.getAliases();
Map<String, List<String>> collectionVsAliases = new HashMap<>();
Map<String, List<String>> aliasVsCollections = aliases.getCollectionAliasListMap();
for (Map.Entry<String, List<String>> entry : aliasVsCollections.entrySet()) {
Expand All @@ -112,18 +173,10 @@ public void getClusterStatus(NamedList<Object> results)
}
}

Map<?, ?> roles = null;
if (zkStateReader.getZkClient().exists(ZkStateReader.ROLES, true)) {
roles =
(Map<?, ?>)
Utils.fromJSON(
zkStateReader.getZkClient().getData(ZkStateReader.ROLES, null, null, true));
}

ClusterState clusterState = zkStateReader.getClusterState();

String routeKey = message.getStr(ShardParams._ROUTE_);
String shard = message.getStr(ZkStateReader.SHARD_ID_PROP);
String routeKey = solrParams.get(ShardParams._ROUTE_);
String shard = solrParams.get(ZkStateReader.SHARD_ID_PROP);

Map<String, DocCollection> collectionsMap = null;
if (collection == null) {
Expand All @@ -139,6 +192,7 @@ public void getClusterStatus(NamedList<Object> results)
if (didNotFindCollection && isAlias) {
// In this case this.collection is an alias name not a collection
// get all collections and filter out collections not in the alias
// clusterState.getCollectionsMap() should be replaced with an inexpensive call
collectionsMap =
clusterState.getCollectionsMap().entrySet().stream()
.filter((entry) -> aliasVsCollections.get(collection).contains(entry.getKey()))
Expand Down Expand Up @@ -191,43 +245,25 @@ public void getClusterStatus(NamedList<Object> results)
}
String configName = clusterStateCollection.getConfigName();
collectionStatus.put("configName", configName);
if (message.getBool("prs", false) && clusterStateCollection.isPerReplicaState()) {
if (solrParams.getBool("prs", false) && clusterStateCollection.isPerReplicaState()) {
PerReplicaStates prs = clusterStateCollection.getPerReplicaStates();
collectionStatus.put("PRS", prs);
}
collectionProps.add(name, collectionStatus);
}

List<String> liveNodes =
zkStateReader.getZkClient().getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true);

// now we need to walk the collectionProps tree to cross-check replica state with live nodes
crossCheckReplicaStateWithLiveNodes(liveNodes, collectionProps);

NamedList<Object> clusterStatus = new SimpleOrderedMap<>();
clusterStatus.add("collections", collectionProps);
}

// read cluster properties
Map<String, Object> clusterProps = zkStateReader.getClusterProperties();
if (clusterProps != null && !clusterProps.isEmpty()) {
clusterStatus.add("properties", clusterProps);
}

private void addAliasMap(Aliases aliases, NamedList<Object> clusterStatus) {
// add the alias map too
Map<String, String> collectionAliasMap = aliases.getCollectionAliasMap(); // comma delim
if (!collectionAliasMap.isEmpty()) {
clusterStatus.add("aliases", collectionAliasMap);
}

// add the roles map
if (roles != null) {
clusterStatus.add("roles", roles);
}

// add live_nodes
clusterStatus.add("live_nodes", liveNodes);

results.add("cluster", clusterStatus);
}

/**
Expand Down
6 changes: 1 addition & 5 deletions solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,6 @@
import static org.apache.solr.common.params.CommonParams.VALUE_LONG;
import static org.apache.solr.common.params.CoreAdminParams.BACKUP_LOCATION;
import static org.apache.solr.common.params.CoreAdminParams.BACKUP_REPOSITORY;
import static org.apache.solr.common.params.ShardParams._ROUTE_;
import static org.apache.solr.common.util.StrUtils.formatString;

import java.lang.invoke.MethodHandles;
Expand Down Expand Up @@ -975,10 +974,7 @@ public Map<String, Object> execute(
CLUSTERSTATUS_OP(
CLUSTERSTATUS,
(req, rsp, h) -> {
Map<String, Object> all =
copy(req.getParams(), null, COLLECTION_PROP, SHARD_ID_PROP, _ROUTE_, "prs");
new ClusterStatus(
h.coreContainer.getZkController().getZkStateReader(), new ZkNodeProps(all))
new ClusterStatus(h.coreContainer.getZkController().getZkStateReader(), req.getParams())
.getClusterStatus(rsp.getValues());
return null;
}),
Expand Down
45 changes: 45 additions & 0 deletions solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,51 @@ Multiple shard names can be specified as a comma-separated list.
+
This can be used if you need the details of the shard where a particular document belongs to and you don't know which shard it falls under.

`aliases`::
+
[%autowidth,frame=none]
|===
|Optional |Default: will default to the default value of `includeAll` parameter specified below
|===
+

`liveNodes`::
+
[%autowidth,frame=none]
|===
|Optional |Default: will default to the default value of `includeAll` parameter specified below
|===
+
If set to true, returns the status of live nodes in the cluster.

`clusterProperties`::
+
[%autowidth,frame=none]
|===
|Optional |Default: will default to the default value of `includeAll` parameter specified below
|===
+
If set to true, returns the properties of the cluster.

`roles`::
+
[%autowidth,frame=none]
|===
|Optional |Default: will default to the default value of `includeAll` parameter specified below
|===
+
If set to true, returns the roles within the cluster.

`includeAll`::
+
[%autowidth,frame=none]
|===
|Optional |Default: true
|===
+
If set to `true`, returns all information pertaining to live nodes, collections, aliases, cluster properties, roles, etc.
If set to `false`, the information returned is based on the other specified parameters.

=== CLUSTERSTATUS Response

The response will include the status of the request and the status of the cluster.
Expand Down
Loading

0 comments on commit c7af8b8

Please sign in to comment.