From 953e1925a7db27eb8e3c79652e62af94d9463446 Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Wed, 21 Jun 2023 13:11:17 -0700 Subject: [PATCH 1/8] [Segment Replication] Verify segment replication stats in bwc test Signed-off-by: Suraj Singh --- .../java/org/opensearch/upgrades/IndexingIT.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java index a758b8e4ccd72..c8383c7333901 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java @@ -39,6 +39,8 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Booleans; import org.opensearch.common.settings.Settings; +import org.opensearch.index.codec.CodecService; +import org.opensearch.index.engine.EngineConfig; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.test.rest.yaml.ObjectPath; @@ -145,6 +147,12 @@ private void waitForClusterHealthWithNoShardMigration(String indexName, String s client().performRequest(waitForStatus); } + private void verifySegmentStats(String indexName) throws IOException { + Request segrepStatsRequest = new Request("GET", "/_cat/segment_replication"); + Response searchTestIndexResponse = client().performRequest(segrepStatsRequest); + logger.info("--> searchTestIndexResponse {}", searchTestIndexResponse.toString()); + } + public void testIndexing() throws IOException, ParseException { switch (CLUSTER_TYPE) { case OLD: @@ -251,6 +259,10 @@ public void testIndexingWithSegRep() throws Exception { .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), shardCount) .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), replicaCount) .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT) + .put( + EngineConfig.INDEX_CODEC_SETTING.getKey(), + randomFrom(CodecService.DEFAULT_CODEC, CodecService.BEST_COMPRESSION_CODEC, CodecService.LUCENE_DEFAULT_CODEC) + ) .put(INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "100ms"); createIndex(indexName, settings.build()); waitForClusterHealthWithNoShardMigration(indexName, "green"); @@ -286,6 +298,7 @@ public void testIndexingWithSegRep() throws Exception { } waitForSearchableDocs(indexName, shardCount); + verifySegmentStats(indexName); assertCount(indexName, expectedCount); if (CLUSTER_TYPE != ClusterType.OLD) { From 083d298108db8dafff6d8b271f7dd3341e2bef0f Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Wed, 21 Jun 2023 14:07:28 -0700 Subject: [PATCH 2/8] Log cleanup Signed-off-by: Suraj Singh --- .../org/opensearch/upgrades/IndexingIT.java | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java index c8383c7333901..cc6a5d38cd596 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java @@ -38,6 +38,7 @@ import org.opensearch.client.Response; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.Booleans; +import org.opensearch.common.io.Streams; import org.opensearch.common.settings.Settings; import org.opensearch.index.codec.CodecService; import org.opensearch.index.engine.EngineConfig; @@ -47,6 +48,7 @@ import java.io.IOException; import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -92,6 +94,7 @@ private void waitForSearchableDocs(String index, int shardCount) throws Exceptio waitForClusterHealthWithNoShardMigration(index, "green"); logger.info("--> _cat/shards before search \n{}", EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/shards?v")).getEntity())); + verifySegmentStats(index); Request request = new Request("GET", index + "/_stats"); request.addParameter("level", "shards"); Response response = client().performRequest(request); @@ -111,14 +114,12 @@ private void waitForSearchableDocs(String index, int shardCount) throws Exceptio logger.info("--> replicaShardToNodeIDMap {}", replicaShardToNodeIDMap); for (int shardNumber = 0; shardNumber < shardCount; shardNumber++) { - logger.info("--> Verify doc count for shard number {}", shardNumber); Request searchTestIndexRequest = new Request("POST", "/" + index + "/_search"); searchTestIndexRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); searchTestIndexRequest.addParameter("filter_path", "hits.total"); searchTestIndexRequest.addParameter("preference", "_shards:" + shardNumber + "|_only_nodes:" + primaryShardToNodeIDMap.get(shardNumber)); Response searchTestIndexResponse = client().performRequest(searchTestIndexRequest); final int primaryHits = ObjectPath.createFromResponse(searchTestIndexResponse).evaluate("hits.total"); - logger.info("--> primaryHits {}", primaryHits); final int shardNum = shardNumber; // Verify replica shard doc count only when available. if (replicaShardToNodeIDMap.get(shardNum) != null) { @@ -129,8 +130,7 @@ private void waitForSearchableDocs(String index, int shardCount) throws Exceptio replicaRequest.addParameter("preference", "_shards:" + shardNum + "|_only_nodes:" + replicaShardToNodeIDMap.get(shardNum)); Response replicaResponse = client().performRequest(replicaRequest); int replicaHits = ObjectPath.createFromResponse(replicaResponse).evaluate("hits.total"); - logger.info("--> ReplicaHits {}", replicaHits); - assertEquals(primaryHits, replicaHits); + assertEquals("Doc count mismatch for shard " + shardNum + " primary hits " + primaryHits + " replica hits " + replicaHits, primaryHits, replicaHits); }, 1, TimeUnit.MINUTES); } } @@ -147,10 +147,16 @@ private void waitForClusterHealthWithNoShardMigration(String indexName, String s client().performRequest(waitForStatus); } - private void verifySegmentStats(String indexName) throws IOException { - Request segrepStatsRequest = new Request("GET", "/_cat/segment_replication"); - Response searchTestIndexResponse = client().performRequest(segrepStatsRequest); - logger.info("--> searchTestIndexResponse {}", searchTestIndexResponse.toString()); + private void verifySegmentStats(String indexName) throws Exception { + assertBusy(() -> { + Request segrepStatsRequest = new Request("GET", "/_cat/segment_replication/" + indexName); + segrepStatsRequest.addParameter("h", "shardId,target_node,checkpoints_behind"); + Response segrepStatsResponse = client().performRequest(segrepStatsRequest); + for (String statLine : Streams.readAllLines(segrepStatsResponse.getEntity().getContent())) { + String[] elements = statLine.split(" +"); + assertEquals("Replica shard " + elements[0] + "not upto date with primary ", 0, Integer.parseInt(elements[2])); + } + }); } public void testIndexing() throws IOException, ParseException { @@ -249,7 +255,7 @@ public void testIndexing() throws IOException, ParseException { public void testIndexingWithSegRep() throws Exception { final String indexName = "test-index-segrep"; final int shardCount = 3; - final int replicaCount = 1; + final int replicaCount = 2; logger.info("--> Case {}", CLUSTER_TYPE); printClusterNodes(); logger.info("--> _cat/shards before test execution \n{}", EntityUtils.toString(client().performRequest(new Request("GET", "/_cat/shards?v")).getEntity())); @@ -298,7 +304,6 @@ public void testIndexingWithSegRep() throws Exception { } waitForSearchableDocs(indexName, shardCount); - verifySegmentStats(indexName); assertCount(indexName, expectedCount); if (CLUSTER_TYPE != ClusterType.OLD) { From dc1d55b3b799d3a80a8a78030495bdd618eb9b1e Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Wed, 21 Jun 2023 14:11:05 -0700 Subject: [PATCH 3/8] Spotless check Signed-off-by: Suraj Singh --- .../src/test/java/org/opensearch/upgrades/IndexingIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java index cc6a5d38cd596..b3231302fa8ba 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java @@ -130,7 +130,7 @@ private void waitForSearchableDocs(String index, int shardCount) throws Exceptio replicaRequest.addParameter("preference", "_shards:" + shardNum + "|_only_nodes:" + replicaShardToNodeIDMap.get(shardNum)); Response replicaResponse = client().performRequest(replicaRequest); int replicaHits = ObjectPath.createFromResponse(replicaResponse).evaluate("hits.total"); - assertEquals("Doc count mismatch for shard " + shardNum + " primary hits " + primaryHits + " replica hits " + replicaHits, primaryHits, replicaHits); + assertEquals("Doc count mismatch for shard " + shardNum + ". Primary hits " + primaryHits + " Replica hits " + replicaHits, primaryHits, replicaHits); }, 1, TimeUnit.MINUTES); } } From ca5a4eeb84a4158b7000b69fd42c34d8f1d1daeb Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Fri, 23 Jun 2023 09:51:17 -0700 Subject: [PATCH 4/8] Add version check to skip test for 1.x bwc branches Signed-off-by: Suraj Singh --- .../src/test/java/org/opensearch/upgrades/IndexingIT.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java index b3231302fa8ba..173aa9f6557d2 100644 --- a/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java @@ -253,6 +253,10 @@ public void testIndexing() throws IOException, ParseException { * @throws Exception */ public void testIndexingWithSegRep() throws Exception { + if (UPGRADE_FROM_VERSION.before(Version.V_2_4_0)) { + logger.info("--> Skip test for version {} where segment replication feature is not available", UPGRADE_FROM_VERSION); + return; + } final String indexName = "test-index-segrep"; final int shardCount = 3; final int replicaCount = 2; From 5ae3f65ad4247dccb2edde5b788c710ee48a455b Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Fri, 23 Jun 2023 10:44:13 -0700 Subject: [PATCH 5/8] Add version check to skip test for 1.x bwc branches for mixed clusters Signed-off-by: Suraj Singh --- .../test/java/org/opensearch/backwards/IndexingIT.java | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java b/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java index a6675a6d0ddb5..a45fe67c533b2 100644 --- a/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java +++ b/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java @@ -114,6 +114,10 @@ private void printClusterRouting() throws IOException, ParseException { * @throws Exception */ public void testIndexingWithPrimaryOnBwcNodes() throws Exception { + if (UPGRADE_FROM_VERSION.before(Version.V_2_4_0)) { + logger.info("--> Skip test for version {} where segment replication feature is not available", UPGRADE_FROM_VERSION); + return; + } Nodes nodes = buildNodeAndVersions(); assumeFalse("new nodes is empty", nodes.getNewNodes().isEmpty()); logger.info("cluster discovered:\n {}", nodes.toString()); @@ -161,6 +165,10 @@ public void testIndexingWithPrimaryOnBwcNodes() throws Exception { * @throws Exception */ public void testIndexingWithReplicaOnBwcNodes() throws Exception { + if (UPGRADE_FROM_VERSION.before(Version.V_2_4_0)) { + logger.info("--> Skip test for version {} where segment replication feature is not available", UPGRADE_FROM_VERSION); + return; + } Nodes nodes = buildNodeAndVersions(); assumeFalse("new nodes is empty", nodes.getNewNodes().isEmpty()); logger.info("cluster discovered:\n {}", nodes.toString()); From 91e6e04c2fde614b06d228d26bbd4ccb170a5a20 Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Fri, 23 Jun 2023 10:56:47 -0700 Subject: [PATCH 6/8] Add version string in build to identify bwc version Signed-off-by: Suraj Singh --- qa/mixed-cluster/build.gradle | 1 + .../src/test/java/org/opensearch/backwards/IndexingIT.java | 3 +++ 2 files changed, 4 insertions(+) diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index 55f900c52f2c2..f2e15a1e7377f 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -86,6 +86,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") } + systemProperty 'tests.upgrade_from_version', baseName systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}" onlyIf { project.bwc_tests_enabled } } diff --git a/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java b/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java index a45fe67c533b2..07bce49b18a0d 100644 --- a/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java +++ b/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java @@ -66,6 +66,9 @@ public class IndexingIT extends OpenSearchRestTestCase { + protected static final Version UPGRADE_FROM_VERSION = Version.fromString(System.getProperty("tests.upgrade_from_version")); + + private int indexDocs(String index, final int idStart, final int numDocs) throws IOException { for (int i = 0; i < numDocs; i++) { final int id = idStart + i; From bc5e7dc60b6fea8d3817cb5828e272f774e9c5cc Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Fri, 23 Jun 2023 11:05:01 -0700 Subject: [PATCH 7/8] Use correct bwc version string Signed-off-by: Suraj Singh --- qa/mixed-cluster/build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/qa/mixed-cluster/build.gradle b/qa/mixed-cluster/build.gradle index f2e15a1e7377f..d64bf245dbf8f 100644 --- a/qa/mixed-cluster/build.gradle +++ b/qa/mixed-cluster/build.gradle @@ -55,6 +55,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { } String baseName = "v${bwcVersion}" + String bwcVersionStr = "${bwcVersion}" /* This project runs the core REST tests against a 4 node cluster where two of the nodes has a different minor. */ @@ -86,7 +87,7 @@ for (Version bwcVersion : BuildParams.bwcVersions.wireCompatible) { nonInputProperties.systemProperty('tests.rest.cluster', "${-> testClusters."${baseName}".allHttpSocketURI.join(",")}") nonInputProperties.systemProperty('tests.clustername', "${-> testClusters."${baseName}".getName()}") } - systemProperty 'tests.upgrade_from_version', baseName + systemProperty 'tests.upgrade_from_version', bwcVersionStr systemProperty 'tests.path.repo', "${buildDir}/cluster/shared/repo/${baseName}" onlyIf { project.bwc_tests_enabled } } From 2bd6457301e1a500f6ee093bf9e89122e71665c1 Mon Sep 17 00:00:00 2001 From: Suraj Singh Date: Fri, 23 Jun 2023 12:59:19 -0700 Subject: [PATCH 8/8] Address review comments from https://github.com/opensearch-project/OpenSearch/pull/7626 Signed-off-by: Suraj Singh --- .../src/test/java/org/opensearch/backwards/IndexingIT.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java b/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java index 07bce49b18a0d..b867b90af333c 100644 --- a/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java +++ b/qa/mixed-cluster/src/test/java/org/opensearch/backwards/IndexingIT.java @@ -126,7 +126,7 @@ public void testIndexingWithPrimaryOnBwcNodes() throws Exception { logger.info("cluster discovered:\n {}", nodes.toString()); final List bwcNamesList = nodes.getBWCNodes().stream().map(Node::getNodeName).collect(Collectors.toList()); final String bwcNames = bwcNamesList.stream().collect(Collectors.joining(",")); - // Exclude bwc nodes from allocation so that primaries gets allocated on current version + // Update allocation settings so that primaries gets allocated only on nodes running on older version Settings.Builder settings = Settings.builder() .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1) .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0) @@ -140,7 +140,7 @@ public void testIndexingWithPrimaryOnBwcNodes() throws Exception { try (RestClient nodeClient = buildClient(restClientSettings(), nodes.getNewNodes().stream().map(Node::getPublishAddress).toArray(HttpHost[]::new))) { - logger.info("allowing replica shards assignment on bwc nodes"); + logger.info("Remove allocation include settings so that shards can be allocated on current version nodes"); updateIndexSettings(index, Settings.builder().putNull("index.routing.allocation.include._name")); // Add replicas so that it can be assigned on higher OS version nodes. updateIndexSettings(index, Settings.builder().put("index.number_of_replicas", 2)); @@ -161,7 +161,7 @@ public void testIndexingWithPrimaryOnBwcNodes() throws Exception { /** - * This test creates a cluster with primary on older version but due to {@link org.opensearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider}; + * This test creates a cluster with primary on higher version but due to {@link org.opensearch.cluster.routing.allocation.decider.NodeVersionAllocationDecider}; * replica shard allocation on lower OpenSearch version is prevented. Thus, this test though cover the use case where * primary shard containing nodes are running on higher OS version while replicas are unassigned. *