From 3262e786fe143ef4de56949593a3b17dbacfe60b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dharmesh=20=F0=9F=92=A4?= Date: Fri, 1 Sep 2023 00:30:35 +0530 Subject: [PATCH] Removing remote store specific cluster settings instead rely on node attributes, fixed UTs and ITs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Dharmesh 💤 --- distribution/src/config/opensearch.yml | 4 +- ...emoteStoreMockRepositoryIntegTestCase.java | 70 ++++++++++---- ...eateRemoteIndexClusterDefaultDocRepIT.java | 4 +- .../remotestore/CreateRemoteIndexIT.java | 51 +--------- .../remotestore/PrimaryTermValidationIT.java | 14 ++- .../RemoteIndexPrimaryRelocationIT.java | 6 +- .../remotestore/RemoteIndexRecoveryIT.java | 39 ++------ .../remotestore/RemoteRestoreSnapshotIT.java | 13 +-- .../RemoteSegmentStatsFromNodesStatsIT.java | 5 +- .../RemoteStoreBaseIntegTestCase.java | 92 ++++++------------- .../remotestore/RemoteStoreForceMergeIT.java | 11 +-- .../opensearch/remotestore/RemoteStoreIT.java | 10 +- .../RemoteStoreRepositoryRegistrationIT.java | 12 --- .../remotestore/RemoteStoreRestoreIT.java | 6 +- .../remotestore/RemoteStoreStatsIT.java | 7 +- .../ReplicaToPrimaryPromotionIT.java | 4 - .../SegmentReplicationUsingRemoteStoreIT.java | 13 ++- ...tReplicationWithRemoteStorePressureIT.java | 11 ++- .../multipart/RemoteStoreMultipartIT.java | 10 -- .../opensearch/snapshots/CloneSnapshotIT.java | 30 +++--- .../snapshots/DeleteSnapshotIT.java | 48 ++++------ .../RemoteIndexSnapshotStatusApiIT.java | 25 +++-- ...rvice.java => RemoteStoreNodeService.java} | 40 ++++---- .../cluster/coordination/Coordinator.java | 6 +- .../cluster/coordination/JoinHelper.java | 4 +- .../coordination/JoinTaskExecutor.java | 20 ++-- .../metadata/MetadataCreateIndexService.java | 34 ++++--- .../cluster/node/DiscoveryNode.java | 13 +-- .../common/settings/ClusterSettings.java | 10 +- .../opensearch/discovery/DiscoveryModule.java | 4 +- .../opensearch/indices/IndicesService.java | 30 ------ .../main/java/org/opensearch/node/Node.java | 19 ++-- .../cluster/coordination/JoinHelperTests.java | 6 +- .../coordination/JoinTaskExecutorTests.java | 42 +++++---- .../cluster/coordination/NodeJoinTests.java | 4 +- .../MetadataCreateIndexServiceTests.java | 25 ++--- .../discovery/DiscoveryModuleTests.java | 6 +- .../indices/cluster/ClusterStateChanges.java | 6 +- .../BlobStoreRepositoryHelperTests.java | 2 +- .../BlobStoreRepositoryRemoteIndexTests.java | 64 +++++++------ .../snapshots/SnapshotResiliencyTests.java | 6 +- .../AbstractCoordinatorTestCase.java | 6 +- .../opensearch/test/InternalTestCluster.java | 7 ++ 43 files changed, 366 insertions(+), 473 deletions(-) rename server/src/main/java/org/opensearch/action/admin/cluster/remotestore/{RemoteStoreService.java => RemoteStoreNodeService.java} (88%) diff --git a/distribution/src/config/opensearch.yml b/distribution/src/config/opensearch.yml index 3c4fe822005e0..07d9b278f6eb9 100644 --- a/distribution/src/config/opensearch.yml +++ b/distribution/src/config/opensearch.yml @@ -92,10 +92,10 @@ ${path.logs} # cluster.remote_store.enabled: true # # Repository to use for segment upload while enforcing remote store for an index -# cluster.remote_store.segment.repository: my-repo-1 +# node.attr.remote_store.segment.repository : my-repo-1 # # Repository to use for translog upload while enforcing remote store for an index -# cluster.remote_store.translog.repository: my-repo-1 +# node.attr.remote_store.translog.repository : my-repo-1 # # ---------------------------------- Experimental Features ----------------------------------- # diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java index 6d2d8df106513..83d8e282c3474 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/AbstractRemoteStoreMockRepositoryIntegTestCase.java @@ -26,10 +26,15 @@ import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; +import java.util.Locale; import java.util.Set; import java.util.stream.Collectors; -import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; public abstract class AbstractRemoteStoreMockRepositoryIntegTestCase extends AbstractSnapshotIntegTestCase { @@ -47,7 +52,6 @@ protected Settings featureFlagSettings() { public void setup() { FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); FeatureFlagSetter.set(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL); - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REPOSITORY_NAME, TRANSLOG_REPOSITORY_NAME)); } @Override @@ -66,6 +70,43 @@ protected Settings remoteStoreIndexSettings(int numberOfReplicas) { .build(); } + public Settings buildRemoteStoreNodeAttributes(Path repoLocation, double ioFailureRate, String skipExceptionBlobList, long maxFailure) { + String segmentRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + REPOSITORY_NAME + ); + String translogRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + TRANSLOG_REPOSITORY_NAME + ); + String segmentRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + REPOSITORY_NAME + ); + String translogRepoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + TRANSLOG_REPOSITORY_NAME + ); + + return Settings.builder() + .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, REPOSITORY_NAME) + .put(segmentRepoTypeAttributeKey, "mock") + .put(segmentRepoSettingsAttributeKeyPrefix + "location", repoLocation) + .put(segmentRepoSettingsAttributeKeyPrefix + "random_control_io_exception_rate", ioFailureRate) + .put(segmentRepoSettingsAttributeKeyPrefix + "skip_exception_on_verification_file", true) + .put(segmentRepoSettingsAttributeKeyPrefix + "skip_exception_on_list_blobs", true) + .put(segmentRepoSettingsAttributeKeyPrefix + "skip_exception_on_blobs", skipExceptionBlobList) + .put(segmentRepoSettingsAttributeKeyPrefix + "max_failure_number", maxFailure) + .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, TRANSLOG_REPOSITORY_NAME) + .put(translogRepoTypeAttributeKey, "mock") + .put(translogRepoSettingsAttributeKeyPrefix + "location", repoLocation) + .build(); + } + protected void deleteRepo() { logger.info("--> Deleting the repository={}", REPOSITORY_NAME); assertAcked(clusterAdmin().prepareDeleteRepository(REPOSITORY_NAME)); @@ -73,27 +114,20 @@ protected void deleteRepo() { assertAcked(clusterAdmin().prepareDeleteRepository(TRANSLOG_REPOSITORY_NAME)); } - protected String setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList, long maxFailure) { + protected String setup(Path repoLocation, double ioFailureRate, String skipExceptionBlobList, long maxFailure) throws Exception { logger.info("--> Creating repository={} at the path={}", REPOSITORY_NAME, repoLocation); + logger.info("--> Creating repository={} at the path={}", TRANSLOG_REPOSITORY_NAME, repoLocation); // The random_control_io_exception_rate setting ensures that 10-25% of all operations to remote store results in /// IOException. skip_exception_on_verification_file & skip_exception_on_list_blobs settings ensures that the // repository creation can happen without failure. - createRepository( - REPOSITORY_NAME, - "mock", - Settings.builder() - .put("location", repoLocation) - .put("random_control_io_exception_rate", ioFailureRate) - .put("skip_exception_on_verification_file", true) - .put("skip_exception_on_list_blobs", true) - // Skipping is required for metadata as it is part of recovery - .put("skip_exception_on_blobs", skipExceptionBlobList) - .put("max_failure_number", maxFailure) - ); - logger.info("--> Creating repository={} at the path={}", TRANSLOG_REPOSITORY_NAME, repoLocation); - createRepository(TRANSLOG_REPOSITORY_NAME, "mock", Settings.builder().put("location", repoLocation)); + Settings settings = Settings.builder() + .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), randomBoolean() ? ReplicationType.SEGMENT : ReplicationType.DOCUMENT) + .put(buildRemoteStoreNodeAttributes(repoLocation, ioFailureRate, skipExceptionBlobList, maxFailure)) + .build(); - String dataNodeName = internalCluster().startDataOnlyNodes(1).get(0); + internalCluster().startClusterManagerOnlyNode(settings); + String dataNodeName = internalCluster().startDataOnlyNode(settings); + // assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME); logger.info("--> Created index={}", INDEX_NAME); ensureYellowAndNoInitializingShards(INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRepIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRepIT.java index 37dab5faaeb57..e1ab101fddf55 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRepIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexClusterDefaultDocRepIT.java @@ -75,8 +75,8 @@ public void testDefaultRemoteStoreNoUserOverrideExceptReplicationTypeSegment() t verifyRemoteStoreIndexSettings( indexSettings, "true", - "my-segment-repo-1", - "my-translog-repo-1", + REPOSITORY_NAME, + REPOSITORY_2_NAME, ReplicationType.SEGMENT.toString(), IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL ); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java index 9991126bb790c..d427a4db84ba2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/CreateRemoteIndexIT.java @@ -13,12 +13,9 @@ import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; -import org.opensearch.common.util.FeatureFlags; import org.opensearch.index.IndexSettings; import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.test.FeatureFlagSetter; import org.opensearch.test.OpenSearchIntegTestCase; -import org.junit.After; import org.junit.Before; import java.util.Locale; @@ -28,53 +25,15 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING; -import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsString; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) -public class CreateRemoteIndexIT extends OpenSearchIntegTestCase { - - @After - public void teardown() { - assertAcked(clusterAdmin().prepareDeleteRepository("my-segment-repo-1")); - assertAcked(clusterAdmin().prepareDeleteRepository("my-translog-repo-1")); - assertAcked(clusterAdmin().prepareDeleteRepository("my-custom-repo")); - } - - @Override - protected Settings nodeSettings(int nodeOriginal) { - Settings settings = super.nodeSettings(nodeOriginal); - Settings.Builder builder = Settings.builder() - .put(remoteStoreClusterSettings("my-segment-repo-1", "my-translog-repo-1")) - .put(settings); - return builder.build(); - } - - @Override - protected Settings featureFlagSettings() { - return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.REMOTE_STORE, "true").build(); - } +public class CreateRemoteIndexIT extends RemoteStoreBaseIntegTestCase { @Before - public void setup() { - FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(); - assertAcked( - clusterAdmin().preparePutRepository("my-segment-repo-1") - .setType("fs") - .setSettings(Settings.builder().put("location", randomRepoPath().toAbsolutePath())) - ); - assertAcked( - clusterAdmin().preparePutRepository("my-translog-repo-1") - .setType("fs") - .setSettings(Settings.builder().put("location", randomRepoPath().toAbsolutePath())) - ); - assertAcked( - clusterAdmin().preparePutRepository("my-custom-repo") - .setType("fs") - .setSettings(Settings.builder().put("location", randomRepoPath().toAbsolutePath())) - ); + public void setup() throws Exception { + internalCluster().startNodes(2); } public void testDefaultRemoteStoreNoUserOverride() throws Exception { @@ -91,8 +50,8 @@ public void testDefaultRemoteStoreNoUserOverride() throws Exception { verifyRemoteStoreIndexSettings( indexSettings, "true", - "my-segment-repo-1", - "my-translog-repo-1", + REPOSITORY_NAME, + REPOSITORY_2_NAME, ReplicationType.SEGMENT.toString(), IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL ); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java index a6d8a987886af..21c8dcfb333ce 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/PrimaryTermValidationIT.java @@ -27,7 +27,9 @@ import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.disruption.NetworkDisruption; import org.opensearch.test.transport.MockTransportService; +import org.junit.Before; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collection; import java.util.HashSet; @@ -43,12 +45,20 @@ public class PrimaryTermValidationIT extends RemoteStoreBaseIntegTestCase { private static final String INDEX_NAME = "remote-store-test-idx-1"; + protected Path absolutePath; + protected Path absolutePath2; @Override protected Collection> nodePlugins() { return Arrays.asList(MockTransportService.TestPlugin.class); } + @Before + public void setup() { + absolutePath = randomRepoPath().toAbsolutePath(); + absolutePath2 = randomRepoPath().toAbsolutePath(); + } + public void testPrimaryTermValidation() throws Exception { // Follower checker interval is lower compared to leader checker so that the cluster manager can remove the node // with network partition faster. The follower check retry count is also kept 1. @@ -59,12 +69,10 @@ public void testPrimaryTermValidation() throws Exception { .put(FollowersChecker.FOLLOWER_CHECK_TIMEOUT_SETTING.getKey(), "1s") .put(FollowersChecker.FOLLOWER_CHECK_INTERVAL_SETTING.getKey(), "1s") .put(FollowersChecker.FOLLOWER_CHECK_RETRY_COUNT_SETTING.getKey(), 1) - .put(remoteStoreClusterSettings(REPOSITORY_NAME, REPOSITORY_2_NAME, true)) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, absolutePath, REPOSITORY_2_NAME, absolutePath2)) .build(); internalCluster().startClusterManagerOnlyNode(clusterSettings); internalCluster().startDataOnlyNodes(2, clusterSettings); - ensureStableCluster(3); - assertRepositoryMetadataPresentInClusterState(); // Create index createIndex(INDEX_NAME, remoteStoreIndexSettings(1)); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexPrimaryRelocationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexPrimaryRelocationIT.java index e4dcd637ac448..345fe7a6db77a 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexPrimaryRelocationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexPrimaryRelocationIT.java @@ -18,7 +18,6 @@ import java.nio.file.Path; import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) public class RemoteIndexPrimaryRelocationIT extends IndexPrimaryRelocationIT { @@ -29,15 +28,12 @@ public class RemoteIndexPrimaryRelocationIT extends IndexPrimaryRelocationIT { public void setup() { absolutePath = randomRepoPath().toAbsolutePath(); - assertAcked( - clusterAdmin().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(Settings.builder().put("location", absolutePath)) - ); } protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) - .put(remoteStoreClusterSettings(REPOSITORY_NAME, REPOSITORY_NAME, false)) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, absolutePath)) .build(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java index d6e5e7613ab20..65f91878d56ca 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteIndexRecoveryIT.java @@ -15,18 +15,14 @@ import org.opensearch.index.IndexSettings; import org.opensearch.indices.recovery.IndexRecoveryIT; import org.opensearch.indices.replication.common.ReplicationType; -import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchIntegTestCase; import org.hamcrest.Matcher; import org.hamcrest.Matchers; import org.junit.After; +import org.junit.Before; -import java.util.Locale; +import java.nio.file.Path; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -35,33 +31,18 @@ public class RemoteIndexRecoveryIT extends IndexRecoveryIT { protected static final String REPOSITORY_NAME = "test-remote-store-repo"; + protected Path repositoryPath; + + @Before + public void setup() { + repositoryPath = randomRepoPath().toAbsolutePath(); + } + @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) - .put(remoteStoreClusterSettings(REPOSITORY_NAME)) - .put(repositoryNodeAttributes(REPOSITORY_NAME, FsRepository.TYPE, randomRepoPath().toAbsolutePath().toString())) - .build(); - } - - private Settings repositoryNodeAttributes(String name, String type, String location) { - String segmentRepoNameAttributeKey = "node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; - String translogRepoNameAttributeKey = "node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; - String typeAttributeKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - name - ); - String settingsAttributeKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, - name - ); - return Settings.builder() - .put(segmentRepoNameAttributeKey, name) - .put(translogRepoNameAttributeKey, name) - .put(typeAttributeKey, type) - .put(settingsAttributeKey + "location", location) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, repositoryPath, REPOSITORY_NAME, repositoryPath)) .build(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java index 70e571604ca53..11039742f5d20 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java @@ -29,7 +29,6 @@ import org.opensearch.snapshots.SnapshotState; import org.opensearch.test.InternalTestCluster; import org.junit.After; -import org.junit.Before; import java.io.IOException; import java.nio.file.Path; @@ -47,23 +46,21 @@ public class RemoteRestoreSnapshotIT extends AbstractSnapshotIntegTestCase { private static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX; private Path remoteRepoPath; - @Before - public void setup() { - remoteRepoPath = randomRepoPath().toAbsolutePath(); - createRepository(BASE_REMOTE_REPO, "fs", remoteRepoPath); - } - @After public void teardown() { + remoteRepoPath = null; assertAcked(clusterAdmin().prepareDeleteRepository(BASE_REMOTE_REPO)); } @Override protected Settings nodeSettings(int nodeOrdinal) { + if (remoteRepoPath == null) { + remoteRepoPath = randomRepoPath().toAbsolutePath(); + } return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) .put(FeatureFlags.REMOTE_STORE, "true") - .put(remoteStoreClusterSettings(BASE_REMOTE_REPO)) + .put(remoteStoreClusterSettings(BASE_REMOTE_REPO, remoteRepoPath)) .build(); } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSegmentStatsFromNodesStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSegmentStatsFromNodesStatsIT.java index 7e273828a86ef..32f3b1066aacd 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSegmentStatsFromNodesStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSegmentStatsFromNodesStatsIT.java @@ -25,15 +25,14 @@ public class RemoteSegmentStatsFromNodesStatsIT extends RemoteStoreBaseIntegTest private static final int CLUSTER_MANAGER_NODE_COUNT = 3; @Before - public void setup() throws Exception { + public void setup() { setupCustomCluster(); } - private void setupCustomCluster() throws Exception { + private void setupCustomCluster() { internalCluster().startClusterManagerOnlyNodes(CLUSTER_MANAGER_NODE_COUNT); internalCluster().startDataOnlyNodes(DATA_NODE_COUNT); ensureStableCluster(DATA_NODE_COUNT + CLUSTER_MANAGER_NODE_COUNT); - assertRepositoryMetadataPresentInClusterState(); } /** diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java index bdc2ef2dedefc..53bb95021f0c9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java @@ -16,7 +16,6 @@ import org.opensearch.action.index.IndexResponse; import org.opensearch.action.support.WriteRequest; import org.opensearch.cluster.metadata.IndexMetadata; -import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; @@ -38,16 +37,12 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; @@ -117,10 +112,15 @@ protected boolean addMockInternalEngine() { @Override protected Settings nodeSettings(int nodeOrdinal) { + Settings nodeAttributes = Settings.EMPTY; + if (segmentRepoPath == null || translogRepoPath == null) { + segmentRepoPath = randomRepoPath().toAbsolutePath(); + translogRepoPath = randomRepoPath().toAbsolutePath(); + } return Settings.builder() .put(super.nodeSettings(nodeOrdinal)) - .put(remoteStoreClusterSettings(REPOSITORY_NAME, REPOSITORY_2_NAME, true)) - .put(remoteStoreNodeAttributes(REPOSITORY_NAME, FsRepository.TYPE, REPOSITORY_2_NAME, FsRepository.TYPE)) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, segmentRepoPath, REPOSITORY_2_NAME, translogRepoPath)) + .put(nodeAttributes) .build(); } @@ -163,81 +163,62 @@ protected BulkResponse indexBulk(String indexName, int numDocs) { return client().bulk(bulkRequest).actionGet(); } - public static Settings remoteStoreClusterSettings(String segmentRepoName) { - return remoteStoreClusterSettings(segmentRepoName, segmentRepoName); + public static Settings remoteStoreClusterSettings(String name, Path path) { + return remoteStoreClusterSettings(name, path, name, path); } public static Settings remoteStoreClusterSettings( String segmentRepoName, + Path segmentRepoPath, String translogRepoName, - boolean randomizeSameRepoForRSSAndRTS + Path translogRepoPath ) { - return remoteStoreClusterSettings( - segmentRepoName, - randomizeSameRepoForRSSAndRTS ? (randomBoolean() ? translogRepoName : segmentRepoName) : translogRepoName - ); - } - - public static Settings remoteStoreClusterSettings(String segmentRepoName, String translogRepoName) { - Settings.Builder settingsBuilder = Settings.builder() - .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), segmentRepoName) - .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), translogRepoName); + Settings.Builder settingsBuilder = Settings.builder(); if (randomBoolean()) { settingsBuilder.put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT); } + settingsBuilder.put(buildRemoteStoreNodeAttributes(segmentRepoName, segmentRepoPath, translogRepoName, translogRepoPath)); + return settingsBuilder.build(); } - public Settings remoteStoreNodeAttributes( + public static Settings buildRemoteStoreNodeAttributes( String segmentRepoName, - String segmentRepoType, + Path segmentRepoPath, String translogRepoName, - String translogRepoType + Path translogRepoPath ) { - if (nodeAttributesSettings != null) { - return nodeAttributesSettings; - } - segmentRepoPath = randomRepoPath().toAbsolutePath(); - translogRepoPath = randomRepoPath().toAbsolutePath(); - String segmentRepoKey = String.format( + String segmentRepoTypeAttributeKey = String.format( Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, segmentRepoName ); - String translogRepoKey = String.format( - Locale.getDefault(), - "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, - translogRepoName - ); - String segmentRepoSettingsPrefix = String.format( + String segmentRepoSettingsAttributeKeyPrefix = String.format( Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, segmentRepoName ); - String translogRepoSettingsPrefix = String.format( + String translogRepoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + translogRepoName + ); + String translogRepoSettingsAttributeKeyPrefix = String.format( Locale.getDefault(), "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, translogRepoName ); - if (segmentRepoName.equals(translogRepoName)) { - segmentRepoPath = translogRepoPath; - segmentRepoKey = translogRepoKey; - segmentRepoSettingsPrefix = translogRepoSettingsPrefix; - } - - nodeAttributesSettings = Settings.builder() + return Settings.builder() .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName) - .put(segmentRepoKey, segmentRepoType) - .put(segmentRepoSettingsPrefix + "location", segmentRepoPath.toString()) + .put(segmentRepoTypeAttributeKey, FsRepository.TYPE) + .put(segmentRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath) .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName) - .put(translogRepoKey, translogRepoType) - .put(translogRepoSettingsPrefix + "location", translogRepoPath.toString()) + .put(translogRepoTypeAttributeKey, FsRepository.TYPE) + .put(translogRepoSettingsAttributeKeyPrefix + "location", translogRepoPath) .build(); - return nodeAttributesSettings; } private Settings defaultIndexSettings() { @@ -295,17 +276,4 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { return filesExisting.get(); } - - public void assertRepositoryMetadataPresentInClusterState() throws Exception { - assertBusy(() -> { - RepositoriesMetadata repositoriesMetadata = client().admin() - .cluster() - .prepareState() - .get() - .getState() - .metadata() - .custom(RepositoriesMetadata.TYPE); - assertTrue(repositoriesMetadata != null && !repositoriesMetadata.repositories().isEmpty()); - }, 30, TimeUnit.SECONDS); - } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java index 61497ed896cf5..0bcde4b44c734 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreForceMergeIT.java @@ -17,6 +17,7 @@ import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -89,10 +90,8 @@ private void verifyRestoredData(Map indexStats, long deletedDocs) } private void testRestoreWithMergeFlow(int numberOfIterations, boolean invokeFlush, boolean flushAfterMerge, long deletedDocs) - throws Exception { + throws IOException { internalCluster().startNodes(3); - ensureStableCluster(3); - assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME, remoteStoreIndexSettings(0)); ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); @@ -124,19 +123,19 @@ private void testRestoreWithMergeFlow(int numberOfIterations, boolean invokeFlus // values for each of the flags, number of integ tests become 16 in comparison to current 2. // We have run all the 16 tests on local and they run fine. @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9294") - public void testRestoreForceMergeSingleIteration() throws Exception { + public void testRestoreForceMergeSingleIteration() throws IOException { boolean invokeFLush = randomBoolean(); boolean flushAfterMerge = randomBoolean(); testRestoreWithMergeFlow(1, invokeFLush, flushAfterMerge, randomIntBetween(0, 10)); } - public void testRestoreForceMergeMultipleIterations() throws Exception { + public void testRestoreForceMergeMultipleIterations() throws IOException { boolean invokeFLush = randomBoolean(); boolean flushAfterMerge = randomBoolean(); testRestoreWithMergeFlow(randomIntBetween(2, 5), invokeFLush, flushAfterMerge, randomIntBetween(0, 10)); } - public void testRestoreForceMergeMultipleIterationsDeleteAll() throws Exception { + public void testRestoreForceMergeMultipleIterationsDeleteAll() throws IOException { boolean invokeFLush = randomBoolean(); boolean flushAfterMerge = randomBoolean(); testRestoreWithMergeFlow(randomIntBetween(2, 3), invokeFLush, flushAfterMerge, -1); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java index 1779ffc734a04..549c309057b1e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java @@ -51,8 +51,6 @@ public Settings indexSettings() { private void testPeerRecovery(int numberOfIterations, boolean invokeFlush) throws Exception { internalCluster().startNodes(3); - ensureStableCluster(3); - assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME, remoteStoreIndexSettings(0)); ensureYellowAndNoInitializingShards(INDEX_NAME); ensureGreen(INDEX_NAME); @@ -112,10 +110,8 @@ public void testPeerRecoveryWithRemoteStoreAndRemoteTranslogRefresh() throws Exc testPeerRecovery(randomIntBetween(2, 5), false); } - public void verifyRemoteStoreCleanup() throws Exception { + private void verifyRemoteStoreCleanup() throws Exception { internalCluster().startNodes(3); - ensureStableCluster(3); - assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME, remoteStoreIndexSettings(1)); indexData(5, randomBoolean(), INDEX_NAME); @@ -142,8 +138,6 @@ public void testRemoteTranslogCleanup() throws Exception { public void testStaleCommitDeletionWithInvokeFlush() throws Exception { internalCluster().startNode(); - ensureStableCluster(1); - assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1)); int numberOfIterations = randomIntBetween(5, 15); indexData(numberOfIterations, true, INDEX_NAME); @@ -171,8 +165,6 @@ public void testStaleCommitDeletionWithInvokeFlush() throws Exception { public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception { internalCluster().startNode(); - ensureStableCluster(1); - assertRepositoryMetadataPresentInClusterState(); createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1)); int numberOfIterations = randomIntBetween(5, 15); indexData(numberOfIterations, false, INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java index 1ca6d2c23cf0d..47d3c29756cd0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRepositoryRegistrationIT.java @@ -15,7 +15,6 @@ import org.opensearch.common.settings.Settings; import org.opensearch.plugins.Plugin; import org.opensearch.repositories.blobstore.BlobStoreRepository; -import org.opensearch.repositories.fs.FsRepository; import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.transport.MockTransportService; @@ -55,7 +54,6 @@ private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String na } private void assertRemoteStoreRepositoryOnAllNodes() throws Exception { - assertRepositoryMetadataPresentInClusterState(); RepositoriesMetadata repositories = internalCluster().getInstance(ClusterService.class, internalCluster().getNodeNames()[0]) .state() .metadata() @@ -75,27 +73,17 @@ private void assertRemoteStoreRepositoryOnAllNodes() throws Exception { public void testSingleNodeClusterRepositoryRegistration() throws Exception { internalCluster().startNode(); - ensureStableCluster(1); assertRemoteStoreRepositoryOnAllNodes(); } public void testMultiNodeClusterRepositoryRegistration() throws Exception { internalCluster().startNodes(3); - ensureStableCluster(3); assertRemoteStoreRepositoryOnAllNodes(); } public void testMultiNodeClusterRepositoryRegistrationWithMultipleMasters() throws Exception { internalCluster().startClusterManagerOnlyNodes(3); internalCluster().startNodes(3); - ensureStableCluster(6); - assertRemoteStoreRepositoryOnAllNodes(); - } - - public void testMultiNodeClusterRepositoryRegistrationDifferent() throws Exception { - internalCluster().startNodes(3); - internalCluster().startNode(remoteStoreNodeAttributes(REPOSITORY_NAME, FsRepository.TYPE, REPOSITORY_NAME, FsRepository.TYPE)); - ensureStableCluster(3); assertRemoteStoreRepositoryOnAllNodes(); } } diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java index 4b6c98ba60afe..aeb4756c21a8b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java @@ -87,11 +87,9 @@ private void verifyRestoredData(Map indexStats, String indexName) ); } - private void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, String indices, int replicaCount, int shardCount) - throws Exception { + private void prepareCluster(int numClusterManagerNodes, int numDataOnlyNodes, String indices, int replicaCount, int shardCount) { internalCluster().startClusterManagerOnlyNodes(numClusterManagerNodes); internalCluster().startDataOnlyNodes(numDataOnlyNodes); - assertRepositoryMetadataPresentInClusterState(); for (String index : indices.split(",")) { createIndex(index, remoteStoreIndexSettings(replicaCount, shardCount)); ensureYellowAndNoInitializingShards(index); @@ -263,7 +261,7 @@ private void testRestoreFlowMultipleIndices(int numberOfIterations, boolean invo } } - public void testRestoreFlowAllShardsNoRedIndex() throws Exception { + public void testRestoreFlowAllShardsNoRedIndex() throws InterruptedException { int shardCount = randomIntBetween(1, 5); prepareCluster(1, 3, INDEX_NAME, 0, shardCount); indexData(randomIntBetween(2, 5), true, INDEX_NAME); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java index 63b556a4401fa..796f74d560067 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreStatsIT.java @@ -47,10 +47,9 @@ public class RemoteStoreStatsIT extends RemoteStoreBaseIntegTestCase { public void setup() throws Exception { internalCluster().startNodes(3); ensureStableCluster(3); - assertRepositoryMetadataPresentInClusterState(); } - public void testStatsResponseFromAllNodes() throws Exception { + public void testStatsResponseFromAllNodes() { // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes // during this time frame. This ensures that the segment upload has started. @@ -106,7 +105,7 @@ public void testStatsResponseFromAllNodes() throws Exception { } } - public void testStatsResponseAllShards() throws Exception { + public void testStatsResponseAllShards() { // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes // during this time frame. This ensures that the segment upload has started. @@ -151,7 +150,7 @@ public void testStatsResponseAllShards() throws Exception { } - public void testStatsResponseFromLocalNode() throws Exception { + public void testStatsResponseFromLocalNode() { // Step 1 - We create cluster, create an index, and then index documents into. We also do multiple refreshes/flushes // during this time frame. This ensures that the segment upload has started. diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java index edea93464e650..3e41c4925ea46 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/ReplicaToPrimaryPromotionIT.java @@ -45,8 +45,6 @@ public Settings indexSettings() { public void testPromoteReplicaToPrimary() throws Exception { internalCluster().startNode(); internalCluster().startNode(); - ensureStableCluster(2); - assertRepositoryMetadataPresentInClusterState(); final String indexName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); shard_count = scaledRandomIntBetween(1, 5); createIndex(indexName); @@ -122,8 +120,6 @@ public void testPromoteReplicaToPrimary() throws Exception { public void testFailoverWhileIndexing() throws Exception { internalCluster().startNode(); internalCluster().startNode(); - ensureStableCluster(2); - assertRepositoryMetadataPresentInClusterState(); final String indexName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT); shard_count = scaledRandomIntBetween(1, 5); createIndex(indexName); diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java index 22250c3b793cf..1b817408596ab 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationUsingRemoteStoreIT.java @@ -30,10 +30,17 @@ public class SegmentReplicationUsingRemoteStoreIT extends SegmentReplicationIT { private static final String REPOSITORY_NAME = "test-remote-store-repo"; + protected Path absolutePath; @Override protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(remoteStoreClusterSettings(REPOSITORY_NAME)).build(); + if (absolutePath == null) { + absolutePath = randomRepoPath().toAbsolutePath(); + } + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, absolutePath)) + .build(); } protected boolean segmentReplicationWithRemoteEnabled() { @@ -52,10 +59,6 @@ protected Settings featureFlagSettings() { @Before public void setup() { internalCluster().startClusterManagerOnlyNode(); - Path absolutePath = randomRepoPath().toAbsolutePath(); - assertAcked( - clusterAdmin().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(Settings.builder().put("location", absolutePath)) - ); } @After diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java index 99927797d5fbc..fa0944e5bfee0 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/SegmentReplicationWithRemoteStorePressureIT.java @@ -29,6 +29,7 @@ public class SegmentReplicationWithRemoteStorePressureIT extends SegmentReplicationPressureIT { private static final String REPOSITORY_NAME = "test-remote-store-repo"; + protected Path absolutePath; @Override protected boolean segmentReplicationWithRemoteEnabled() { @@ -46,16 +47,16 @@ protected Settings featureFlagSettings() { @Override protected Settings nodeSettings(int nodeOrdinal) { - return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(remoteStoreClusterSettings(REPOSITORY_NAME)).build(); + return Settings.builder() + .put(super.nodeSettings(nodeOrdinal)) + .put(remoteStoreClusterSettings(REPOSITORY_NAME, absolutePath)) + .build(); } @Before public void setup() { + absolutePath = randomRepoPath().toAbsolutePath(); internalCluster().startClusterManagerOnlyNode(); - Path absolutePath = randomRepoPath().toAbsolutePath(); - assertAcked( - clusterAdmin().preparePutRepository(REPOSITORY_NAME).setType("fs").setSettings(Settings.builder().put("location", absolutePath)) - ); } @After diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java index da8ba77436f7f..d7d80f90089d9 100644 --- a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/RemoteStoreMultipartIT.java @@ -34,16 +34,6 @@ protected Collection> nodePlugins() { return Stream.concat(super.nodePlugins().stream(), Stream.of(MockFsRepositoryPlugin.class)).collect(Collectors.toList()); } - @Override - public Settings remoteStoreNodeAttributes( - String segmentRepoName, - String segmentRepoType, - String translogRepoName, - String translogRepoType - ) { - return super.remoteStoreNodeAttributes(segmentRepoName, MockFsRepositoryPlugin.TYPE, translogRepoName, MockFsRepositoryPlugin.TYPE); - } - public void testRateLimitedRemoteUploads() throws Exception { internalCluster().startNode(); Client client = client(); diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java index 072e03e8a2f79..55ce2805fcb94 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotIT.java @@ -161,8 +161,9 @@ public void testCloneShallowSnapshotIndex() throws Exception { disableRepoConsistencyCheck("This test uses remote store repository"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); final String remoteStoreRepoName = "remote-store-repo-name"; - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName)); - internalCluster().startDataOnlyNode(); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName, remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName, remoteStoreRepoPath)); final String snapshotRepoName = "snapshot-repo-name"; final Path snapshotRepoPath = randomRepoPath(); @@ -172,9 +173,6 @@ public void testCloneShallowSnapshotIndex() throws Exception { final Path shallowSnapshotRepoPath = randomRepoPath(); createRepository(shallowSnapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy(shallowSnapshotRepoPath)); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); - final String indexName = "index-1"; createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); @@ -208,10 +206,14 @@ public void testShallowCloneNameAvailability() throws Exception { disableRepoConsistencyCheck("This test uses remote store repository"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); final String remoteStoreRepoName = "remote-store-repo-name"; + final Path remoteStorePath = randomRepoPath().toAbsolutePath(); internalCluster().startClusterManagerOnlyNode( - Settings.builder().put(LARGE_SNAPSHOT_POOL_SETTINGS).put(remoteStoreClusterSettings(remoteStoreRepoName)).build() + Settings.builder() + .put(LARGE_SNAPSHOT_POOL_SETTINGS) + .put(remoteStoreClusterSettings(remoteStoreRepoName, remoteStorePath)) + .build() ); - internalCluster().startDataOnlyNode(); + internalCluster().startDataOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName, remoteStorePath)); final String shallowSnapshotRepoName = "shallow-snapshot-repo-name"; final Path shallowSnapshotRepoPath = randomRepoPath(); @@ -245,16 +247,14 @@ public void testCloneAfterRepoShallowSettingEnabled() throws Exception { disableRepoConsistencyCheck("This test uses remote store repository"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); final String remoteStoreRepoName = "remote-store-repo-name"; - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName)); - internalCluster().startDataOnlyNode(); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName, remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName, remoteStoreRepoPath)); final String snapshotRepoName = "snapshot-repo-name"; final Path snapshotRepoPath = randomRepoPath(); createRepository(snapshotRepoName, "fs", snapshotRepoPath); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); - final String indexName = "index-1"; createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); @@ -282,16 +282,14 @@ public void testCloneAfterRepoShallowSettingDisabled() throws Exception { disableRepoConsistencyCheck("This test uses remote store repository"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); final String remoteStoreRepoName = "remote-store-repo-name"; - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName)); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(remoteStoreRepoName, remoteStoreRepoPath)); internalCluster().startDataOnlyNode(); final String snapshotRepoName = "snapshot-repo-name"; final Path snapshotRepoPath = randomRepoPath(); createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy(snapshotRepoPath)); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); - final String indexName = "index-1"; createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java index 3a9eacd6ac183..448b860683668 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java @@ -42,16 +42,14 @@ public class DeleteSnapshotIT extends AbstractSnapshotIntegTestCase { public void testDeleteSnapshot() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME)); - internalCluster().startDataOnlyNode(); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); final String snapshotRepoName = "snapshot-repo-name"; final Path snapshotRepoPath = randomRepoPath(); createRepository(snapshotRepoName, "fs", snapshotRepoPath); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(REMOTE_REPO_NAME, "fs", remoteStoreRepoPath); - final String indexName = "index-1"; createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); @@ -72,15 +70,13 @@ public void testDeleteSnapshot() throws Exception { public void testDeleteShallowCopySnapshot() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME)); - internalCluster().startDataOnlyNode(); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); final String snapshotRepoName = "snapshot-repo-name"; createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy()); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(REMOTE_REPO_NAME, "fs", remoteStoreRepoPath); - final String indexName = "index-1"; createIndexWithRandomDocs(indexName, randomIntBetween(5, 10)); @@ -104,14 +100,12 @@ public void testDeleteMultipleShallowCopySnapshotsCase1() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME)); - internalCluster().startDataOnlyNode(); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); final Client clusterManagerClient = internalCluster().clusterManagerClient(); ensureStableCluster(2); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(REMOTE_REPO_NAME, "fs", remoteStoreRepoPath); - final String snapshotRepoName = "snapshot-repo-name"; final Path snapshotRepoPath = randomRepoPath(); createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(snapshotRepoPath)); @@ -150,8 +144,9 @@ public void testDeleteMultipleShallowCopySnapshotsCase2() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME)); - final String dataNode = internalCluster().startDataOnlyNode(); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); + final String dataNode = internalCluster().startDataOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); ensureStableCluster(2); final String clusterManagerNode = internalCluster().getClusterManagerName(); @@ -161,9 +156,6 @@ public void testDeleteMultipleShallowCopySnapshotsCase2() throws Exception { final String testIndex = "index-test"; createIndexWithContent(testIndex); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(REMOTE_REPO_NAME, "fs", remoteStoreRepoPath); - final String remoteStoreEnabledIndexName = "remote-index-1"; final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); @@ -238,8 +230,9 @@ public void testDeleteMultipleShallowCopySnapshotsCase3() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME)); - internalCluster().startDataOnlyNode(); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); final Client clusterManagerClient = internalCluster().clusterManagerClient(); ensureStableCluster(2); @@ -247,9 +240,6 @@ public void testDeleteMultipleShallowCopySnapshotsCase3() throws Exception { final Path snapshotRepoPath = randomRepoPath(); createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(snapshotRepoPath)); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(REMOTE_REPO_NAME, "fs", remoteStoreRepoPath); - final String testIndex = "index-test"; createIndexWithContent(testIndex); @@ -300,8 +290,9 @@ public void testRemoteStoreCleanupForDeletedIndex() throws Exception { disableRepoConsistencyCheck("Remote store repository is being used in the test"); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); - internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME)); - internalCluster().startDataOnlyNode(); + final Path remoteStoreRepoPath = randomRepoPath(); + internalCluster().startClusterManagerOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); + internalCluster().startDataOnlyNode(remoteStoreClusterSettings(REMOTE_REPO_NAME, remoteStoreRepoPath)); final Client clusterManagerClient = internalCluster().clusterManagerClient(); ensureStableCluster(2); @@ -309,9 +300,6 @@ public void testRemoteStoreCleanupForDeletedIndex() throws Exception { final Path snapshotRepoPath = randomRepoPath(); createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy(snapshotRepoPath)); - final Path remoteStoreRepoPath = randomRepoPath(); - createRepository(REMOTE_REPO_NAME, "fs", remoteStoreRepoPath); - final String testIndex = "index-test"; createIndexWithContent(testIndex); diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java index fb91b1d7a006c..231e8ad3788be 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RemoteIndexSnapshotStatusApiIT.java @@ -40,7 +40,9 @@ import org.opensearch.common.action.ActionFuture; import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.threadpool.ThreadPool; +import org.junit.Before; import java.nio.file.Path; @@ -49,8 +51,17 @@ import static org.hamcrest.Matchers.greaterThan; import static org.hamcrest.Matchers.is; +@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST) public class RemoteIndexSnapshotStatusApiIT extends AbstractSnapshotIntegTestCase { + protected Path absolutePath; + final String remoteStoreRepoName = "remote-store-repo-name"; + + @Before + public void setup() { + absolutePath = randomRepoPath().toAbsolutePath(); + } + @Override protected Settings nodeSettings(int nodeOrdinal) { return Settings.builder() @@ -58,7 +69,7 @@ protected Settings nodeSettings(int nodeOrdinal) { .put(ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING.getKey(), 0) // We have tests that check by-timestamp order .put(FeatureFlags.REMOTE_STORE, "true") .put(FeatureFlags.SEGMENT_REPLICATION_EXPERIMENTAL, "true") - .put(remoteStoreClusterSettings("remote-store-repo-name")) + .put(remoteStoreClusterSettings(remoteStoreRepoName, absolutePath)) .build(); } @@ -70,10 +81,6 @@ public void testStatusAPICallForShallowCopySnapshot() throws Exception { final String snapshotRepoName = "snapshot-repo-name"; createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy()); - final Path remoteStoreRepoPath = randomRepoPath(); - final String remoteStoreRepoName = "remote-store-repo-name"; - createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); - final String remoteStoreEnabledIndexName = "remote-index-1"; final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); @@ -109,10 +116,6 @@ public void testStatusAPIStatsForBackToBackShallowSnapshot() throws Exception { final String snapshotRepoName = "snapshot-repo-name"; createRepository(snapshotRepoName, "fs", snapshotRepoSettingsForShallowCopy()); - final Path remoteStoreRepoPath = randomRepoPath(); - final String remoteStoreRepoName = "remote-store-repo-name"; - createRepository(remoteStoreRepoName, "fs", remoteStoreRepoPath); - final String remoteStoreEnabledIndexName = "remote-index-1"; final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); @@ -157,10 +160,6 @@ public void testStatusAPICallInProgressShallowSnapshot() throws Exception { final String snapshotRepoName = "snapshot-repo-name"; createRepository(snapshotRepoName, "mock", snapshotRepoSettingsForShallowCopy().put("block_on_data", true)); - final Path remoteStoreRepoPath = randomRepoPath(); - final String remoteStoreRepoName = "remote-store-repo-name"; - createRepository(remoteStoreRepoName, "mock", remoteStoreRepoPath); - final String remoteStoreEnabledIndexName = "remote-index-1"; final Settings remoteStoreEnabledIndexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreEnabledIndexName, remoteStoreEnabledIndexSettings); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNodeService.java similarity index 88% rename from server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java rename to server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNodeService.java index e2cee2f2f6449..6b6733c7f1ab7 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreService.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/remotestore/RemoteStoreNodeService.java @@ -30,9 +30,9 @@ /** * Contains all the method needed for a remote store backed node lifecycle. */ -public class RemoteStoreService { +public class RemoteStoreNodeService { - private static final Logger logger = LogManager.getLogger(RemoteStoreService.class); + private static final Logger logger = LogManager.getLogger(RemoteStoreNodeService.class); private final Supplier repositoriesService; private final ThreadPool threadPool; public static final Setting REMOTE_STORE_COMPATIBILITY_MODE_SETTING = new Setting<>( @@ -74,39 +74,37 @@ public static CompatibilityMode parseString(String compatibilityMode) { } } - public RemoteStoreService(Supplier repositoriesService, ThreadPool threadPool) { + public RemoteStoreNodeService(Supplier repositoriesService, ThreadPool threadPool) { this.repositoriesService = repositoriesService; this.threadPool = threadPool; } /** - * Performs repository verification during node startup post its creation by invoking verify method against - * repository mentioned. This verification will happen on a local node to validate if the node is able to connect - * to the repository. + * Creates a repository during a node startup and performs verification by invoking verify method against + * mentioned repository. This verification will happen on a local node to validate if the node is able to connect + * to the repository with appropriate permissions. */ - public void verifyRepositoriesLocally(List repositories, DiscoveryNode localNode) { - for (Repository repository : repositories) { - String repositoryName = repository.getMetadata().name(); - String verificationToken = repository.startVerification(); - repository.verify(verificationToken, localNode); - repository.endVerification(verificationToken); - logger.info(() -> new ParameterizedMessage("successfully verified [{}] repository", repositoryName)); - } - } - - /** - * Creates a repository during a node startup. - */ - public List createRepositories(RemoteStoreNode node) { + public List createAndVerifyRepositories(DiscoveryNode localNode) { + RemoteStoreNode node = new RemoteStoreNode(localNode); List repositories = new ArrayList<>(); for (RepositoryMetadata repositoryMetadata : node.getRepositoriesMetadata().repositories()) { - RepositoriesService.validate(repositoryMetadata.name()); + String repositoryName = repositoryMetadata.name(); + + // Create Repository + RepositoriesService.validate(repositoryName); Repository repository = repositoriesService.get().createRepository(repositoryMetadata); logger.info( "remote backed storage repository with name {} and type {} created.", repository.getMetadata().name(), repository.getMetadata().type() ); + + // Verify Repository + String verificationToken = repository.startVerification(); + repository.verify(verificationToken, localNode); + repository.endVerification(verificationToken); + logger.info(() -> new ParameterizedMessage("successfully verified [{}] repository", repositoryName)); + repositories.add(repository); } return repositories; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index bd642208c501d..98a2336f11bb7 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -35,7 +35,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.message.ParameterizedMessage; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterChangedEvent; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -182,7 +182,7 @@ public class Coordinator extends AbstractLifecycleComponent implements Discovery private JoinHelper.JoinAccumulator joinAccumulator; private Optional currentPublication = Optional.empty(); private final NodeHealthService nodeHealthService; - private final RemoteStoreService remoteStoreService; + private final RemoteStoreNodeService remoteStoreService; /** * @param nodeName The name of the node, used to name the {@link java.util.concurrent.ExecutorService} of the {@link SeedHostsResolver}. @@ -204,7 +204,7 @@ public Coordinator( RerouteService rerouteService, ElectionStrategy electionStrategy, NodeHealthService nodeHealthService, - RemoteStoreService remoteStoreService + RemoteStoreNodeService remoteStoreService ) { this.settings = settings; this.transportService = transportService; diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java index 50a824f5ae300..14f30c272a3a4 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinHelper.java @@ -37,7 +37,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.Version; import org.opensearch.action.ActionListenerResponseHandler; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskConfig; import org.opensearch.cluster.ClusterStateTaskListener; @@ -136,7 +136,7 @@ public class JoinHelper { AllocationService allocationService, ClusterManagerService clusterManagerService, TransportService transportService, - RemoteStoreService remoteStoreService, + RemoteStoreNodeService remoteStoreService, LongSupplier currentTermSupplier, Supplier currentStateSupplier, BiConsumer joinHandler, diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index 53b25970a1876..d25f327fdb37d 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -34,7 +34,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.Version; import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskExecutor; import org.opensearch.cluster.NotClusterManagerException; @@ -61,9 +61,9 @@ import java.util.function.BiConsumer; import java.util.stream.Collectors; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreService.CompatibilityMode; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreService.CompatibilityMode.STRICT; -import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService.CompatibilityMode; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService.CompatibilityMode.STRICT; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING; import static org.opensearch.cluster.decommission.DecommissionHelper.nodeCommissioned; import static org.opensearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; @@ -79,7 +79,7 @@ public class JoinTaskExecutor implements ClusterStateTaskExecutor execute(ClusterState currentState, List jo final DiscoveryNode node = joinTask.node(); if (joinTask.isBecomeClusterManagerTask() || joinTask.isFinishElectionTask()) { // noop - } else if (currentNodes.nodeExists(node)) { - if (currentNodes.nodeExistsWithSameRoles(node)) { - logger.debug("received a join request for an existing node [{}]", node); - } + } else if (currentNodes.nodeExistsWithSameRoles(node)) { + logger.debug("received a join request for an existing node [{}]", node); + // TODO: Fix this by moving it out of this if condition, Had to add this code back here as this was + // leading to failure of JoinTaskExecutorTests::testUpdatesNodeWithNewRoles test. if (node.isRemoteStoreNode()) { /** cluster state is updated here as elect leader task can have same node present in join task as * well as current node. We want the repositories to be added in cluster state during first node diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index fd7fe29442eb2..9bef81d9019cb 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -39,6 +39,7 @@ import org.opensearch.OpenSearchException; import org.opensearch.ResourceAlreadyExistsException; import org.opensearch.Version; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode; import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.admin.indices.create.CreateIndexClusterStateUpdateRequest; import org.opensearch.action.admin.indices.shrink.ResizeType; @@ -94,6 +95,7 @@ import org.opensearch.indices.ShardLimitValidator; import org.opensearch.indices.SystemIndices; import org.opensearch.indices.replication.common.ReplicationType; +import org.opensearch.node.Node; import org.opensearch.threadpool.ThreadPool; import java.io.IOException; @@ -132,9 +134,6 @@ import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE; import static org.opensearch.cluster.metadata.Metadata.DEFAULT_REPLICA_COUNT_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; /** @@ -927,7 +926,8 @@ static Settings aggregateIndexSettings( } /** - * Updates index settings to set replication strategy by default based on cluster level settings + * Updates index settings to set replication strategy by default based on cluster level settings or remote store + * node attributes * @param settingsBuilder index settings builder to be updated with relevant settings * @param requestSettings settings passed in during index create request * @param clusterSettings cluster level settings @@ -937,23 +937,31 @@ private static void updateReplicationStrategy(Settings.Builder settingsBuilder, settingsBuilder.put(SETTING_REPLICATION_TYPE, INDEX_REPLICATION_TYPE_SETTING.get(requestSettings)); } else if (CLUSTER_REPLICATION_TYPE_SETTING.exists(clusterSettings)) { settingsBuilder.put(SETTING_REPLICATION_TYPE, CLUSTER_REPLICATION_TYPE_SETTING.get(clusterSettings)); - } else if (CLUSTER_REMOTE_STORE_ENABLED_SETTING.get(clusterSettings)) { - settingsBuilder.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT); - } else { - settingsBuilder.put(SETTING_REPLICATION_TYPE, CLUSTER_REPLICATION_TYPE_SETTING.getDefault(clusterSettings)); - } + } else if (clusterSettings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNode.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX) + .isEmpty() == false) { + settingsBuilder.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT); + } else { + settingsBuilder.put(SETTING_REPLICATION_TYPE, CLUSTER_REPLICATION_TYPE_SETTING.getDefault(clusterSettings)); + } } /** - * Updates index settings to enable remote store by default based on cluster level settings + * Updates index settings to enable remote store by default based on node attributes * @param settingsBuilder index settings builder to be updated with relevant settings * @param clusterSettings cluster level settings */ private static void updateRemoteStoreSettings(Settings.Builder settingsBuilder, Settings clusterSettings) { - if (CLUSTER_REMOTE_STORE_ENABLED_SETTING.get(clusterSettings) == true) { + if (clusterSettings.getByPrefix(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNode.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX) + .isEmpty() == false) { settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true) - .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.get(clusterSettings)) - .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.get(clusterSettings)); + .put( + SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, + clusterSettings.get(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY) + ) + .put( + SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, + clusterSettings.get(Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY) + ); } } diff --git a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java index 30f5be2730ac9..1df6984c5659e 100644 --- a/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java +++ b/server/src/main/java/org/opensearch/cluster/node/DiscoveryNode.java @@ -33,8 +33,7 @@ package org.opensearch.cluster.node; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.common.UUIDs; import org.opensearch.common.annotation.PublicApi; import org.opensearch.common.settings.Setting; @@ -46,13 +45,11 @@ import org.opensearch.core.xcontent.ToXContentFragment; import org.opensearch.core.xcontent.XContentBuilder; import org.opensearch.node.Node; -import org.opensearch.repositories.Repository; import java.io.IOException; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; -import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; @@ -287,11 +284,11 @@ public static DiscoveryNode createLocal(Settings settings, TransportAddress publ } /** Creates a DiscoveryNode representing the local node and verifies the repository. */ - public static DiscoveryNode createLocal( + public static DiscoveryNode createRemoteNodeLocal( Settings settings, TransportAddress publishAddress, String nodeId, - RemoteStoreService remoteStoreService + RemoteStoreNodeService remoteStoreService ) { Map attributes = Node.NODE_ATTRIBUTES.getAsMap(settings); Set roles = getRolesFromSettings(settings); @@ -303,9 +300,7 @@ public static DiscoveryNode createLocal( roles, Version.CURRENT ); - RemoteStoreNode remoteStoreNode = new RemoteStoreNode(discoveryNode); - List repositories = remoteStoreService.createRepositories(remoteStoreNode); - remoteStoreService.verifyRepositoriesLocally(repositories, discoveryNode); + remoteStoreService.createAndVerifyRepositories(discoveryNode); return discoveryNode; } diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 5ea9de2589ae5..e5d22714f2915 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -33,7 +33,7 @@ import org.apache.logging.log4j.LogManager; import org.opensearch.action.admin.cluster.configuration.TransportAddVotingConfigExclusionsAction; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.action.admin.indices.close.TransportCloseIndexAction; import org.opensearch.action.search.CreatePitController; import org.opensearch.action.search.TransportSearchAction; @@ -672,7 +672,7 @@ public void apply(Settings value, Settings current, Settings previous) { RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING, RemoteClusterStateService.REMOTE_CLUSTER_STATE_REPOSITORY_SETTING, //RemoteStoreService.REMOTE_STORE_MIGRATION_SETTING, - RemoteStoreService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING + RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING ) ) ); @@ -685,12 +685,6 @@ public void apply(Settings value, Settings current, Settings previous) { * setting should be moved to {@link #BUILT_IN_CLUSTER_SETTINGS}. */ public static final Map, List> FEATURE_FLAGGED_CLUSTER_SETTINGS = Map.of( - List.of(FeatureFlags.REMOTE_STORE), - List.of( - IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING, - IndicesService.CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING, - IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING - ), List.of(FeatureFlags.CONCURRENT_SEGMENT_SEARCH), List.of( SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING, diff --git a/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java b/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java index 4f9ead6a3883f..8a59132af9ab6 100644 --- a/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java +++ b/server/src/main/java/org/opensearch/discovery/DiscoveryModule.java @@ -34,7 +34,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.coordination.Coordinator; import org.opensearch.cluster.coordination.ElectionStrategy; @@ -131,7 +131,7 @@ public DiscoveryModule( GatewayMetaState gatewayMetaState, RerouteService rerouteService, NodeHealthService nodeHealthService, - RemoteStoreService remoteStoreService + RemoteStoreNodeService remoteStoreService ) { final Collection> joinValidators = new ArrayList<>(); final Map> hostProviders = new HashMap<>(); diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java index 13a82ce5bdf0d..d249466e3d1d7 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesService.java +++ b/server/src/main/java/org/opensearch/indices/IndicesService.java @@ -242,36 +242,6 @@ public class IndicesService extends AbstractLifecycleComponent Property.Final ); - /** - * Used to specify if all indexes are to create with remote store enabled by default - */ - public static final Setting CLUSTER_REMOTE_STORE_ENABLED_SETTING = Setting.boolSetting( - "cluster.remote_store.enabled", - false, - Property.NodeScope, - Property.Final - ); - - /** - * Used to specify default repo to use for segment upload for remote store backed indices - */ - public static final Setting CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING = Setting.simpleString( - "cluster.remote_store.segment.repository", - "", - Property.NodeScope, - Property.Final - ); - - /** - * Used to specify default repo to use for translog upload for remote store backed indices - */ - public static final Setting CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING = Setting.simpleString( - "cluster.remote_store.translog.repository", - "", - Property.NodeScope, - Property.Final - ); - /** * This setting is used to set the refresh interval when the {@code index.refresh_interval} index setting is not * provided during index creation or when the existing {@code index.refresh_interval} index setting is set as null. diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index c8856c4bf7e02..3fd16cc71d9ac 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -43,8 +43,7 @@ import org.opensearch.action.ActionModule; import org.opensearch.action.ActionModule.DynamicActionRegistry; import org.opensearch.action.ActionType; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.action.admin.cluster.snapshots.status.TransportNodesSnapshotsStatus; import org.opensearch.action.search.SearchExecutionStatsCollector; import org.opensearch.action.search.SearchPhaseController; @@ -263,6 +262,7 @@ import java.util.stream.Stream; import static java.util.stream.Collectors.toList; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX; import static org.opensearch.common.util.FeatureFlags.TELEMETRY; import static org.opensearch.env.NodeEnvironment.collectFileCacheDataPath; import static org.opensearch.index.ShardIndexingPressureSettings.SHARD_INDEXING_PRESSURE_ENABLED_ATTRIBUTE_KEY; @@ -525,7 +525,7 @@ protected Node( final ThreadPool threadPool = new ThreadPool(settings, runnableTaskListener, executorBuilders.toArray(new ExecutorBuilder[0])); final SetOnce repositoriesServiceReference = new SetOnce<>(); - final RemoteStoreService remoteStoreService = new RemoteStoreService(repositoriesServiceReference::get, threadPool); + final RemoteStoreNodeService remoteStoreService = new RemoteStoreNodeService(repositoriesServiceReference::get, threadPool); localNodeFactory = new LocalNodeFactory(settings, nodeEnvironment.nodeId(), remoteStoreService); resourcesToClose.add(() -> ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS)); final ResourceWatcherService resourceWatcherService = new ResourceWatcherService(settings, threadPool); @@ -1732,9 +1732,9 @@ private static class LocalNodeFactory implements Function localNode = new SetOnce<>(); private final String persistentNodeId; private final Settings settings; - private final RemoteStoreService remoteStoreService; + private final RemoteStoreNodeService remoteStoreService; - private LocalNodeFactory(Settings settings, String persistentNodeId, RemoteStoreService remoteStoreService) { + private LocalNodeFactory(Settings settings, String persistentNodeId, RemoteStoreNodeService remoteStoreService) { this.persistentNodeId = persistentNodeId; this.settings = settings; this.remoteStoreService = remoteStoreService; @@ -1745,9 +1745,14 @@ public DiscoveryNode apply(BoundTransportAddress boundTransportAddress) { if (Node.NODE_ATTRIBUTES.getAsMap(settings) .keySet() .stream() - .anyMatch(key -> key.startsWith(RemoteStoreNode.REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX))) { + .anyMatch(key -> key.startsWith(REMOTE_STORE_NODE_ATTRIBUTE_KEY_PREFIX))) { localNode.set( - DiscoveryNode.createLocal(settings, boundTransportAddress.publishAddress(), persistentNodeId, remoteStoreService) + DiscoveryNode.createRemoteNodeLocal( + settings, + boundTransportAddress.publishAddress(), + persistentNodeId, + remoteStoreService + ) ); } else { localNode.set(DiscoveryNode.createLocal(settings, boundTransportAddress.publishAddress(), persistentNodeId)); diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java index d3fbb0cfce09d..5d6c632b3a499 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinHelperTests.java @@ -34,7 +34,7 @@ import org.apache.logging.log4j.Level; import org.opensearch.Version; import org.opensearch.action.ActionListenerResponseHandler; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.action.support.PlainActionFuture; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; @@ -509,7 +509,7 @@ private TestClusterSetup getTestClusterSetup(Version version, boolean isCapturin return new TestClusterSetup(deterministicTaskQueue, localNode, transportService, localClusterState, joinHelper, capturingTransport); } - private RemoteStoreService buildRemoteStoreService(TransportService transportService, ThreadPool threadPool) { + private RemoteStoreNodeService buildRemoteStoreService(TransportService transportService, ThreadPool threadPool) { RepositoriesService repositoriesService = new RepositoriesService( Settings.EMPTY, mock(ClusterService.class), @@ -518,7 +518,7 @@ private RemoteStoreService buildRemoteStoreService(TransportService transportSer Collections.emptyMap(), threadPool ); - return new RemoteStoreService(new SetOnce<>(repositoriesService)::get, threadPool); + return new RemoteStoreNodeService(new SetOnce<>(repositoriesService)::get, threadPool); } private static class TestClusterSetup { diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java index a886bf172c508..fc66f9bd45dda 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java @@ -33,7 +33,7 @@ import org.opensearch.LegacyESVersion; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskExecutor; @@ -54,6 +54,7 @@ import org.opensearch.common.UUIDs; import org.opensearch.common.settings.Settings; import org.opensearch.repositories.RepositoriesService; +import org.opensearch.repositories.blobstore.BlobStoreRepository; import org.opensearch.test.OpenSearchTestCase; import org.opensearch.test.VersionUtils; @@ -187,7 +188,7 @@ public void testUpdatesNodeWithNewRoles() throws Exception { final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - final RemoteStoreService remoteStoreService = mock(RemoteStoreService.class); + final RemoteStoreNodeService remoteStoreService = mock(RemoteStoreNodeService.class); final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( Settings.EMPTY, @@ -291,7 +292,7 @@ public void testJoinFailedForDecommissionedNode() throws Exception { final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - final RemoteStoreService remoteStoreService = mock(RemoteStoreService.class); + final RemoteStoreNodeService remoteStoreService = mock(RemoteStoreNodeService.class); final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( Settings.EMPTY, @@ -427,7 +428,7 @@ public void testPreventJoinClusterWithRemoteStoreNodeWithDifferentAttributesJoin if (nodeAttribute.getKey() != REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY && nodeAttribute.getKey() != REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY) { remoteStoreNodeAttributes.put(nodeAttribute.getKey(), nodeAttribute.getValue() + "-new"); - validateAttributes(remoteStoreNodeAttributes, nodeAttribute, currentState, existingNode); + validateAttributes(remoteStoreNodeAttributes, currentState, existingNode); remoteStoreNodeAttributes.put(nodeAttribute.getKey(), nodeAttribute.getValue()); } } @@ -449,10 +450,10 @@ public void testPreventJoinClusterWithRemoteStoreNodeWithDifferentNameAttributes for (Map.Entry nodeAttribute : existingNodeAttributes.entrySet()) { if (REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY.equals(nodeAttribute.getKey())) { Map remoteStoreNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO + "new", TRANSLOG_REPO); - validateAttributes(remoteStoreNodeAttributes, nodeAttribute, currentState, existingNode); + validateAttributes(remoteStoreNodeAttributes, currentState, existingNode); } else if (REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY.equals(nodeAttribute.getKey())) { Map remoteStoreNodeAttributes = remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO + "new"); - validateAttributes(remoteStoreNodeAttributes, nodeAttribute, currentState, existingNode); + validateAttributes(remoteStoreNodeAttributes, currentState, existingNode); } } } @@ -511,7 +512,10 @@ public void testUpdatesClusterStateWithSingleNodeCluster() throws Exception { final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - final RemoteStoreService remoteStoreService = new RemoteStoreService(new SetOnce<>(mock(RepositoriesService.class))::get, null); + final RemoteStoreNodeService remoteStoreService = new RemoteStoreNodeService( + new SetOnce<>(mock(RepositoriesService.class))::get, + null + ); final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( Settings.EMPTY, @@ -553,7 +557,10 @@ public void testUpdatesClusterStateWithMultiNodeCluster() throws Exception { final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - final RemoteStoreService remoteStoreService = new RemoteStoreService(new SetOnce<>(mock(RepositoriesService.class))::get, null); + final RemoteStoreNodeService remoteStoreService = new RemoteStoreNodeService( + new SetOnce<>(mock(RepositoriesService.class))::get, + null + ); final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( Settings.EMPTY, @@ -613,7 +620,10 @@ public void testUpdatesClusterStateWithSingleNodeClusterAndSameRepository() thro final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - final RemoteStoreService remoteStoreService = new RemoteStoreService(new SetOnce<>(mock(RepositoriesService.class))::get, null); + final RemoteStoreNodeService remoteStoreService = new RemoteStoreNodeService( + new SetOnce<>(mock(RepositoriesService.class))::get, + null + ); final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( Settings.EMPTY, @@ -655,7 +665,10 @@ public void testUpdatesClusterStateWithMultiNodeClusterAndSameRepository() throw final AllocationService allocationService = mock(AllocationService.class); when(allocationService.adaptAutoExpandReplicas(any())).then(invocationOnMock -> invocationOnMock.getArguments()[0]); final RerouteService rerouteService = (reason, priority, listener) -> listener.onResponse(null); - final RemoteStoreService remoteStoreService = new RemoteStoreService(new SetOnce<>(mock(RepositoriesService.class))::get, null); + final RemoteStoreNodeService remoteStoreService = new RemoteStoreNodeService( + new SetOnce<>(mock(RepositoriesService.class))::get, + null + ); final JoinTaskExecutor joinTaskExecutor = new JoinTaskExecutor( Settings.EMPTY, @@ -782,12 +795,7 @@ private Map remoteStoreNodeAttributes(String segmentRepoName, St }; } - private void validateAttributes( - Map remoteStoreNodeAttributes, - Map.Entry existingNodeAttribute, - ClusterState currentState, - DiscoveryNode existingNode - ) { + private void validateAttributes(Map remoteStoreNodeAttributes, ClusterState currentState, DiscoveryNode existingNode) { DiscoveryNode joiningNode = newDiscoveryNode(remoteStoreNodeAttributes); Exception e = assertThrows( IllegalStateException.class, @@ -820,6 +828,8 @@ private RepositoryMetadata buildRepositoryMetadata(DiscoveryNode node, String na Settings.Builder settings = Settings.builder(); settingsMap.entrySet().forEach(entry -> settings.put(entry.getKey(), entry.getValue())); + settings.put(BlobStoreRepository.SYSTEM_REPOSITORY_SETTING.getKey(), true); + return new RepositoryMetadata(name, type, settings.build()); } } diff --git a/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java index 783379fd20c7e..e697b29aceffb 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java @@ -33,7 +33,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterName; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.OpenSearchAllocationTestCase; @@ -264,7 +264,7 @@ protected void onSendRequest( (s, p, r) -> {}, ElectionStrategy.DEFAULT_INSTANCE, nodeHealthService, - Mockito.mock(RemoteStoreService.class) + Mockito.mock(RemoteStoreNodeService.class) ); transportService.start(); transportService.acceptIncomingRequests(); diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index ad5fbb18138b3..820836f41249d 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -114,6 +114,8 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_BLOCK; @@ -136,11 +138,9 @@ import static org.opensearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.opensearch.indices.ShardLimitValidatorTests.createTestShardLimitService; +import static org.opensearch.node.Node.NODE_ATTRIBUTES; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; @@ -158,6 +158,10 @@ public class MetadataCreateIndexServiceTests extends OpenSearchTestCase { private CreateIndexClusterStateUpdateRequest request; private QueryShardContext queryShardContext; private ClusterSettings clusterSettings; + private static final String segmentRepositoryNameAttributeKey = NODE_ATTRIBUTES.getKey() + + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; + private static final String translogRepositoryNameAttributeKey = NODE_ATTRIBUTES.getKey() + + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; @Before public void setup() throws Exception { @@ -1214,9 +1218,8 @@ public void testvalidateIndexSettings() { public void testRemoteStoreNoUserOverrideExceptReplicationTypeSegmentIndexSettings() { Settings settings = Settings.builder() .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.DOCUMENT) - .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") + .put(segmentRepositoryNameAttributeKey, "my-segment-repo-1") + .put(translogRepositoryNameAttributeKey, "my-translog-repo-1") .build(); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); @@ -1247,9 +1250,8 @@ public void testRemoteStoreNoUserOverrideExceptReplicationTypeSegmentIndexSettin public void testRemoteStoreImplicitOverrideReplicationTypeToSegmentForRemoteStore() { Settings settings = Settings.builder() - .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") + .put(segmentRepositoryNameAttributeKey, "my-segment-repo-1") + .put(translogRepositoryNameAttributeKey, "my-translog-repo-1") .build(); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); @@ -1280,9 +1282,8 @@ public void testRemoteStoreImplicitOverrideReplicationTypeToSegmentForRemoteStor public void testRemoteStoreNoUserOverrideIndexSettings() { Settings settings = Settings.builder() .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) - .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), "my-segment-repo-1") - .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "my-translog-repo-1") + .put(segmentRepositoryNameAttributeKey, "my-segment-repo-1") + .put(translogRepositoryNameAttributeKey, "my-translog-repo-1") .build(); FeatureFlagSetter.set(FeatureFlags.REMOTE_STORE); diff --git a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java index 1ed723117eb71..b5ab1013288f3 100644 --- a/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java +++ b/server/src/test/java/org/opensearch/discovery/DiscoveryModuleTests.java @@ -32,7 +32,7 @@ package org.opensearch.discovery; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.coordination.Coordinator; import org.opensearch.cluster.node.DiscoveryNode; @@ -77,7 +77,7 @@ public class DiscoveryModuleTests extends OpenSearchTestCase { private ClusterSettings clusterSettings; private GatewayMetaState gatewayMetaState; - private RemoteStoreService remoteStoreService; + private RemoteStoreNodeService remoteStoreService; public interface DummyHostsProviderPlugin extends DiscoveryPlugin { Map> impl(); @@ -101,7 +101,7 @@ public void setupDummyServices() { clusterApplier = mock(ClusterApplier.class); clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); gatewayMetaState = mock(GatewayMetaState.class); - remoteStoreService = mock(RemoteStoreService.class); + remoteStoreService = mock(RemoteStoreNodeService.class); } @After diff --git a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java index 06b604152bf11..31bf54e9bcc0c 100644 --- a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java @@ -36,7 +36,7 @@ import org.apache.logging.log4j.Logger; import org.opensearch.ExceptionsHelper; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.action.admin.cluster.reroute.ClusterRerouteRequest; import org.opensearch.action.admin.cluster.reroute.TransportClusterRerouteAction; import org.opensearch.action.admin.indices.close.CloseIndexRequest; @@ -154,7 +154,7 @@ public class ClusterStateChanges { private final TransportClusterRerouteAction transportClusterRerouteAction; private final TransportCreateIndexAction transportCreateIndexAction; private final RepositoriesService repositoriesService; - private final RemoteStoreService remoteStoreService; + private final RemoteStoreNodeService remoteStoreService; private final NodeRemovalClusterStateTaskExecutor nodeRemovalExecutor; private final JoinTaskExecutor joinTaskExecutor; @@ -376,7 +376,7 @@ public IndexMetadata upgradeIndexMetadata(IndexMetadata indexMetadata, Version m threadPool ); - remoteStoreService = new RemoteStoreService(new SetOnce<>(repositoriesService)::get, threadPool); + remoteStoreService = new RemoteStoreNodeService(new SetOnce<>(repositoriesService)::get, threadPool); nodeRemovalExecutor = new NodeRemovalClusterStateTaskExecutor(allocationService, logger); joinTaskExecutor = new JoinTaskExecutor(Settings.EMPTY, allocationService, logger, (s, p, r) -> {}, remoteStoreService); diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java index 0dbc0372458b5..a24fd04d3d4f6 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java @@ -111,7 +111,7 @@ protected void updateRepository(Client client, String repoName, Settings repoSet createRepository(client, repoName, repoSettings); } - protected Settings getRemoteStoreBackedIndexSettings(String remoteStoreRepo) { + protected Settings getRemoteStoreBackedIndexSettings() { return Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, "1") .put("index.refresh_interval", "300s") diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java index d69fb1fd7b349..d59a33b4cbbdd 100644 --- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java +++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryRemoteIndexTests.java @@ -39,24 +39,29 @@ import org.opensearch.common.settings.Settings; import org.opensearch.common.util.FeatureFlags; import org.opensearch.core.index.shard.ShardId; +import org.opensearch.env.Environment; import org.opensearch.index.IndexSettings; import org.opensearch.index.snapshots.blobstore.RemoteStoreShardShallowCopySnapshot; import org.opensearch.indices.replication.common.ReplicationType; import org.opensearch.repositories.IndexId; import org.opensearch.repositories.RepositoriesService; import org.opensearch.repositories.RepositoryData; +import org.opensearch.repositories.fs.FsRepository; import org.opensearch.snapshots.SnapshotId; import org.opensearch.test.FeatureFlagSetter; import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; +import java.nio.file.Path; import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.stream.Collectors; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_ENABLED_SETTING; -import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY; +import static org.opensearch.action.admin.cluster.remotestore.RemoteStoreNode.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY; import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING; import static org.hamcrest.Matchers.equalTo; @@ -71,12 +76,36 @@ protected Settings featureFlagSettings() { @Override protected Settings nodeSettings() { + Path tempDir = createTempDir(); return Settings.builder() .put(super.nodeSettings()) .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT) - .put(CLUSTER_REMOTE_STORE_ENABLED_SETTING.getKey(), true) - .put(CLUSTER_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), "test-rs-repo") - .put(CLUSTER_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "test-rs-repo") + .put(buildRemoteStoreNodeAttributes("test-rs-repo", tempDir.resolve("repo"))) + .put(Environment.PATH_HOME_SETTING.getKey(), tempDir) + .put(Environment.PATH_REPO_SETTING.getKey(), tempDir.resolve("repo")) + .put(Environment.PATH_SHARED_DATA_SETTING.getKey(), tempDir.getParent()) + .build(); + } + + public static Settings buildRemoteStoreNodeAttributes(String repoName, Path repoPath) { + String repoTypeAttributeKey = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT, + repoName + ); + String repoSettingsAttributeKeyPrefix = String.format( + Locale.getDefault(), + "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX, + repoName + ); + + return Settings.builder() + .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName) + .put(repoTypeAttributeKey, FsRepository.TYPE) + .put(repoSettingsAttributeKeyPrefix + "location", repoPath) + .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, repoName) + .put(repoTypeAttributeKey, FsRepository.TYPE) + .put(repoSettingsAttributeKeyPrefix + "location", repoPath) .build(); } @@ -95,13 +124,6 @@ public void testRetrieveShallowCopySnapshotCase1() throws IOException { .build(); createRepository(client, snapshotRepositoryName, snapshotRepoSettings); - logger.info("--> creating remote store repository"); - Settings remoteStoreRepoSettings = Settings.builder() - .put(node().settings()) - .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) - .build(); - createRepository(client, remoteStoreRepositoryName, remoteStoreRepoSettings); - logger.info("--> creating an index and indexing documents"); final String indexName = "test-idx"; createIndex(indexName); @@ -110,7 +132,7 @@ public void testRetrieveShallowCopySnapshotCase1() throws IOException { logger.info("--> creating a remote store enabled index and indexing documents"); final String remoteStoreIndexName = "test-rs-idx"; - Settings indexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepositoryName); + Settings indexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreIndexName, indexSettings); indexDocuments(client, remoteStoreIndexName); @@ -195,16 +217,9 @@ public void testGetRemoteStoreShallowCopyShardMetadata() throws IOException { .build(); createRepository(client, snapshotRepositoryName, snapshotRepoSettings); - logger.info("--> creating remote store repository"); - Settings remoteStoreRepoSettings = Settings.builder() - .put(node().settings()) - .put("location", OpenSearchIntegTestCase.randomRepoPath(node().settings())) - .build(); - createRepository(client, remoteStoreRepositoryName, remoteStoreRepoSettings); - logger.info("--> creating a remote store enabled index and indexing documents"); final String remoteStoreIndexName = "test-rs-idx"; - Settings indexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepositoryName); + Settings indexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreIndexName, indexSettings); indexDocuments(client, remoteStoreIndexName); @@ -266,9 +281,6 @@ public void testRetrieveShallowCopySnapshotCase2() throws IOException { assertFalse(updatedRepositoryMetadata.settings().getAsBoolean(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), false)); - logger.info("--> creating remote store repository"); - createRepository(client, remoteStoreRepositoryName); - logger.info("--> creating an index and indexing documents"); final String indexName = "test-idx"; createIndex(indexName); @@ -277,7 +289,7 @@ public void testRetrieveShallowCopySnapshotCase2() throws IOException { logger.info("--> creating a remote store enabled index and indexing documents"); final String remoteStoreIndexName = "test-rs-idx"; - Settings indexSettings = getRemoteStoreBackedIndexSettings(remoteStoreRepositoryName); + Settings indexSettings = getRemoteStoreBackedIndexSettings(); createIndex(remoteStoreIndexName, indexSettings); indexDocuments(client, remoteStoreIndexName); diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index e904765fe89f0..1dbb547e5717d 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -40,7 +40,7 @@ import org.opensearch.action.ActionType; import org.opensearch.action.RequestValidators; import org.opensearch.action.StepListener; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryAction; import org.opensearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryRequest; import org.opensearch.action.admin.cluster.repositories.cleanup.CleanupRepositoryResponse; @@ -1891,7 +1891,7 @@ private final class TestClusterNode { private final ClusterInfoService clusterInfoService; private Coordinator coordinator; - private RemoteStoreService remoteStoreService; + private RemoteStoreNodeService remoteStoreService; private Map actions = new HashMap<>(); @@ -1997,7 +1997,7 @@ public void onFailure(final Exception e) { emptyMap(), threadPool ); - remoteStoreService = new RemoteStoreService(new SetOnce<>(repositoriesService)::get, threadPool); + remoteStoreService = new RemoteStoreNodeService(new SetOnce<>(repositoriesService)::get, threadPool); final ActionFilters actionFilters = new ActionFilters(emptySet()); snapshotsService = new SnapshotsService( settings, diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java index 0c677044c033c..bb2bef7912c7b 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java @@ -39,7 +39,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.opensearch.OpenSearchException; import org.opensearch.Version; -import org.opensearch.action.admin.cluster.remotestore.RemoteStoreService; +import org.opensearch.action.admin.cluster.remotestore.RemoteStoreNodeService; import org.opensearch.cluster.ClusterModule; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.ClusterStateTaskListener; @@ -1037,7 +1037,7 @@ class ClusterNode { private DisruptableMockTransport mockTransport; private NodeHealthService nodeHealthService; private RepositoriesService repositoriesService; - private RemoteStoreService remoteStoreService; + private RemoteStoreNodeService remoteStoreService; List> extraJoinValidators = new ArrayList<>(); ClusterNode(int nodeIndex, boolean clusterManagerEligible, Settings nodeSettings, NodeHealthService nodeHealthService) { @@ -1138,7 +1138,7 @@ protected Optional getDisruptableMockTransport(Transpo Collections.emptyMap(), threadPool ); - remoteStoreService = new RemoteStoreService(new SetOnce<>(repositoriesService)::get, threadPool); + remoteStoreService = new RemoteStoreNodeService(new SetOnce<>(repositoriesService)::get, threadPool); final Collection> onJoinValidators = Collections.singletonList( (dn, cs) -> extraJoinValidators.forEach(validator -> validator.accept(dn, cs)) ); diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java index 11e847e29a097..95832dc9544ce 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java @@ -55,6 +55,7 @@ import org.opensearch.cluster.coordination.ClusterBootstrapService; import org.opensearch.cluster.coordination.NoClusterManagerBlockService; import org.opensearch.cluster.metadata.IndexMetadata; +import org.opensearch.cluster.metadata.RepositoriesMetadata; import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.node.DiscoveryNodeRole; import org.opensearch.cluster.node.DiscoveryNodes; @@ -1318,6 +1319,12 @@ public synchronized void validateClusterFormed() { assertTrue("Expected node to exist: " + expectedNode + debugString, discoveryNodes.nodeExists(expectedNode)); } }); + states.forEach(cs -> { + if (cs.nodes().getNodes().values().stream().findFirst().get().isRemoteStoreNode()) { + RepositoriesMetadata repositoriesMetadata = cs.metadata().custom(RepositoriesMetadata.TYPE); + assertTrue(repositoriesMetadata != null && !repositoriesMetadata.repositories().isEmpty()); + } + }); }, 30, TimeUnit.SECONDS); } catch (AssertionError ae) { throw new IllegalStateException("cluster failed to form", ae);