diff --git a/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java b/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java index d8dc1a66a6ef9..6faab8b91a81a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/gateway/MetadataNodesIT.java @@ -23,6 +23,7 @@ import org.elasticsearch.test.InternalTestCluster.RestartCallback; import java.nio.file.Files; +import java.nio.file.Path; import java.util.List; import java.util.Map; @@ -179,7 +180,12 @@ protected void assertIndexInMetaState(final String nodeName, final String indexN private boolean indexDirectoryExists(String nodeName, Index index) { NodeEnvironment nodeEnv = ((InternalTestCluster) cluster()).getInstance(NodeEnvironment.class, nodeName); - return Files.exists(nodeEnv.indexPath(index)); + for (Path path : nodeEnv.indexPaths(index)) { + if (Files.exists(path)) { + return true; + } + } + return false; } private ImmutableOpenMap getIndicesMetadataOnNode(String nodeName) { diff --git a/server/src/internalClusterTest/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java b/server/src/internalClusterTest/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java index 6f87758ef02e0..52e975df396d5 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/indices/store/IndicesStoreIntegrationIT.java @@ -452,7 +452,9 @@ public void onFailure(Exception e) { private Path indexDirectory(String server, Index index) { NodeEnvironment env = internalCluster().getInstance(NodeEnvironment.class, server); - return env.indexPath(index); + final Path[] paths = env.indexPaths(index); + assert paths.length == 1; + return paths[0]; } private Path shardDirectory(String server, Index index, int shard) { diff --git a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java index cb8aee7b8a82c..f5e1083d21ffd 100644 --- a/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java +++ b/server/src/main/java/org/elasticsearch/env/NodeEnvironment.java @@ -694,10 +694,10 @@ public void deleteIndexDirectorySafe( * @param indexSettings settings for the index being deleted */ public void deleteIndexDirectoryUnderLock(Index index, IndexSettings indexSettings, Consumer listener) throws IOException { - final Path indexPath = indexPath(index); - logger.trace("deleting index {} directory: [{}]", index, indexPath); - listener.accept(new Path[] { indexPath }); - IOUtils.rm(indexPath); + final Path[] indexPaths = indexPaths(index); + logger.trace("deleting index {} directory, paths({}): [{}]", index, indexPaths.length, indexPaths); + listener.accept(indexPaths); + IOUtils.rm(indexPaths); if (indexSettings.hasCustomDataPath()) { Path customLocation = resolveIndexCustomLocation(indexSettings.customDataPath(), index.getUUID()); logger.trace("deleting custom index {} directory [{}]", index, customLocation); @@ -941,9 +941,13 @@ public NodePath[] nodePaths() { /** * Returns all index paths. */ - public Path indexPath(Index index) { + public Path[] indexPaths(Index index) { assertEnvIsLocked(); - return nodePaths[0].resolve(index); + Path[] indexPaths = new Path[nodePaths.length]; + for (int i = 0; i < nodePaths.length; i++) { + indexPaths[i] = nodePaths[i].resolve(index); + } + return indexPaths; } diff --git a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java index c630b1e5f1f53..5c4c7992de34b 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetaStateService.java @@ -142,7 +142,7 @@ private Tuple loadFullStateBWC() throws IOException { */ @Nullable public IndexMetadata loadIndexState(Index index) throws IOException { - return INDEX_METADATA_FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.indexPath(index)); + return INDEX_METADATA_FORMAT.loadLatestState(logger, namedXContentRegistry, nodeEnv.indexPaths(index)); } /** @@ -205,7 +205,7 @@ public long writeIndex(String reason, IndexMetadata indexMetadata) throws WriteS logger.trace("[{}] writing state, reason [{}]", index, reason); try { long generation = INDEX_METADATA_FORMAT.write(indexMetadata, - nodeEnv.indexPath(indexMetadata.getIndex())); + nodeEnv.indexPaths(indexMetadata.getIndex())); logger.trace("[{}] state written", index); return generation; } catch (WriteStateException ex) { @@ -246,7 +246,7 @@ void cleanupGlobalState(long currentGeneration) { * @param currentGeneration current state generation to keep in the index directory. */ public void cleanupIndex(Index index, long currentGeneration) { - INDEX_METADATA_FORMAT.cleanupOldFiles(currentGeneration, nodeEnv.indexPath(index)); + INDEX_METADATA_FORMAT.cleanupOldFiles(currentGeneration, nodeEnv.indexPaths(index)); } /** diff --git a/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java b/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java index 7215b1bcdcd7e..2280cb4e20d8b 100644 --- a/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java +++ b/server/src/main/java/org/elasticsearch/gateway/MetadataStateFormat.java @@ -318,7 +318,7 @@ protected Directory newDirectory(Path dir) throws IOException { * @param currentGeneration state generation to keep. * @param locations state paths. */ - public void cleanupOldFiles(final long currentGeneration, Path... locations) { + public void cleanupOldFiles(final long currentGeneration, Path[] locations) { final String fileNameToKeep = getStateFileName(currentGeneration); for (Path location : locations) { logger.trace("cleanupOldFiles: cleaning up {}", location); diff --git a/server/src/main/java/org/elasticsearch/index/IndexService.java b/server/src/main/java/org/elasticsearch/index/IndexService.java index 0e1e68f5c3e75..6da918e6bd3e0 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexService.java +++ b/server/src/main/java/org/elasticsearch/index/IndexService.java @@ -346,7 +346,7 @@ public synchronized void writeDanglingIndicesInfo() { return; } try { - IndexMetadata.FORMAT.writeAndCleanup(getMetadata(), nodeEnv.indexPath(index())); + IndexMetadata.FORMAT.writeAndCleanup(getMetadata(), nodeEnv.indexPaths(index())); } catch (WriteStateException e) { logger.warn(() -> new ParameterizedMessage("failed to write dangling indices state for index {}", index()), e); } @@ -358,7 +358,7 @@ public synchronized void deleteDanglingIndicesInfo() { return; } try { - MetadataStateFormat.deleteMetaState(nodeEnv.indexPath(index())); + MetadataStateFormat.deleteMetaState(nodeEnv.indexPaths(index())); } catch (IOException e) { logger.warn(() -> new ParameterizedMessage("failed to delete dangling indices state for index {}", index()), e); } diff --git a/server/src/main/java/org/elasticsearch/indices/IndicesService.java b/server/src/main/java/org/elasticsearch/indices/IndicesService.java index 227526b51922f..ab81722755874 100644 --- a/server/src/main/java/org/elasticsearch/indices/IndicesService.java +++ b/server/src/main/java/org/elasticsearch/indices/IndicesService.java @@ -940,7 +940,7 @@ private void deleteIndexStoreIfDeletionAllowed(final String reason, final Index } // this is a pure protection to make sure this index doesn't get re-imported as a dangling index. // we should in the future rather write a tombstone rather than wiping the metadata. - MetadataStateFormat.deleteMetaState(nodeEnv.indexPath(index)); + MetadataStateFormat.deleteMetaState(nodeEnv.indexPaths(index)); } } @@ -1029,7 +1029,7 @@ public IndexMetadata verifyIndexIsDeleted(final Index index, final ClusterState if (clusterState.metadata().index(index) != null) { throw new IllegalStateException("Cannot delete index [" + index + "], it is still part of the cluster state."); } - if (nodeEnv.hasNodeFile() && Files.exists(nodeEnv.indexPath(index))) { + if (nodeEnv.hasNodeFile() && FileSystemUtils.exists(nodeEnv.indexPaths(index))) { final IndexMetadata metadata; try { metadata = metaStateService.loadIndexState(index); diff --git a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java index 7660c5b7f3c6e..27b65de3c5ba4 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java +++ b/server/src/test/java/org/elasticsearch/env/NodeEnvironmentTests.java @@ -102,9 +102,10 @@ public void testShardLock() throws Exception { } catch (ShardLockObtainFailedException ex) { // expected } - Path path = env.indexPath(index); - Files.createDirectories(path.resolve("0")); - Files.createDirectories(path.resolve("1")); + for (Path path : env.indexPaths(index)) { + Files.createDirectories(path.resolve("0")); + Files.createDirectories(path.resolve("1")); + } try { env.lockAllForIndex(index, idxSettings, "3", randomIntBetween(0, 10)); fail("shard 0 is locked"); @@ -134,9 +135,10 @@ public void testAvailableIndexFolders() throws Exception { Set actualPaths = new HashSet<>(); for (int i = 0; i < numIndices; i++) { Index index = new Index("foo" + i, "fooUUID" + i); - Path path = env.indexPath(index); - Files.createDirectories(path.resolve(MetadataStateFormat.STATE_DIR_NAME)); - actualPaths.add(path.getFileName().toString()); + for (Path path : env.indexPaths(index)) { + Files.createDirectories(path.resolve(MetadataStateFormat.STATE_DIR_NAME)); + actualPaths.add(path.getFileName().toString()); + } } assertThat(actualPaths, equalTo(env.availableIndexFolders())); @@ -151,11 +153,12 @@ public void testAvailableIndexFoldersWithExclusions() throws Exception { Set actualPaths = new HashSet<>(); for (int i = 0; i < numIndices; i++) { Index index = new Index("foo" + i, "fooUUID" + i); - Path path = env.indexPath(index); - Files.createDirectories(path.resolve(MetadataStateFormat.STATE_DIR_NAME)); - actualPaths.add(path.getFileName().toString()); + for (Path path : env.indexPaths(index)) { + Files.createDirectories(path.resolve(MetadataStateFormat.STATE_DIR_NAME)); + actualPaths.add(path.getFileName().toString()); + } if (randomBoolean()) { - excludedPaths.add(env.indexPath(index).getFileName().toString()); + excludedPaths.add(env.indexPaths(index)[0].getFileName().toString()); } } @@ -170,15 +173,17 @@ public void testResolveIndexFolders() throws Exception { Map> actualIndexDataPaths = new HashMap<>(); for (int i = 0; i < numIndices; i++) { Index index = new Index("foo" + i, "fooUUID" + i); - Path path = env.indexPath(index); - Files.createDirectories(path); - String fileName = path.getFileName().toString(); - List paths = actualIndexDataPaths.get(fileName); - if (paths == null) { - paths = new ArrayList<>(); + Path[] indexPaths = env.indexPaths(index); + for (Path path : indexPaths) { + Files.createDirectories(path); + String fileName = path.getFileName().toString(); + List paths = actualIndexDataPaths.get(fileName); + if (paths == null) { + paths = new ArrayList<>(); + } + paths.add(path); + actualIndexDataPaths.put(fileName, paths); } - paths.add(path); - actualIndexDataPaths.put(fileName, paths); } for (Map.Entry> actualIndexDataPathEntry : actualIndexDataPaths.entrySet()) { List actual = actualIndexDataPathEntry.getValue(); @@ -195,18 +200,20 @@ public void testDeleteSafe() throws Exception { final ShardLock fooLock = env.shardLock(new ShardId(index, 0), "1"); assertEquals(new ShardId(index, 0), fooLock.getShardId()); - Path path = env.indexPath(index); - Files.createDirectories(path.resolve("0")); - Files.createDirectories(path.resolve("1")); + for (Path path : env.indexPaths(index)) { + Files.createDirectories(path.resolve("0")); + Files.createDirectories(path.resolve("1")); + } expectThrows(ShardLockObtainFailedException.class, () -> env.deleteShardDirectorySafe(new ShardId(index, 0), idxSettings, shardPaths -> { assert false : "should not be called " + shardPaths; })); - path = env.indexPath(index); - assertTrue(Files.exists(path.resolve("0"))); - assertTrue(Files.exists(path.resolve("1"))); + for (Path path : env.indexPaths(index)) { + assertTrue(Files.exists(path.resolve("0"))); + assertTrue(Files.exists(path.resolve("1"))); + } { SetOnce listener = new SetOnce<>(); @@ -217,9 +224,10 @@ public void testDeleteSafe() throws Exception { } } - path = env.indexPath(index); - assertTrue(Files.exists(path.resolve("0"))); - assertFalse(Files.exists(path.resolve("1"))); + for (Path path : env.indexPaths(index)) { + assertTrue(Files.exists(path.resolve("0"))); + assertFalse(Files.exists(path.resolve("1"))); + } expectThrows(ShardLockObtainFailedException.class, () -> env.deleteIndexDirectorySafe(index, randomIntBetween(0, 10), idxSettings, indexPaths -> { @@ -228,8 +236,9 @@ public void testDeleteSafe() throws Exception { fooLock.close(); - path = env.indexPath(index); - assertTrue(Files.exists(path)); + for (Path path : env.indexPaths(index)) { + assertTrue(Files.exists(path)); + } final AtomicReference threadException = new AtomicReference<>(); final CountDownLatch latch = new CountDownLatch(1); @@ -265,11 +274,12 @@ protected void doRun() throws Exception { final SetOnce listener = new SetOnce<>(); env.deleteIndexDirectorySafe(index, 5000, idxSettings, listener::set); - assertThat(listener.get()[0], equalTo(env.indexPath(index))); + assertArrayEquals(env.indexPaths(index), listener.get()); assertNull(threadException.get()); - path = env.indexPath(index); - assertFalse(Files.exists(path)); + for (Path path : env.indexPaths(index)) { + assertFalse(Files.exists(path)); + } latch.await(); assertTrue("LockedShards: " + env.lockedShards(), env.lockedShards().isEmpty()); env.close(); @@ -351,7 +361,7 @@ public void testCustomDataPaths() throws Exception { equalTo(dataPath.resolve("indices/" + index.getUUID() + "/0"))); assertThat("index paths uses the regular template", - env.indexPath(index), equalTo(dataPath.resolve("indices/" + index.getUUID()))); + env.indexPaths(index)[0], equalTo(dataPath.resolve("indices/" + index.getUUID()))); assertThat(env.availableShardPaths(sid), equalTo(env.availableShardPaths(sid))); assertThat(env.resolveCustomLocation("/tmp/foo", sid).toAbsolutePath(), @@ -362,7 +372,7 @@ public void testCustomDataPaths() throws Exception { equalTo(dataPath.resolve("indices/" + index.getUUID() + "/0"))); assertThat("index paths uses the regular template", - env.indexPath(index), equalTo(dataPath.resolve("indices/" + index.getUUID()))); + env.indexPaths(index)[0], equalTo(dataPath.resolve("indices/" + index.getUUID()))); env.close(); } @@ -405,8 +415,10 @@ public void testEnsureNoShardDataOrIndexMetadata() throws IOException { Path indexPath; try (NodeEnvironment env = newNodeEnvironment(settings)) { - indexPath = env.indexPath(index); - Files.createDirectories(indexPath.resolve(MetadataStateFormat.STATE_DIR_NAME)); + for (Path path : env.indexPaths(index)) { + Files.createDirectories(path.resolve(MetadataStateFormat.STATE_DIR_NAME)); + } + indexPath = env.indexPaths(index)[0]; } verifyFailsOnMetadata(noDataNoMasterSettings, indexPath); @@ -418,7 +430,9 @@ public void testEnsureNoShardDataOrIndexMetadata() throws IOException { // test that we can create data=false env with only meta information. Also create shard data for following asserts try (NodeEnvironment env = newNodeEnvironment(noDataSettings)) { - Files.createDirectories(env.indexPath(index).resolve(shardDataDirName)); + for (Path path : env.indexPaths(index)) { + Files.createDirectories(path.resolve(shardDataDirName)); + } } verifyFailsOnShardData(noDataSettings, indexPath, shardDataDirName); @@ -434,7 +448,9 @@ public void testEnsureNoShardDataOrIndexMetadata() throws IOException { // test that we can create data=true, master=true env. Also remove state dir to leave only shard data for following asserts try (NodeEnvironment env = newNodeEnvironment(settings)) { - Files.delete(env.indexPath(index).resolve(MetadataStateFormat.STATE_DIR_NAME)); + for (Path path : env.indexPaths(index)) { + Files.delete(path.resolve(MetadataStateFormat.STATE_DIR_NAME)); + } } // assert that we fail on shard data even without the metadata dir. diff --git a/server/src/test/java/org/elasticsearch/env/NodeRepurposeCommandTests.java b/server/src/test/java/org/elasticsearch/env/NodeRepurposeCommandTests.java index 03f47df85a199..88995d8982471 100644 --- a/server/src/test/java/org/elasticsearch/env/NodeRepurposeCommandTests.java +++ b/server/src/test/java/org/elasticsearch/env/NodeRepurposeCommandTests.java @@ -227,10 +227,11 @@ private void createIndexDataFiles(Settings settings, int shardCount, boolean wri .build()); } } - Path path = env.indexPath(INDEX); - for (int i = 0; i < shardCount; ++i) { - Files.createDirectories(path.resolve(Integer.toString(shardDataDirNumber))); - shardDataDirNumber += randomIntBetween(1,10); + for (Path path : env.indexPaths(INDEX)) { + for (int i = 0; i < shardCount; ++i) { + Files.createDirectories(path.resolve(Integer.toString(shardDataDirNumber))); + shardDataDirNumber += randomIntBetween(1,10); + } } } } diff --git a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java index d07f018a80b29..441063c1a0fdf 100644 --- a/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java +++ b/server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java @@ -24,6 +24,7 @@ import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.Strings; import org.elasticsearch.common.UUIDs; +import org.elasticsearch.common.io.FileSystemUtils; import org.elasticsearch.common.settings.Setting; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.core.TimeValue; @@ -62,7 +63,6 @@ import org.elasticsearch.test.hamcrest.RegexMatcher; import java.io.IOException; -import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -351,7 +351,7 @@ public void testVerifyIfIndexContentDeleted() throws Exception { .metadata(Metadata.builder(csWithIndex.metadata()).remove(index.getName())) .build(); indicesService.verifyIndexIsDeleted(index, withoutIndex); - assertFalse("index files should be deleted", Files.exists(nodeEnv.indexPath(index))); + assertFalse("index files should be deleted", FileSystemUtils.exists(nodeEnv.indexPaths(index))); } public void testDanglingIndicesWithAliasConflict() throws Exception {