-
Notifications
You must be signed in to change notification settings - Fork 1.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Include file cache stats in node stats
And add an event listener to clean up cache entries when the index is deleted Signed-off-by: Rabi Panda <adnapibar@gmail.com>
- Loading branch information
Showing
16 changed files
with
493 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
87 changes: 87 additions & 0 deletions
87
server/src/main/java/org/opensearch/index/store/remote/filecache/FileCacheCleaner.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.index.store.remote.filecache; | ||
|
|
||
| import org.apache.logging.log4j.LogManager; | ||
| import org.apache.logging.log4j.Logger; | ||
| import org.apache.logging.log4j.message.ParameterizedMessage; | ||
| import org.apache.logging.log4j.util.Supplier; | ||
| import org.opensearch.common.settings.Settings; | ||
| import org.opensearch.common.xcontent.NamedXContentRegistry; | ||
| import org.opensearch.env.NodeEnvironment; | ||
| import org.opensearch.index.IndexModule; | ||
| import org.opensearch.index.shard.IndexEventListener; | ||
| import org.opensearch.index.shard.ShardId; | ||
| import org.opensearch.index.shard.ShardStateMetadata; | ||
|
|
||
| import java.io.IOException; | ||
| import java.nio.file.DirectoryStream; | ||
| import java.nio.file.Files; | ||
| import java.nio.file.Path; | ||
|
|
||
| import static org.opensearch.index.store.remote.directory.RemoteSnapshotDirectoryFactory.LOCAL_STORE_LOCATION; | ||
|
|
||
| /** | ||
| * IndexEventListener to clean up file cache when the index is deleted | ||
| * | ||
| * @opensearch.internal | ||
| */ | ||
| public class FileCacheCleaner implements IndexEventListener { | ||
| private static final Logger log = LogManager.getLogger(FileCacheCleaner.class); | ||
|
|
||
| private final NodeEnvironment nodeEnvironment; | ||
| private final FileCache fileCache; | ||
|
|
||
| public FileCacheCleaner(NodeEnvironment nodeEnvironment, FileCache fileCache) { | ||
| this.nodeEnvironment = nodeEnvironment; | ||
| this.fileCache = fileCache; | ||
| } | ||
|
|
||
| private Path getShardPath(ShardId shardId) throws IOException { | ||
| final Path[] paths = nodeEnvironment.availableShardPaths(shardId); | ||
| for (Path path : paths) { | ||
| ShardStateMetadata metadata = ShardStateMetadata.FORMAT.loadLatestState(log, NamedXContentRegistry.EMPTY, path); | ||
| if (metadata != null) { | ||
| return path; | ||
| } | ||
| } | ||
| return null; | ||
| } | ||
|
|
||
| /** | ||
| * before shard deleted and after shard closed, cleans up the corresponding index file path entries from FC. | ||
| * @param shardId The shard id | ||
| * @param settings the shards index settings | ||
| */ | ||
| @Override | ||
| public void beforeIndexShardDeleted(ShardId shardId, Settings settings) { | ||
| try { | ||
| String storeType = settings.get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey()); | ||
| if (IndexModule.Type.REMOTE_SNAPSHOT.match(storeType)) { | ||
| Path localShardPath = getShardPath(shardId); | ||
| if (localShardPath != null) { | ||
| Path localStorePath = localShardPath.resolve(LOCAL_STORE_LOCATION); | ||
| try (DirectoryStream<Path> ds = Files.newDirectoryStream(localStorePath)) { | ||
| for (Path subPath : ds) { | ||
| fileCache.remove(subPath.toRealPath()); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } catch (IOException ioe) { | ||
| log.error( | ||
| (Supplier<?>) () -> new ParameterizedMessage( | ||
| "thrown an exception when removing items from file cache for being deleted shard {})", | ||
| shardId | ||
| ), | ||
| ioe | ||
| ); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.