Add support for custom remote store segment path prefix to support clusterless configurations

CHANGELOG.md

@@ -46,6 +46,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Expand fetch phase profiling to multi-shard queries ([#18887](https://github.com/opensearch-project/OpenSearch/pull/18887))
 - Prevent shard initialization failure due to streaming consumer errors ([#18877](https://github.com/opensearch-project/OpenSearch/pull/18877))
 - APIs for stream transport and new stream-based search api action ([#18722](https://github.com/opensearch-project/OpenSearch/pull/18722))
+- Add support for custom remote store segment path prefix to support clusterless configurations ([#18750](https://github.com/opensearch-project/OpenSearch/issues/18750))
 - Added the core process for warming merged segments in remote-store enabled domains ([#18683](https://github.com/opensearch-project/OpenSearch/pull/18683))
 - Optimize Composite Aggregations by removing unnecessary object allocations ([#18531](https://github.com/opensearch-project/OpenSearch/pull/18531))
 - [Star-Tree] Add search support for ip field type ([#18671](https://github.com/opensearch-project/OpenSearch/pull/18671))

server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java

@@ -458,6 +458,57 @@ public Iterator<Setting<?>> settings() {
         Property.Dynamic
     );
 
+    /**
+     * Used to specify a custom path prefix for remote store segments. This allows injecting a unique identifier
+     * (e.g., writer node ID) into the remote store path to support clusterless configurations where multiple
+     * writers may write to the same shard.
+     */
+    public static final Setting<String> INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX = Setting.simpleString(
+        "index.remote_store.segment.path_prefix",
+        "",
+        new Setting.Validator<>() {
+
+            @Override
+            public void validate(final String value) {}
+
+            @Override
+            public void validate(final String value, final Map<Setting<?>, Object> settings) {
+                // Only validate if the value is not null and not empty
+                if (value != null && !value.trim().isEmpty()) {
+                    // Validate that remote store is enabled when this setting is used
+                    final Boolean isRemoteSegmentStoreEnabled = (Boolean) settings.get(INDEX_REMOTE_STORE_ENABLED_SETTING);
+                    if (isRemoteSegmentStoreEnabled == null || isRemoteSegmentStoreEnabled == false) {
+                        throw new IllegalArgumentException(
+                            "Setting "
+                                + INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey()
+                                + " can only be set when "
+                                + INDEX_REMOTE_STORE_ENABLED_SETTING.getKey()
+                                + " is set to true"
+                        );
+                    }
+
+                    // Validate that the path prefix doesn't contain invalid characters for file paths
+                    if (value.contains("/") || value.contains("\\") || value.contains(":")) {
+                        throw new IllegalArgumentException(
+                            "Setting "
+                                + INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey()
+                                + " cannot contain path separators (/ or \\) or drive specifiers (:)"
+                        );
+                    }
+                }
+            }
+
+            @Override
+            public Iterator<Setting<?>> settings() {
+                final List<Setting<?>> settings = Collections.singletonList(INDEX_REMOTE_STORE_ENABLED_SETTING);
+                return settings.iterator();
+            }
+        },
+        Property.IndexScope,
+        Property.PrivateIndex,
+        Property.Dynamic
+    );
+
     private static void validateRemoteStoreSettingEnabled(final Map<Setting<?>, Object> settings, Setting<?> setting) {
         final Boolean isRemoteSegmentStoreEnabled = (Boolean) settings.get(INDEX_REMOTE_STORE_ENABLED_SETTING);
         if (isRemoteSegmentStoreEnabled == false) {

server/src/main/java/org/opensearch/index/IndexService.java

@@ -695,7 +695,8 @@ public synchronized IndexShard createShard(
                         RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(this.indexSettings.getNodeSettings()),
                         this.indexSettings.getUUID(),
                         shardId,
-                        this.indexSettings.getRemoteStorePathStrategy()
+                        this.indexSettings.getRemoteStorePathStrategy(),
+                        this.indexSettings.getRemoteStoreSegmentPathPrefix()
                     );
                 }
                 // When an instance of Store is created, a shardlock is created which is released on closing the instance of store.

server/src/main/java/org/opensearch/index/IndexSettings.java

@@ -809,6 +809,7 @@ public static IndexMergePolicy fromString(String text) {
     private volatile TimeValue remoteTranslogUploadBufferInterval;
     private volatile String remoteStoreTranslogRepository;
     private volatile String remoteStoreRepository;
+    private volatile String remoteStoreSegmentPathPrefix;
     private int remoteTranslogKeepExtraGen;
     private boolean autoForcemergeEnabled;
 
@@ -1025,6 +1026,9 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
         remoteTranslogUploadBufferInterval = INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(settings);
         remoteStoreRepository = settings.get(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY);
         this.remoteTranslogKeepExtraGen = INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING.get(settings);
+        String rawPrefix = IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(settings);
+        // Only set the prefix if it's explicitly set and not empty
+        this.remoteStoreSegmentPathPrefix = (rawPrefix != null && !rawPrefix.trim().isEmpty()) ? rawPrefix : null;
         this.searchThrottled = INDEX_SEARCH_THROTTLED.get(settings);
         this.shouldCleanupUnreferencedFiles = INDEX_UNREFERENCED_FILE_CLEANUP.get(settings);
         this.queryStringLenient = QUERY_STRING_LENIENT_SETTING.get(settings);
@@ -1426,6 +1430,13 @@ public String getRemoteStoreTranslogRepository() {
         return remoteStoreTranslogRepository;
     }
 
+    /**
+     * Returns the custom path prefix for remote store segments, if set.
+     */
+    public String getRemoteStoreSegmentPathPrefix() {
+        return remoteStoreSegmentPathPrefix;
+    }
+
     /**
      * Returns true if the index is writable warm index and has partial store locality.
      */

server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java

@@ -230,12 +230,14 @@ public static class ShardDataPathInput extends PathInput {
         private final String shardId;
         private final DataCategory dataCategory;
         private final DataType dataType;
+        private final String indexFixedPrefix;
 
         public ShardDataPathInput(Builder builder) {
             super(builder);
             this.shardId = Objects.requireNonNull(builder.shardId);
             this.dataCategory = Objects.requireNonNull(builder.dataCategory);
             this.dataType = Objects.requireNonNull(builder.dataType);
+            this.indexFixedPrefix = builder.indexFixedPrefix; // Can be null
             assert dataCategory.isSupportedDataType(dataType) : "category:"
                 + dataCategory
                 + " type:"
@@ -258,7 +260,12 @@ DataType dataType() {
 
         @Override
         BlobPath fixedSubPath() {
-            return super.fixedSubPath().add(shardId).add(dataCategory.getName()).add(dataType.getName());
+            BlobPath path = super.fixedSubPath().add(shardId);
+            // Only add index fixed prefix if it's explicitly set and not empty
+            if (indexFixedPrefix != null && !indexFixedPrefix.trim().isEmpty()) {
+                path = path.add(indexFixedPrefix);
+            }
+            return path.add(dataCategory.getName()).add(dataType.getName());
         }
 
         /**
@@ -279,6 +286,7 @@ public static class Builder extends PathInput.Builder<Builder> {
             private String shardId;
             private DataCategory dataCategory;
             private DataType dataType;
+            private String indexFixedPrefix;
 
             public Builder shardId(String shardId) {
                 this.shardId = shardId;
@@ -295,6 +303,11 @@ public Builder dataType(DataType dataType) {
                 return this;
             }
 
+            public Builder indexFixedPrefix(String indexFixedPrefix) {
+                this.indexFixedPrefix = indexFixedPrefix;
+                return this;
+            }
+
             @Override
             protected Builder self() {
                 return this;

server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java

@@ -65,6 +65,16 @@ public Directory newDirectory(IndexSettings indexSettings, ShardPath path) throw
 
     public Directory newDirectory(String repositoryName, String indexUUID, ShardId shardId, RemoteStorePathStrategy pathStrategy)
         throws IOException {
+        return newDirectory(repositoryName, indexUUID, shardId, pathStrategy, null);
+    }
+
+    public Directory newDirectory(
+        String repositoryName,
+        String indexUUID,
+        ShardId shardId,
+        RemoteStorePathStrategy pathStrategy,
+        String indexFixedPrefix
+    ) throws IOException {
         assert Objects.nonNull(pathStrategy);
         try (Repository repository = repositoriesService.get().repository(repositoryName)) {
 
@@ -81,6 +91,7 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s
                 .dataCategory(SEGMENTS)
                 .dataType(DATA)
                 .fixedPrefix(segmentsPathFixedPrefix)
+                .indexFixedPrefix(indexFixedPrefix)
                 .build();
             // Derive the path for data directory of SEGMENTS
             BlobPath dataPath = pathStrategy.generatePath(dataPathInput);
@@ -100,6 +111,7 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s
                 .dataCategory(SEGMENTS)
                 .dataType(METADATA)
                 .fixedPrefix(segmentsPathFixedPrefix)
+                .indexFixedPrefix(indexFixedPrefix)
                 .build();
             // Derive the path for metadata directory of SEGMENTS
             BlobPath mdPath = pathStrategy.generatePath(mdPathInput);
@@ -112,7 +124,8 @@ public Directory newDirectory(String repositoryName, String indexUUID, ShardId s
                 indexUUID,
                 shardIdStr,
                 pathStrategy,
-                segmentsPathFixedPrefix
+                segmentsPathFixedPrefix,
+                indexFixedPrefix
             );
 
             return new RemoteSegmentStoreDirectory(

server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java

@@ -44,7 +44,7 @@ public RemoteStoreLockManager newLockManager(
         String shardId,
         RemoteStorePathStrategy pathStrategy
     ) {
-        return newLockManager(repositoriesService.get(), repositoryName, indexUUID, shardId, pathStrategy, segmentsPathFixedPrefix);
+        return newLockManager(repositoriesService.get(), repositoryName, indexUUID, shardId, pathStrategy, segmentsPathFixedPrefix, null);
     }
 
     public static RemoteStoreMetadataLockManager newLockManager(
@@ -54,6 +54,18 @@ public static RemoteStoreMetadataLockManager newLockManager(
         String shardId,
         RemoteStorePathStrategy pathStrategy,
         String segmentsPathFixedPrefix
+    ) {
+        return newLockManager(repositoriesService, repositoryName, indexUUID, shardId, pathStrategy, segmentsPathFixedPrefix, null);
+    }
+
+    public static RemoteStoreMetadataLockManager newLockManager(
+        RepositoriesService repositoriesService,
+        String repositoryName,
+        String indexUUID,
+        String shardId,
+        RemoteStorePathStrategy pathStrategy,
+        String segmentsPathFixedPrefix,
+        String indexFixedPrefix
     ) {
         try (Repository repository = repositoriesService.repository(repositoryName)) {
             assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository";
@@ -66,6 +78,7 @@ public static RemoteStoreMetadataLockManager newLockManager(
                 .dataCategory(SEGMENTS)
                 .dataType(LOCK_FILES)
                 .fixedPrefix(segmentsPathFixedPrefix)
+                .indexFixedPrefix(indexFixedPrefix)
                 .build();
             BlobPath lockDirectoryPath = pathStrategy.generatePath(lockFilesPathInput);
             BlobContainer lockDirectoryBlobContainer = ((BlobStoreRepository) repository).blobStore().blobContainer(lockDirectoryPath);

server/src/test/java/org/opensearch/cluster/metadata/IndexMetadataTests.java

@@ -606,4 +606,84 @@ public void testIndicesMetadataDiffSystemFlagFlipped() {
         assertThat(indexMetadataAfterDiffApplied.getVersion(), equalTo(nextIndexMetadata.getVersion()));
     }
 
+    /**
+     * Test validation for remote store segment path prefix setting
+     */
+    public void testRemoteStoreSegmentPathPrefixValidation() {
+        // Test empty value (should be allowed)
+        final Settings emptySettings = Settings.builder()
+            .put(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.getKey(), true)
+            .put(IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey(), "")
+            .build();
+
+        IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(emptySettings);
+
+        final Settings whitespaceSettings = Settings.builder()
+            .put(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.getKey(), true)
+            .put(IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey(), "   ")
+            .build();
+
+        IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(whitespaceSettings);
+
+        final Settings validSettings = Settings.builder()
+            .put(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.getKey(), true)
+            .put(IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey(), "writer-node-1")
+            .build();
+
+        String value = IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(validSettings);
+        assertEquals("writer-node-1", value);
+
+        final Settings disabledSettings = Settings.builder()
+            .put(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.getKey(), false)
+            .put(IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey(), "writer-node-1")
+            .build();
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> {
+            IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(disabledSettings);
+        });
+        assertTrue(e.getMessage().contains("can only be set when"));
+
+        final Settings noRemoteStoreSettings = Settings.builder()
+            .put(IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey(), "writer-node-1")
+            .build();
+
+        e = expectThrows(
+            IllegalArgumentException.class,
+            () -> { IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(noRemoteStoreSettings); }
+        );
+        assertTrue(e.getMessage().contains("can only be set when"));
+
+        final Settings invalidPathSettings = Settings.builder()
+            .put(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.getKey(), true)
+            .put(IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey(), "writer/node")
+            .build();
+
+        e = expectThrows(
+            IllegalArgumentException.class,
+            () -> { IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(invalidPathSettings); }
+        );
+        assertTrue(e.getMessage().contains("cannot contain path separators"));
+
+        final Settings backslashSettings = Settings.builder()
+            .put(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.getKey(), true)
+            .put(IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey(), "writer\\node")
+            .build();
+
+        e = expectThrows(
+            IllegalArgumentException.class,
+            () -> { IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(backslashSettings); }
+        );
+        assertTrue(e.getMessage().contains("cannot contain path separators"));
+
+        final Settings colonSettings = Settings.builder()
+            .put(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.getKey(), true)
+            .put(IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.getKey(), "writer:node")
+            .build();
+
+        e = expectThrows(
+            IllegalArgumentException.class,
+            () -> { IndexMetadata.INDEX_REMOTE_STORE_SEGMENT_PATH_PREFIX.get(colonSettings); }
+        );
+        assertTrue(e.getMessage().contains("cannot contain path separators"));
+    }
 }