From 77d05a75548c05c62a770498a810b3c1b2acaa95 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 17 Sep 2025 15:02:59 +0200 Subject: [PATCH 1/4] Remove track_live_docs_in_memory_bytes feature flag --- .../elasticsearch/index/engine/Engine.java | 28 ++++++++----------- .../index/shard/ShardFieldStats.java | 1 - .../index/shard/IndexShardTests.java | 25 +++++++---------- 3 files changed, 21 insertions(+), 33 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index 8271a29dfd995..ed6a462365222 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -301,26 +301,20 @@ protected static ShardFieldStats shardFieldStats(List leaves, } else { usages = -1; } - boolean trackPostingsMemoryEnabled = isStateless; - boolean trackLiveDocsMemoryEnabled = ShardFieldStats.TRACK_LIVE_DOCS_IN_MEMORY_BYTES.isEnabled(); - if (trackLiveDocsMemoryEnabled || trackPostingsMemoryEnabled) { + if (isStateless) { SegmentReader segmentReader = Lucene.tryUnwrapSegmentReader(leaf.reader()); if (segmentReader != null) { - if (trackPostingsMemoryEnabled) { - String postingBytes = segmentReader.getSegmentInfo().info.getAttribute( - TrackingPostingsInMemoryBytesCodec.IN_MEMORY_POSTINGS_BYTES_KEY - ); - if (postingBytes != null) { - totalPostingBytes += Long.parseLong(postingBytes); - } + String postingBytes = segmentReader.getSegmentInfo().info.getAttribute( + TrackingPostingsInMemoryBytesCodec.IN_MEMORY_POSTINGS_BYTES_KEY + ); + if (postingBytes != null) { + totalPostingBytes += Long.parseLong(postingBytes); } - if (trackLiveDocsMemoryEnabled) { - var liveDocs = segmentReader.getLiveDocs(); - if (liveDocs != null) { - assert validateLiveDocsClass(liveDocs); - long liveDocsBytes = getLiveDocsBytes(liveDocs); - totalLiveDocsBytes += liveDocsBytes; - } + var liveDocs = segmentReader.getLiveDocs(); + if (liveDocs != null) { + assert validateLiveDocsClass(liveDocs); + long liveDocsBytes = getLiveDocsBytes(liveDocs); + totalLiveDocsBytes += liveDocsBytes; } } } diff --git a/server/src/main/java/org/elasticsearch/index/shard/ShardFieldStats.java b/server/src/main/java/org/elasticsearch/index/shard/ShardFieldStats.java index 17f9c7240f979..6d972f0ee7b08 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/ShardFieldStats.java +++ b/server/src/main/java/org/elasticsearch/index/shard/ShardFieldStats.java @@ -26,7 +26,6 @@ */ public record ShardFieldStats(int numSegments, int totalFields, long fieldUsages, long postingsInMemoryBytes, long liveDocsBytes) { - public static final FeatureFlag TRACK_LIVE_DOCS_IN_MEMORY_BYTES = new FeatureFlag("track_live_docs_in_memory_bytes"); public static final long FIXED_BITSET_BASE_RAM_BYTES_USED = RamUsageEstimator.shallowSizeOfInstance(FixedBitSet.class); } diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 9a89cf3387291..6f883ad88877c 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -1989,7 +1989,6 @@ public void testShardFieldStatsWithDeletes() throws IOException { IndexShard shard = newShard(true, settings); assertNull(shard.getShardFieldStats()); recoverShardFromStore(shard); - boolean liveDocsTrackingEnabled = ShardFieldStats.TRACK_LIVE_DOCS_IN_MEMORY_BYTES.isEnabled(); // index some documents int numDocs = 10; @@ -2015,12 +2014,10 @@ public void testShardFieldStatsWithDeletes() throws IOException { // More segments because delete operation is stored in the new segment for replication purposes. assertThat(stats.numSegments(), equalTo(2)); long expectedLiveDocsSize = 0; - if (liveDocsTrackingEnabled) { - // Delete op is stored in new segment, but marked as deleted. All segements have live docs: - expectedLiveDocsSize += new FixedBitSet(numDocs).ramBytesUsed(); - // Second segment the delete operation that is marked as deleted: - expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed(); - } + // Delete op is stored in new segment, but marked as deleted. All segements have live docs: + expectedLiveDocsSize += new FixedBitSet(numDocs).ramBytesUsed(); + // Second segment the delete operation that is marked as deleted: + expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed(); assertThat(stats.liveDocsBytes(), equalTo(expectedLiveDocsSize)); // delete another doc: @@ -2033,14 +2030,12 @@ public void testShardFieldStatsWithDeletes() throws IOException { // More segments because delete operation is stored in the new segment for replication purposes. assertThat(stats.numSegments(), equalTo(3)); expectedLiveDocsSize = 0; - if (liveDocsTrackingEnabled) { - // Delete op is stored in new segment, but marked as deleted. All segements have live docs: - // First segment with deletes - expectedLiveDocsSize += new FixedBitSet(numDocs).ramBytesUsed(); - // Second and third segments the delete operation that is marked as deleted: - expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed(); - expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed(); - } + // Delete op is stored in new segment, but marked as deleted. All segements have live docs: + // First segment with deletes + expectedLiveDocsSize += new FixedBitSet(numDocs).ramBytesUsed(); + // Second and third segments the delete operation that is marked as deleted: + expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed(); + expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed(); assertThat(stats.liveDocsBytes(), equalTo(expectedLiveDocsSize)); closeShards(shard); From 56ff93b0577f8ff3d5ef6ad2bd18546451495f0f Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 17 Sep 2025 13:20:35 +0000 Subject: [PATCH 2/4] [CI] Auto commit changes from spotless --- .../main/java/org/elasticsearch/index/shard/ShardFieldStats.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/shard/ShardFieldStats.java b/server/src/main/java/org/elasticsearch/index/shard/ShardFieldStats.java index 6d972f0ee7b08..b47fa3d6cdd3f 100644 --- a/server/src/main/java/org/elasticsearch/index/shard/ShardFieldStats.java +++ b/server/src/main/java/org/elasticsearch/index/shard/ShardFieldStats.java @@ -11,7 +11,6 @@ import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.RamUsageEstimator; -import org.elasticsearch.common.util.FeatureFlag; /** * A per shard stats including the number of segments and total fields across those segments. From 27ec57cd23b5efd121637ce59b8a6e64dd04a424 Mon Sep 17 00:00:00 2001 From: Martijn van Groningen Date: Wed, 17 Sep 2025 16:00:44 +0200 Subject: [PATCH 3/4] fix test by moving to its suite and overwrite node settings --- .../index/shard/IndexShardTests.java | 60 ------------- .../index/shard/LiveDocsEstimationTests.java | 90 +++++++++++++++++++ .../index/shard/IndexShardTestCase.java | 6 +- 3 files changed, 95 insertions(+), 61 deletions(-) create mode 100644 server/src/test/java/org/elasticsearch/index/shard/LiveDocsEstimationTests.java diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 6f883ad88877c..641c2fd9e216d 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -1981,66 +1981,6 @@ public void testShardFieldStats() throws IOException { closeShards(shard); } - public void testShardFieldStatsWithDeletes() throws IOException { - Settings settings = Settings.builder() - .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) - .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.MINUS_ONE) - .build(); - IndexShard shard = newShard(true, settings); - assertNull(shard.getShardFieldStats()); - recoverShardFromStore(shard); - - // index some documents - int numDocs = 10; - for (int i = 0; i < numDocs; i++) { - indexDoc(shard, "_doc", "first_" + i, """ - { - "f1": "foo", - "f2": "bar" - } - """); - } - shard.refresh("test"); - var stats = shard.getShardFieldStats(); - assertThat(stats.numSegments(), equalTo(1)); - assertThat(stats.liveDocsBytes(), equalTo(0L)); - - // delete a doc - deleteDoc(shard, "first_0"); - - // Refresh and fetch new stats: - shard.refresh("test"); - stats = shard.getShardFieldStats(); - // More segments because delete operation is stored in the new segment for replication purposes. - assertThat(stats.numSegments(), equalTo(2)); - long expectedLiveDocsSize = 0; - // Delete op is stored in new segment, but marked as deleted. All segements have live docs: - expectedLiveDocsSize += new FixedBitSet(numDocs).ramBytesUsed(); - // Second segment the delete operation that is marked as deleted: - expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed(); - assertThat(stats.liveDocsBytes(), equalTo(expectedLiveDocsSize)); - - // delete another doc: - deleteDoc(shard, "first_1"); - shard.getMinRetainedSeqNo(); - - // Refresh and fetch new stats: - shard.refresh("test"); - stats = shard.getShardFieldStats(); - // More segments because delete operation is stored in the new segment for replication purposes. - assertThat(stats.numSegments(), equalTo(3)); - expectedLiveDocsSize = 0; - // Delete op is stored in new segment, but marked as deleted. All segements have live docs: - // First segment with deletes - expectedLiveDocsSize += new FixedBitSet(numDocs).ramBytesUsed(); - // Second and third segments the delete operation that is marked as deleted: - expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed(); - expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed(); - assertThat(stats.liveDocsBytes(), equalTo(expectedLiveDocsSize)); - - closeShards(shard); - } - public void testIndexingOperationsListeners() throws IOException { IndexShard shard = newStartedShard(true); indexDoc(shard, "_doc", "0", "{\"foo\" : \"bar\"}"); diff --git a/server/src/test/java/org/elasticsearch/index/shard/LiveDocsEstimationTests.java b/server/src/test/java/org/elasticsearch/index/shard/LiveDocsEstimationTests.java new file mode 100644 index 0000000000000..e4374f816162a --- /dev/null +++ b/server/src/test/java/org/elasticsearch/index/shard/LiveDocsEstimationTests.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +package org.elasticsearch.index.shard; + +import org.apache.lucene.util.FixedBitSet; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.TimeValue; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.MergePolicyConfig; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class LiveDocsEstimationTests extends IndexShardTestCase { + + @Override + protected Settings nodeSettings() { + return Settings.builder().put(DiscoveryNode.STATELESS_ENABLED_SETTING_NAME, true).build(); + } + + public void testShardFieldStatsWithDeletes() throws IOException { + Settings settings = Settings.builder() + .put(MergePolicyConfig.INDEX_MERGE_ENABLED, false) + .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.MINUS_ONE) + .build(); + IndexShard shard = newShard(true, settings); + assertNull(shard.getShardFieldStats()); + recoverShardFromStore(shard); + + // index some documents + int numDocs = 10; + for (int i = 0; i < numDocs; i++) { + indexDoc(shard, "_doc", "first_" + i, """ + { + "f1": "foo", + "f2": "bar" + } + """); + } + shard.refresh("test"); + var stats = shard.getShardFieldStats(); + assertThat(stats.numSegments(), equalTo(1)); + assertThat(stats.liveDocsBytes(), equalTo(0L)); + + // delete a doc + deleteDoc(shard, "first_0"); + + // Refresh and fetch new stats: + shard.refresh("test"); + stats = shard.getShardFieldStats(); + // More segments because delete operation is stored in the new segment for replication purposes. + assertThat(stats.numSegments(), equalTo(2)); + long expectedLiveDocsSize = 0; + // Delete op is stored in new segment, but marked as deleted. All segements have live docs: + expectedLiveDocsSize += new FixedBitSet(numDocs).ramBytesUsed(); + // Second segment the delete operation that is marked as deleted: + expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed(); + assertThat(stats.liveDocsBytes(), equalTo(expectedLiveDocsSize)); + + // delete another doc: + deleteDoc(shard, "first_1"); + shard.getMinRetainedSeqNo(); + + // Refresh and fetch new stats: + shard.refresh("test"); + stats = shard.getShardFieldStats(); + // More segments because delete operation is stored in the new segment for replication purposes. + assertThat(stats.numSegments(), equalTo(3)); + expectedLiveDocsSize = 0; + // Delete op is stored in new segment, but marked as deleted. All segements have live docs: + // First segment with deletes + expectedLiveDocsSize += new FixedBitSet(numDocs).ramBytesUsed(); + // Second and third segments the delete operation that is marked as deleted: + expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed(); + expectedLiveDocsSize += new FixedBitSet(1).ramBytesUsed(); + assertThat(stats.liveDocsBytes(), equalTo(expectedLiveDocsSize)); + + closeShards(shard); + } + +} diff --git a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java index 667ee5cb68b59..394134c978c79 100644 --- a/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/index/shard/IndexShardTestCase.java @@ -616,7 +616,7 @@ protected IndexShard newShard( List soListener, IndexingOperationListener... listeners ) throws IOException { - final Settings nodeSettings = Settings.builder().put("node.name", routing.currentNodeId()).build(); + final Settings nodeSettings = Settings.builder().put(nodeSettings()).put("node.name", routing.currentNodeId()).build(); final IndexSettings indexSettings = new IndexSettings(indexMetadata, nodeSettings); final IndexShard indexShard; if (storeProvider == null) { @@ -1336,4 +1336,8 @@ public static Engine.Warmer createTestWarmer(IndexSettings indexSettings) { public static long recoverLocallyUpToGlobalCheckpoint(IndexShard indexShard) { return safeAwait(indexShard::recoverLocallyUpToGlobalCheckpoint); } + + protected Settings nodeSettings() { + return Settings.EMPTY; + } } From ed02c7f11d199ea4afb72d4bdf999c4435b3ec8a Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 18 Sep 2025 04:52:30 +0000 Subject: [PATCH 4/4] [CI] Auto commit changes from spotless --- .../java/org/elasticsearch/index/shard/IndexShardTests.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java index 641c2fd9e216d..6a41c39e0bb99 100644 --- a/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java +++ b/server/src/test/java/org/elasticsearch/index/shard/IndexShardTests.java @@ -27,7 +27,6 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Constants; -import org.apache.lucene.util.FixedBitSet; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; @@ -78,7 +77,6 @@ import org.elasticsearch.index.IndexModule; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.IndexVersion; -import org.elasticsearch.index.MergePolicyConfig; import org.elasticsearch.index.codec.CodecService; import org.elasticsearch.index.engine.CommitStats; import org.elasticsearch.index.engine.DocIdSeqNoAndSource;