From ef001886d55afd6c0c1d18b6e38e17ec81b3a5a3 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Tue, 25 Jul 2023 16:48:38 -0500 Subject: [PATCH 01/10] Expose segment file sizes --- .../elasticsearch/index/engine/Engine.java | 19 +++++++++++++++++++ .../index/engine/SegmentFileSize.java | 14 ++++++++++++++ 2 files changed, 33 insertions(+) create mode 100644 server/src/main/java/org/elasticsearch/index/engine/SegmentFileSize.java diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index e3f831f958e0e..1f0330ff5e1a7 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -25,6 +25,7 @@ import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.util.SetOnce; +import org.apache.lucene.util.StringHelper; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.index.IndexRequest; @@ -67,6 +68,7 @@ import java.io.Closeable; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; @@ -177,6 +179,23 @@ public DocsStats docStats() { } } + /** + * Returns the file sizes for the current commit + */ + public List getLastCommittedSegmentFileSizes() { + SegmentInfos segmentInfos = getLastCommittedSegmentInfos(); + List segmentFileSizes = new ArrayList<>(segmentInfos.size()); + for (final SegmentCommitInfo commitInfo : segmentInfos) { + String commitId = StringHelper.idToString(commitInfo.getId()); + try { + segmentFileSizes.add(new SegmentFileSize(commitId, commitInfo.sizeInBytes())); + } catch (IOException err) { + logger.warn("Failed to read file size for shard: [{}], id: [{}], err: [{}]", shardId, commitId, err); + } + } + return segmentFileSizes; + } + protected final DocsStats docsStats(IndexReader indexReader) { long numDocs = 0; long numDeletedDocs = 0; diff --git a/server/src/main/java/org/elasticsearch/index/engine/SegmentFileSize.java b/server/src/main/java/org/elasticsearch/index/engine/SegmentFileSize.java new file mode 100644 index 0000000000000..cc6ad09642008 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/engine/SegmentFileSize.java @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.engine; + +/** + * Total File size and commit id for a segment + */ +public record SegmentFileSize(String commitId, long size) {} From 2766e6bd18acd650f250e8cd840ef6e9da505b10 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 26 Jul 2023 13:08:34 -0500 Subject: [PATCH 02/10] getLastCommittedSegmentFileSizes(): List -> Map --- .../org/elasticsearch/index/engine/Engine.java | 7 +++---- .../index/engine/SegmentFileSize.java | 14 -------------- 2 files changed, 3 insertions(+), 18 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/index/engine/SegmentFileSize.java diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index 1f0330ff5e1a7..10e1dcb29369f 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -68,7 +68,6 @@ import java.io.Closeable; import java.io.IOException; import java.io.UncheckedIOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; @@ -182,13 +181,13 @@ public DocsStats docStats() { /** * Returns the file sizes for the current commit */ - public List getLastCommittedSegmentFileSizes() { + public Map getLastCommittedSegmentFileSizes() { SegmentInfos segmentInfos = getLastCommittedSegmentInfos(); - List segmentFileSizes = new ArrayList<>(segmentInfos.size()); + Map segmentFileSizes = new HashMap<>(); for (final SegmentCommitInfo commitInfo : segmentInfos) { String commitId = StringHelper.idToString(commitInfo.getId()); try { - segmentFileSizes.add(new SegmentFileSize(commitId, commitInfo.sizeInBytes())); + segmentFileSizes.put(commitId, commitInfo.sizeInBytes()); } catch (IOException err) { logger.warn("Failed to read file size for shard: [{}], id: [{}], err: [{}]", shardId, commitId, err); } diff --git a/server/src/main/java/org/elasticsearch/index/engine/SegmentFileSize.java b/server/src/main/java/org/elasticsearch/index/engine/SegmentFileSize.java deleted file mode 100644 index cc6ad09642008..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/engine/SegmentFileSize.java +++ /dev/null @@ -1,14 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.index.engine; - -/** - * Total File size and commit id for a segment - */ -public record SegmentFileSize(String commitId, long size) {} From a3a28ad0e7e44825615ceb1b7a4021084699373a Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Wed, 26 Jul 2023 16:42:49 -0500 Subject: [PATCH 03/10] Add test --- .../index/engine/InternalEngineTests.java | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index f3103fa5666f6..241932f302a36 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -188,6 +188,8 @@ import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.sameInstance; +import static org.hamcrest.Matchers.aMapWithSize; +import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsInRelativeOrder; @@ -511,6 +513,63 @@ public void testSegments() throws Exception { } } + public void testSegmentFileSizes() throws Exception { + final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); + try ( + Store store = createStore(); + InternalEngine engine = createEngine( + config(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null, null, globalCheckpoint::get) + ) + ) { + assertThat(engine.segments(), empty()); + int numDocsSeg1 = 100; + for (int i = 0; i < numDocsSeg1; i++) { + String id = Integer.toString(i); + ParsedDocument doc = testParsedDocument(id, null, testDocument(), B_1, null); + engine.index(indexForDoc(doc)); + } + Map fileSizes = engine.getLastCommittedSegmentFileSizes(); + assertThat(fileSizes, anEmptyMap()); + List segments = engine.segments(); + assertThat(segments, empty()); + + engine.flush(); + segments = engine.segments(); + fileSizes = engine.getLastCommittedSegmentFileSizes(); + assertThat(fileSizes, aMapWithSize(1)); + Map.Entry seg1 = fileSizes.entrySet().iterator().next(); + assertThat(seg1.getValue(), greaterThan(0L)); + assertThat(segments, hasSize(1)); + assertThat(segments.get(0).getNumDocs(), equalTo(numDocsSeg1)); + assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); + assertTrue(segments.get(0).committed); + int numDocsSeg2 = 20; + for (int i = 0; i < numDocsSeg2; i++) { + engine.index(indexForDoc(testParsedDocument(UUIDs.randomBase64UUID(), null, testDocument(), B_1, null))); + } + + engine.flush(); + segments = engine.segments(); + fileSizes = engine.getLastCommittedSegmentFileSizes(); + assertThat(fileSizes, aMapWithSize(2)); + assertThat(segments, hasSize(2)); + assertThat(segments.get(0).getNumDocs(), equalTo(numDocsSeg1)); + assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); + assertTrue(segments.get(0).committed); + + Map.Entry seg2 = null; + for (Map.Entry seg : fileSizes.entrySet()) { + if (seg1.getKey().equals(seg.getKey()) == false) { + seg2 = seg; + } + } + assertNotNull(seg2); + assertThat(segments.get(1).getNumDocs(), equalTo(numDocsSeg2)); + assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); + assertTrue(segments.get(1).committed); + } + } + public void testCommitStats() throws IOException { final AtomicLong maxSeqNo = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); final AtomicLong localCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); From 507732b79e7dae7ce30a47baadfa6dca8eaeba1a Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 27 Jul 2023 10:42:32 -0500 Subject: [PATCH 04/10] Map getLastCommittedSegmentFileSizes -> long getLastCommitSizeInBytes --- .../elasticsearch/index/engine/Engine.java | 16 ++++++----- .../index/engine/InternalEngineTests.java | 27 +++++-------------- 2 files changed, 16 insertions(+), 27 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index 10e1dcb29369f..b82eb17571d9c 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -181,18 +181,22 @@ public DocsStats docStats() { /** * Returns the file sizes for the current commit */ - public Map getLastCommittedSegmentFileSizes() { + public long getLastCommitSizeInBytes() { SegmentInfos segmentInfos = getLastCommittedSegmentInfos(); - Map segmentFileSizes = new HashMap<>(); + long total = 0; for (final SegmentCommitInfo commitInfo : segmentInfos) { - String commitId = StringHelper.idToString(commitInfo.getId()); try { - segmentFileSizes.put(commitId, commitInfo.sizeInBytes()); + total += commitInfo.sizeInBytes(); } catch (IOException err) { - logger.warn("Failed to read file size for shard: [{}], id: [{}], err: [{}]", shardId, commitId, err); + logger.warn( + "Failed to read file size for shard: [{}], id: [{}], err: [{}]", + shardId, + StringHelper.idToString(commitInfo.getId()), + err + ); } } - return segmentFileSizes; + return total; } protected final DocsStats docsStats(IndexReader indexReader) { diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 241932f302a36..d0d0eb03031c2 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -188,8 +188,6 @@ import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_SEQ_NO; import static org.hamcrest.CoreMatchers.instanceOf; import static org.hamcrest.CoreMatchers.sameInstance; -import static org.hamcrest.Matchers.aMapWithSize; -import static org.hamcrest.Matchers.anEmptyMap; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsInRelativeOrder; @@ -528,17 +526,15 @@ public void testSegmentFileSizes() throws Exception { ParsedDocument doc = testParsedDocument(id, null, testDocument(), B_1, null); engine.index(indexForDoc(doc)); } - Map fileSizes = engine.getLastCommittedSegmentFileSizes(); - assertThat(fileSizes, anEmptyMap()); + long sizes = engine.getLastCommitSizeInBytes(); + assertThat(sizes, equalTo(0L)); List segments = engine.segments(); assertThat(segments, empty()); engine.flush(); segments = engine.segments(); - fileSizes = engine.getLastCommittedSegmentFileSizes(); - assertThat(fileSizes, aMapWithSize(1)); - Map.Entry seg1 = fileSizes.entrySet().iterator().next(); - assertThat(seg1.getValue(), greaterThan(0L)); + sizes = engine.getLastCommitSizeInBytes(); + assertThat(sizes, greaterThan(0L)); assertThat(segments, hasSize(1)); assertThat(segments.get(0).getNumDocs(), equalTo(numDocsSeg1)); assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); @@ -550,23 +546,12 @@ public void testSegmentFileSizes() throws Exception { engine.flush(); segments = engine.segments(); - fileSizes = engine.getLastCommittedSegmentFileSizes(); - assertThat(fileSizes, aMapWithSize(2)); + sizes = engine.getLastCommitSizeInBytes(); + assertThat(sizes, greaterThan(0L)); assertThat(segments, hasSize(2)); assertThat(segments.get(0).getNumDocs(), equalTo(numDocsSeg1)); assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); assertTrue(segments.get(0).committed); - - Map.Entry seg2 = null; - for (Map.Entry seg : fileSizes.entrySet()) { - if (seg1.getKey().equals(seg.getKey()) == false) { - seg2 = seg; - } - } - assertNotNull(seg2); - assertThat(segments.get(1).getNumDocs(), equalTo(numDocsSeg2)); - assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); - assertTrue(segments.get(1).committed); } } From 08693974b2656cfc8aa610bfc200a50e420385cb Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 27 Jul 2023 12:30:09 -0500 Subject: [PATCH 05/10] Return commitinfo list --- .../elasticsearch/index/engine/Engine.java | 24 ++++--------------- .../index/engine/InternalEngineTests.java | 22 +++++++++++------ 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index b82eb17571d9c..fbfc6cfae815f 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -25,7 +25,6 @@ import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.util.SetOnce; -import org.apache.lucene.util.StringHelper; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.index.IndexRequest; @@ -68,6 +67,7 @@ import java.io.Closeable; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; @@ -178,25 +178,9 @@ public DocsStats docStats() { } } - /** - * Returns the file sizes for the current commit - */ - public long getLastCommitSizeInBytes() { - SegmentInfos segmentInfos = getLastCommittedSegmentInfos(); - long total = 0; - for (final SegmentCommitInfo commitInfo : segmentInfos) { - try { - total += commitInfo.sizeInBytes(); - } catch (IOException err) { - logger.warn( - "Failed to read file size for shard: [{}], id: [{}], err: [{}]", - shardId, - StringHelper.idToString(commitInfo.getId()), - err - ); - } - } - return total; + /** Returns the SegmentCommitInfo list for the current commit */ + public List getLastCommittedSegmentInfoList() { + return new ArrayList<>(getLastCommittedSegmentInfos().asList()); } protected final DocsStats docsStats(IndexReader indexReader) { diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index d0d0eb03031c2..9b0f8494bc0d4 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -40,6 +40,7 @@ import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.SegmentCommitInfo; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.index.SoftDeletesRetentionMergePolicy; import org.apache.lucene.index.Term; @@ -526,15 +527,17 @@ public void testSegmentFileSizes() throws Exception { ParsedDocument doc = testParsedDocument(id, null, testDocument(), B_1, null); engine.index(indexForDoc(doc)); } - long sizes = engine.getLastCommitSizeInBytes(); - assertThat(sizes, equalTo(0L)); + List commits = engine.getLastCommittedSegmentInfoList(); + assertThat(commits, empty()); List segments = engine.segments(); assertThat(segments, empty()); engine.flush(); segments = engine.segments(); - sizes = engine.getLastCommitSizeInBytes(); - assertThat(sizes, greaterThan(0L)); + commits = engine.getLastCommittedSegmentInfoList(); + assertThat(commits, hasSize(1)); + SegmentCommitInfo seg1 = commits.get(0); + assertThat(seg1.sizeInBytes(), greaterThan(0L)); assertThat(segments, hasSize(1)); assertThat(segments.get(0).getNumDocs(), equalTo(numDocsSeg1)); assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); @@ -546,15 +549,20 @@ public void testSegmentFileSizes() throws Exception { engine.flush(); segments = engine.segments(); - sizes = engine.getLastCommitSizeInBytes(); - assertThat(sizes, greaterThan(0L)); + commits = engine.getLastCommittedSegmentInfoList(); + assertThat(commits, hasSize(2)); assertThat(segments, hasSize(2)); assertThat(segments.get(0).getNumDocs(), equalTo(numDocsSeg1)); assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); assertTrue(segments.get(0).committed); + + SegmentCommitInfo seg2 = commits.get(0).getId().equals(seg1.getId()) ? commits.get(1) : commits.get(0); + assertNotNull(seg2); + assertThat(segments.get(1).getNumDocs(), equalTo(numDocsSeg2)); + assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); + assertTrue(segments.get(1).committed); } } - public void testCommitStats() throws IOException { final AtomicLong maxSeqNo = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); final AtomicLong localCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); From e9fbcb43b0dd23ef47e156b9646cf4aeb6da90ec Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 27 Jul 2023 13:03:48 -0500 Subject: [PATCH 06/10] Lazily load file size --- .../elasticsearch/index/engine/Engine.java | 8 +++++ .../index/engine/SegmentFileSize.java | 35 +++++++++++++++++++ .../index/engine/InternalEngineTests.java | 21 ++++++----- 3 files changed, 53 insertions(+), 11 deletions(-) create mode 100644 server/src/main/java/org/elasticsearch/index/engine/SegmentFileSize.java diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index fbfc6cfae815f..c4d915ab75f4e 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -88,6 +88,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Consumer; import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; @@ -149,6 +151,12 @@ public final EngineConfig config() { protected abstract SegmentInfos getLastCommittedSegmentInfos(); + public List getLastCommittedSegmentFileSizes() { + return StreamSupport.stream(getLastCommittedSegmentInfos().spliterator(), false) + .map(SegmentFileSize::new) + .collect(Collectors.toList()); + } + public MergeStats getMergeStats() { return new MergeStats(); } diff --git a/server/src/main/java/org/elasticsearch/index/engine/SegmentFileSize.java b/server/src/main/java/org/elasticsearch/index/engine/SegmentFileSize.java new file mode 100644 index 0000000000000..7b58a276c333f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/index/engine/SegmentFileSize.java @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +package org.elasticsearch.index.engine; + +import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.util.StringHelper; + +import java.io.IOException; + +/** + * Lazily get the file size for a segment. + */ +public class SegmentFileSize { + public final String id; + private final SegmentCommitInfo commit; + + public SegmentFileSize(SegmentCommitInfo commit) { + this.id = StringHelper.idToString(commit.getId()); + this.commit = commit; + } + + public String id() { + return id; + } + + public long sizeInBytes() throws IOException { + return commit.sizeInBytes(); + } +} diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 9b0f8494bc0d4..af0cb71659067 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -40,7 +40,6 @@ import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.PointValues; -import org.apache.lucene.index.SegmentCommitInfo; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.index.SoftDeletesRetentionMergePolicy; import org.apache.lucene.index.Term; @@ -527,17 +526,16 @@ public void testSegmentFileSizes() throws Exception { ParsedDocument doc = testParsedDocument(id, null, testDocument(), B_1, null); engine.index(indexForDoc(doc)); } - List commits = engine.getLastCommittedSegmentInfoList(); - assertThat(commits, empty()); + List fileSizes = engine.getLastCommittedSegmentFileSizes(); + assertThat(fileSizes, empty()); List segments = engine.segments(); assertThat(segments, empty()); engine.flush(); segments = engine.segments(); - commits = engine.getLastCommittedSegmentInfoList(); - assertThat(commits, hasSize(1)); - SegmentCommitInfo seg1 = commits.get(0); - assertThat(seg1.sizeInBytes(), greaterThan(0L)); + fileSizes = engine.getLastCommittedSegmentFileSizes(); + assertThat(fileSizes, hasSize(1)); + assertThat(fileSizes.get(0).sizeInBytes(), greaterThan(0L)); assertThat(segments, hasSize(1)); assertThat(segments.get(0).getNumDocs(), equalTo(numDocsSeg1)); assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); @@ -549,20 +547,21 @@ public void testSegmentFileSizes() throws Exception { engine.flush(); segments = engine.segments(); - commits = engine.getLastCommittedSegmentInfoList(); - assertThat(commits, hasSize(2)); + fileSizes = engine.getLastCommittedSegmentFileSizes(); + assertThat(fileSizes, hasSize(2)); assertThat(segments, hasSize(2)); assertThat(segments.get(0).getNumDocs(), equalTo(numDocsSeg1)); assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); assertTrue(segments.get(0).committed); + assertThat(fileSizes.get(0).sizeInBytes(), greaterThan(0L)); - SegmentCommitInfo seg2 = commits.get(0).getId().equals(seg1.getId()) ? commits.get(1) : commits.get(0); - assertNotNull(seg2); assertThat(segments.get(1).getNumDocs(), equalTo(numDocsSeg2)); assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); assertTrue(segments.get(1).committed); + assertThat(fileSizes.get(1).sizeInBytes(), greaterThan(0L)); } } + public void testCommitStats() throws IOException { final AtomicLong maxSeqNo = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); final AtomicLong localCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); From 934065838831739fa56f09ab75166ff25987cba9 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 27 Jul 2023 13:07:03 -0500 Subject: [PATCH 07/10] remove getLastCommittedSegmentInfoList --- .../main/java/org/elasticsearch/index/engine/Engine.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index c4d915ab75f4e..c922b4621fa6d 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -151,6 +151,9 @@ public final EngineConfig config() { protected abstract SegmentInfos getLastCommittedSegmentInfos(); + /** + * Get the a lazily loaded file size for all segments in the last commit + */ public List getLastCommittedSegmentFileSizes() { return StreamSupport.stream(getLastCommittedSegmentInfos().spliterator(), false) .map(SegmentFileSize::new) @@ -186,11 +189,6 @@ public DocsStats docStats() { } } - /** Returns the SegmentCommitInfo list for the current commit */ - public List getLastCommittedSegmentInfoList() { - return new ArrayList<>(getLastCommittedSegmentInfos().asList()); - } - protected final DocsStats docsStats(IndexReader indexReader) { long numDocs = 0; long numDeletedDocs = 0; From 710fa2f879421e59a5324c8814aa4d246213d25a Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 27 Jul 2023 15:04:27 -0500 Subject: [PATCH 08/10] spotless --- server/src/main/java/org/elasticsearch/index/engine/Engine.java | 1 - 1 file changed, 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index c922b4621fa6d..a83843ef5b103 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -67,7 +67,6 @@ import java.io.Closeable; import java.io.IOException; import java.io.UncheckedIOException; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Comparator; From d5074c61b6814548c8a758592e642fe7419ceb8a Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 27 Jul 2023 17:22:06 -0500 Subject: [PATCH 09/10] Change visibility of Engine.getLastCommittedSegmentInfos() to public --- .../elasticsearch/index/engine/Engine.java | 13 +---- .../index/engine/InternalEngine.java | 2 +- .../index/engine/ReadOnlyEngine.java | 2 +- .../index/engine/SegmentFileSize.java | 35 ------------- .../index/engine/InternalEngineTests.java | 52 ------------------- 5 files changed, 3 insertions(+), 101 deletions(-) delete mode 100644 server/src/main/java/org/elasticsearch/index/engine/SegmentFileSize.java diff --git a/server/src/main/java/org/elasticsearch/index/engine/Engine.java b/server/src/main/java/org/elasticsearch/index/engine/Engine.java index a83843ef5b103..953c77e2826f4 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/Engine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/Engine.java @@ -87,8 +87,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Consumer; import java.util.function.Function; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; import static org.elasticsearch.core.Strings.format; import static org.elasticsearch.index.seqno.SequenceNumbers.UNASSIGNED_PRIMARY_TERM; @@ -148,16 +146,7 @@ public final EngineConfig config() { return engineConfig; } - protected abstract SegmentInfos getLastCommittedSegmentInfos(); - - /** - * Get the a lazily loaded file size for all segments in the last commit - */ - public List getLastCommittedSegmentFileSizes() { - return StreamSupport.stream(getLastCommittedSegmentInfos().spliterator(), false) - .map(SegmentFileSize::new) - .collect(Collectors.toList()); - } + public abstract SegmentInfos getLastCommittedSegmentInfos(); public MergeStats getMergeStats() { return new MergeStats(); diff --git a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java index a17441b1f37b2..684214b2692b5 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java @@ -2478,7 +2478,7 @@ protected boolean maybeFailEngine(String source, Exception e) { } @Override - protected SegmentInfos getLastCommittedSegmentInfos() { + public SegmentInfos getLastCommittedSegmentInfos() { return lastCommittedSegmentInfos; } diff --git a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java index d428bdbfb13db..b38cc24e107ea 100644 --- a/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java +++ b/server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java @@ -287,7 +287,7 @@ protected ReferenceManager getReferenceManager(Sea } @Override - protected SegmentInfos getLastCommittedSegmentInfos() { + public SegmentInfos getLastCommittedSegmentInfos() { return lastCommittedSegmentInfos; } diff --git a/server/src/main/java/org/elasticsearch/index/engine/SegmentFileSize.java b/server/src/main/java/org/elasticsearch/index/engine/SegmentFileSize.java deleted file mode 100644 index 7b58a276c333f..0000000000000 --- a/server/src/main/java/org/elasticsearch/index/engine/SegmentFileSize.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.index.engine; - -import org.apache.lucene.index.SegmentCommitInfo; -import org.apache.lucene.util.StringHelper; - -import java.io.IOException; - -/** - * Lazily get the file size for a segment. - */ -public class SegmentFileSize { - public final String id; - private final SegmentCommitInfo commit; - - public SegmentFileSize(SegmentCommitInfo commit) { - this.id = StringHelper.idToString(commit.getId()); - this.commit = commit; - } - - public String id() { - return id; - } - - public long sizeInBytes() throws IOException { - return commit.sizeInBytes(); - } -} diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index af0cb71659067..0a6105b314dd6 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -510,58 +510,6 @@ public void testSegments() throws Exception { assertThat(segments.get(1).committed, equalTo(committed)); } } - - public void testSegmentFileSizes() throws Exception { - final AtomicLong globalCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); - try ( - Store store = createStore(); - InternalEngine engine = createEngine( - config(defaultSettings, store, createTempDir(), NoMergePolicy.INSTANCE, null, null, globalCheckpoint::get) - ) - ) { - assertThat(engine.segments(), empty()); - int numDocsSeg1 = 100; - for (int i = 0; i < numDocsSeg1; i++) { - String id = Integer.toString(i); - ParsedDocument doc = testParsedDocument(id, null, testDocument(), B_1, null); - engine.index(indexForDoc(doc)); - } - List fileSizes = engine.getLastCommittedSegmentFileSizes(); - assertThat(fileSizes, empty()); - List segments = engine.segments(); - assertThat(segments, empty()); - - engine.flush(); - segments = engine.segments(); - fileSizes = engine.getLastCommittedSegmentFileSizes(); - assertThat(fileSizes, hasSize(1)); - assertThat(fileSizes.get(0).sizeInBytes(), greaterThan(0L)); - assertThat(segments, hasSize(1)); - assertThat(segments.get(0).getNumDocs(), equalTo(numDocsSeg1)); - assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); - assertTrue(segments.get(0).committed); - int numDocsSeg2 = 20; - for (int i = 0; i < numDocsSeg2; i++) { - engine.index(indexForDoc(testParsedDocument(UUIDs.randomBase64UUID(), null, testDocument(), B_1, null))); - } - - engine.flush(); - segments = engine.segments(); - fileSizes = engine.getLastCommittedSegmentFileSizes(); - assertThat(fileSizes, hasSize(2)); - assertThat(segments, hasSize(2)); - assertThat(segments.get(0).getNumDocs(), equalTo(numDocsSeg1)); - assertThat(segments.get(0).getDeletedDocs(), equalTo(0)); - assertTrue(segments.get(0).committed); - assertThat(fileSizes.get(0).sizeInBytes(), greaterThan(0L)); - - assertThat(segments.get(1).getNumDocs(), equalTo(numDocsSeg2)); - assertThat(segments.get(1).getDeletedDocs(), equalTo(0)); - assertTrue(segments.get(1).committed); - assertThat(fileSizes.get(1).sizeInBytes(), greaterThan(0L)); - } - } - public void testCommitStats() throws IOException { final AtomicLong maxSeqNo = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); final AtomicLong localCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); From 37c3042fabe3c76f16b47599be8a8f3cfce30a53 Mon Sep 17 00:00:00 2001 From: Stuart Tettemer Date: Thu, 27 Jul 2023 17:26:54 -0500 Subject: [PATCH 10/10] add line back to InternalEngineTests --- .../java/org/elasticsearch/index/engine/InternalEngineTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java index 0a6105b314dd6..f3103fa5666f6 100644 --- a/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java +++ b/server/src/test/java/org/elasticsearch/index/engine/InternalEngineTests.java @@ -510,6 +510,7 @@ public void testSegments() throws Exception { assertThat(segments.get(1).committed, equalTo(committed)); } } + public void testCommitStats() throws IOException { final AtomicLong maxSeqNo = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED); final AtomicLong localCheckpoint = new AtomicLong(SequenceNumbers.NO_OPS_PERFORMED);