Skip to content

Commit 3563648

Browse files
camilojdcamilojd
committed
Implementation of Segment disk stats aggregating sizes by index file extension.
Use 'includeSegmentFileSizes' as the flag name to report disk usage. Added test that verifies reported segment disk usage is growing accordingly after adding a document. Documentation: Reference the new parameter as part of indices stats.
1 parent ea2792e commit 3563648

File tree

13 files changed

+273
-37
lines changed

13 files changed

+273
-37
lines changed

‎core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStats.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -166,7 +166,7 @@ public CommonStats(IndicesQueryCache indicesQueryCache, IndexShard indexShard, C
166166
completion = indexShard.completionStats(flags.completionDataFields());
167167
break;
168168
case Segments:
169-
segments = indexShard.segmentStats();
169+
segments = indexShard.segmentStats(flags.includeSegmentFileSizes());
170170
break;
171171
case Percolate:
172172
percolate = indexShard.percolateStats();

‎core/src/main/java/org/elasticsearch/action/admin/indices/stats/CommonStatsFlags.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.action.admin.indices.stats;
2121

22+
import org.elasticsearch.Version;
2223
import org.elasticsearch.common.io.stream.StreamInput;
2324
import org.elasticsearch.common.io.stream.StreamOutput;
2425
import org.elasticsearch.common.io.stream.Streamable;
@@ -38,6 +39,7 @@ public class CommonStatsFlags implements Streamable, Cloneable {
3839
private String[] groups = null;
3940
private String[] fieldDataFields = null;
4041
private String[] completionDataFields = null;
42+
private boolean includeSegmentFileSizes = false;
4143

4244

4345
/**
@@ -62,6 +64,7 @@ public CommonStatsFlags all() {
6264
groups = null;
6365
fieldDataFields = null;
6466
completionDataFields = null;
67+
includeSegmentFileSizes = false;
6568
return this;
6669
}
6770

@@ -74,6 +77,7 @@ public CommonStatsFlags clear() {
7477
groups = null;
7578
fieldDataFields = null;
7679
completionDataFields = null;
80+
includeSegmentFileSizes = false;
7781
return this;
7882
}
7983

@@ -137,6 +141,15 @@ public String[] completionDataFields() {
137141
return this.completionDataFields;
138142
}
139143

144+
public CommonStatsFlags includeSegmentFileSizes(boolean includeSegmentFileSizes) {
145+
this.includeSegmentFileSizes = includeSegmentFileSizes;
146+
return this;
147+
}
148+
149+
public boolean includeSegmentFileSizes() {
150+
return this.includeSegmentFileSizes;
151+
}
152+
140153
public boolean isSet(Flag flag) {
141154
return flags.contains(flag);
142155
}
@@ -177,6 +190,9 @@ public void writeTo(StreamOutput out) throws IOException {
177190
out.writeStringArrayNullable(groups);
178191
out.writeStringArrayNullable(fieldDataFields);
179192
out.writeStringArrayNullable(completionDataFields);
193+
if (out.getVersion().onOrAfter(Version.V_5_0_0)) {
194+
out.writeBoolean(includeSegmentFileSizes);
195+
}
180196
}
181197

182198
@Override
@@ -192,6 +208,11 @@ public void readFrom(StreamInput in) throws IOException {
192208
groups = in.readStringArray();
193209
fieldDataFields = in.readStringArray();
194210
completionDataFields = in.readStringArray();
211+
if (in.getVersion().onOrAfter(Version.V_5_0_0)) {
212+
includeSegmentFileSizes = in.readBoolean();
213+
} else {
214+
includeSegmentFileSizes = false;
215+
}
195216
}
196217

197218
@Override

‎core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequest.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -265,6 +265,15 @@ public boolean recovery() {
265265
return flags.isSet(Flag.Recovery);
266266
}
267267

268+
public boolean includeSegmentFileSizes() {
269+
return flags.includeSegmentFileSizes();
270+
}
271+
272+
public IndicesStatsRequest includeSegmentFileSizes(boolean includeSegmentFileSizes) {
273+
flags.includeSegmentFileSizes(includeSegmentFileSizes);
274+
return this;
275+
}
276+
268277
@Override
269278
public void writeTo(StreamOutput out) throws IOException {
270279
super.writeTo(out);

‎core/src/main/java/org/elasticsearch/action/admin/indices/stats/IndicesStatsRequestBuilder.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,4 +166,9 @@ public IndicesStatsRequestBuilder setRecovery(boolean recovery) {
166166
request.recovery(recovery);
167167
return this;
168168
}
169+
170+
public IndicesStatsRequestBuilder setIncludeSegmentFileSizes(boolean includeSegmentFileSizes) {
171+
request.includeSegmentFileSizes(includeSegmentFileSizes);
172+
return this;
173+
}
169174
}

‎core/src/main/java/org/elasticsearch/action/admin/indices/stats/TransportIndicesStatsAction.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,7 @@ protected ShardStats shardOperation(IndicesStatsRequest request, ShardRouting sh
144144
}
145145
if (request.segments()) {
146146
flags.set(CommonStatsFlags.Flag.Segments);
147+
flags.includeSegmentFileSizes(request.includeSegmentFileSizes());
147148
}
148149
if (request.completion()) {
149150
flags.set(CommonStatsFlags.Flag.Completion);

‎core/src/main/java/org/elasticsearch/index/engine/Engine.java‎

Lines changed: 78 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,11 +22,13 @@
2222
import org.apache.lucene.index.DirectoryReader;
2323
import org.apache.lucene.index.FilterLeafReader;
2424
import org.apache.lucene.index.IndexCommit;
25+
import org.apache.lucene.index.IndexFileNames;
2526
import org.apache.lucene.index.IndexReader;
2627
import org.apache.lucene.index.IndexWriter;
2728
import org.apache.lucene.index.LeafReader;
2829
import org.apache.lucene.index.LeafReaderContext;
2930
import org.apache.lucene.index.SegmentCommitInfo;
31+
import org.apache.lucene.index.SegmentInfo;
3032
import org.apache.lucene.index.SegmentInfos;
3133
import org.apache.lucene.index.SegmentReader;
3234
import org.apache.lucene.index.SnapshotDeletionPolicy;
@@ -36,12 +38,15 @@
3638
import org.apache.lucene.search.SearcherManager;
3739
import org.apache.lucene.search.join.BitSetProducer;
3840
import org.apache.lucene.store.AlreadyClosedException;
41+
import org.apache.lucene.store.Directory;
42+
import org.apache.lucene.store.IOContext;
3943
import org.apache.lucene.util.Accountable;
4044
import org.apache.lucene.util.Accountables;
4145
import org.elasticsearch.ExceptionsHelper;
4246
import org.elasticsearch.common.Base64;
4347
import org.elasticsearch.common.Nullable;
4448
import org.elasticsearch.common.bytes.BytesReference;
49+
import org.elasticsearch.common.collect.ImmutableOpenMap;
4550
import org.elasticsearch.common.io.stream.StreamInput;
4651
import org.elasticsearch.common.io.stream.StreamOutput;
4752
import org.elasticsearch.common.io.stream.Writeable;
@@ -64,8 +69,11 @@
6469
import org.elasticsearch.index.translog.Translog;
6570

6671
import java.io.Closeable;
72+
import java.io.FileNotFoundException;
6773
import java.io.IOException;
74+
import java.nio.file.NoSuchFileException;
6875
import java.util.Arrays;
76+
import java.util.Collection;
6977
import java.util.Comparator;
7078
import java.util.HashMap;
7179
import java.util.List;
@@ -406,7 +414,7 @@ protected static SegmentInfos readLastCommittedSegmentInfos(final SearcherManage
406414
/**
407415
* Global stats on segments.
408416
*/
409-
public final SegmentsStats segmentsStats() {
417+
public final SegmentsStats segmentsStats(boolean includeSegmentFileSizes) {
410418
ensureOpen();
411419
try (final Searcher searcher = acquireSearcher("segments_stats")) {
412420
SegmentsStats stats = new SegmentsStats();
@@ -418,12 +426,81 @@ public final SegmentsStats segmentsStats() {
418426
stats.addTermVectorsMemoryInBytes(guardedRamBytesUsed(segmentReader.getTermVectorsReader()));
419427
stats.addNormsMemoryInBytes(guardedRamBytesUsed(segmentReader.getNormsReader()));
420428
stats.addDocValuesMemoryInBytes(guardedRamBytesUsed(segmentReader.getDocValuesReader()));
429+
430+
if (includeSegmentFileSizes) {
431+
// TODO: consider moving this to StoreStats
432+
stats.addFileSizes(getSegmentFileSizes(segmentReader));
433+
}
421434
}
422435
writerSegmentStats(stats);
423436
return stats;
424437
}
425438
}
426439

440+
private ImmutableOpenMap<String, Long> getSegmentFileSizes(SegmentReader segmentReader) {
441+
Directory directory = null;
442+
SegmentCommitInfo segmentCommitInfo = segmentReader.getSegmentInfo();
443+
boolean useCompoundFile = segmentCommitInfo.info.getUseCompoundFile();
444+
if (useCompoundFile) {
445+
try {
446+
directory = engineConfig.getCodec().compoundFormat().getCompoundReader(segmentReader.directory(), segmentCommitInfo.info, IOContext.READ);
447+
} catch (IOException e) {
448+
logger.warn("Error when opening compound reader for Directory [{}] and SegmentCommitInfo [{}]", e,
449+
segmentReader.directory(), segmentCommitInfo);
450+
451+
return ImmutableOpenMap.of();
452+
}
453+
} else {
454+
directory = segmentReader.directory();
455+
}
456+
457+
assert directory != null;
458+
459+
String[] files;
460+
if (useCompoundFile) {
461+
try {
462+
files = directory.listAll();
463+
} catch (IOException e) {
464+
logger.warn("Couldn't list Compound Reader Directory [{}]", e, directory);
465+
return ImmutableOpenMap.of();
466+
}
467+
} else {
468+
try {
469+
files = segmentReader.getSegmentInfo().files().toArray(new String[]{});
470+
} catch (IOException e) {
471+
logger.warn("Couldn't list Directory from SegmentReader [{}] and SegmentInfo [{}]", e, segmentReader, segmentReader.getSegmentInfo());
472+
return ImmutableOpenMap.of();
473+
}
474+
}
475+
476+
ImmutableOpenMap.Builder<String, Long> map = ImmutableOpenMap.builder();
477+
for (String file : files) {
478+
String extension = IndexFileNames.getExtension(file);
479+
long length = 0L;
480+
try {
481+
length = directory.fileLength(file);
482+
} catch (NoSuchFileException | FileNotFoundException e) {
483+
logger.warn("Tried to query fileLength but file is gone [{}] [{}]", e, directory, file);
484+
} catch (IOException e) {
485+
logger.warn("Error when trying to query fileLength [{}] [{}]", e, directory, file);
486+
}
487+
if (length == 0L) {
488+
continue;
489+
}
490+
map.put(extension, length);
491+
}
492+
493+
if (useCompoundFile && directory != null) {
494+
try {
495+
directory.close();
496+
} catch (IOException e) {
497+
logger.warn("Error when closing compound reader on Directory [{}]", e, directory);
498+
}
499+
}
500+
501+
return map.build();
502+
}
503+
427504
protected void writerSegmentStats(SegmentsStats stats) {
428505
// by default we don't have a writer here... subclasses can override this
429506
stats.addVersionMapMemoryInBytes(0);

0 commit comments

Comments
 (0)