Skip to content

Commit df0370f

Browse files
camilojdcamilojd
committed
Implementation of store stats with aggregated size information per index file extensions. Only grabs data from committed segments though
1 parent b59d361 commit df0370f

File tree

3 files changed

+165
-14
lines changed

3 files changed

+165
-14
lines changed

‎core/src/main/java/org/elasticsearch/index/store/Store.java‎

Lines changed: 92 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.elasticsearch.index.store;
2121

22+
import org.apache.lucene.codecs.Codec;
2223
import org.apache.lucene.codecs.CodecUtil;
2324
import org.apache.lucene.index.CorruptIndexException;
2425
import org.apache.lucene.index.IndexCommit;
@@ -28,6 +29,7 @@
2829
import org.apache.lucene.index.IndexNotFoundException;
2930
import org.apache.lucene.index.IndexWriter;
3031
import org.apache.lucene.index.SegmentCommitInfo;
32+
import org.apache.lucene.index.SegmentInfo;
3133
import org.apache.lucene.index.SegmentInfos;
3234
import org.apache.lucene.store.AlreadyClosedException;
3335
import org.apache.lucene.store.BufferedChecksum;
@@ -49,6 +51,7 @@
4951
import org.elasticsearch.ExceptionsHelper;
5052
import org.elasticsearch.common.Strings;
5153
import org.elasticsearch.common.bytes.BytesReference;
54+
import org.elasticsearch.common.collect.ImmutableOpenMap;
5255
import org.elasticsearch.common.collect.Tuple;
5356
import org.elasticsearch.common.inject.Inject;
5457
import org.elasticsearch.common.io.Streams;
@@ -85,6 +88,7 @@
8588
import java.nio.file.Path;
8689
import java.sql.Time;
8790
import java.util.ArrayList;
91+
import java.util.Arrays;
8892
import java.util.Collections;
8993
import java.util.Comparator;
9094
import java.util.HashMap;
@@ -158,7 +162,7 @@ public Store(ShardId shardId, IndexSettings indexSettings, DirectoryService dire
158162
this.shardLock = shardLock;
159163
this.onClose = onClose;
160164
final TimeValue refreshInterval = indexSettings.getValue(INDEX_STORE_STATS_REFRESH_INTERVAL_SETTING);
161-
this.statsCache = new StoreStatsCache(refreshInterval, directory, directoryService);
165+
this.statsCache = new StoreStatsCache(refreshInterval, directoryService, this);
162166
logger.debug("store stats are refreshed with refresh_interval [{}]", refreshInterval);
163167

164168
assert onClose != null;
@@ -1546,35 +1550,113 @@ public void handle(ShardLock Lock) {
15461550
}
15471551

15481552
private static class StoreStatsCache extends SingleObjectCache<StoreStats> {
1549-
private final Directory directory;
1553+
private final Store store;
15501554
private final DirectoryService directoryService;
15511555

1552-
public StoreStatsCache(TimeValue refreshInterval, Directory directory, DirectoryService directoryService) throws IOException {
1553-
super(refreshInterval, new StoreStats(estimateSize(directory), directoryService.throttleTimeInNanos()));
1554-
this.directory = directory;
1556+
public StoreStatsCache(TimeValue refreshInterval, DirectoryService directoryService, Store store) throws IOException {
1557+
super(refreshInterval, estimateSize(store, directoryService.throttleTimeInNanos()));
1558+
this.store = store;
15551559
this.directoryService = directoryService;
15561560
}
15571561

15581562
@Override
15591563
protected StoreStats refresh() {
15601564
try {
1561-
return new StoreStats(estimateSize(directory), directoryService.throttleTimeInNanos());
1565+
return estimateSize(store, directoryService.throttleTimeInNanos());
15621566
} catch (IOException ex) {
15631567
throw new ElasticsearchException("failed to refresh store stats", ex);
15641568
}
15651569
}
15661570

1567-
private static long estimateSize(Directory directory) throws IOException {
1571+
private static StoreStats estimateSize(Store store, final long throttleTimeInNanos) throws IOException {
15681572
long estimatedSize = 0;
1569-
String[] files = directory.listAll();
1573+
Directory storeDirectory = store.directory();
1574+
String[] files = new String[0];
1575+
try {
1576+
files = storeDirectory.listAll();
1577+
} catch (IOException e) {
1578+
store.logger.warn("Couldn't list Store Directory [{}]", e, storeDirectory);
1579+
}
1580+
1581+
boolean segmentsFileExists = false;
15701582
for (String file : files) {
15711583
try {
1572-
estimatedSize += directory.fileLength(file);
1584+
estimatedSize += storeDirectory.fileLength(file);
1585+
if (file.startsWith(IndexFileNames.SEGMENTS)) {
1586+
segmentsFileExists = true;
1587+
}
15731588
} catch (NoSuchFileException | FileNotFoundException e) {
15741589
// ignore, the file is not there no more
15751590
}
15761591
}
1577-
return estimatedSize;
1592+
1593+
StoreStats basicStoreStats = new StoreStats(estimatedSize, ImmutableOpenMap.of(), throttleTimeInNanos);
1594+
if (!segmentsFileExists) {
1595+
return basicStoreStats;
1596+
}
1597+
1598+
ImmutableOpenMap.Builder<String, Long> map = ImmutableOpenMap.builder();
1599+
SegmentInfos segmentInfos = null;
1600+
try {
1601+
segmentInfos = store.readLastCommittedSegmentsInfo();
1602+
} catch (IOException e) {
1603+
store.logger.warn("Unable to retrieve segment infos", e);
1604+
return basicStoreStats;
1605+
}
1606+
1607+
for (SegmentCommitInfo segmentCommitInfo : segmentInfos) {
1608+
Directory directory = null;
1609+
boolean useCompoundFile = segmentCommitInfo.info.getUseCompoundFile();
1610+
if (useCompoundFile) {
1611+
try {
1612+
directory = segmentCommitInfo.info.getCodec().compoundFormat()
1613+
.getCompoundReader(segmentCommitInfo.info.dir, segmentCommitInfo.info, IOContext.READ);
1614+
} catch (IOException e) {
1615+
store.logger.warn("Error when opening compound reader for Directory [{}] and SegmentCommitInfo [{}]", e,
1616+
segmentCommitInfo.info.dir, segmentCommitInfo);
1617+
}
1618+
} else {
1619+
directory = segmentCommitInfo.info.dir;
1620+
}
1621+
1622+
// Couldn't open compound reader, skip
1623+
if (directory == null) {
1624+
continue;
1625+
}
1626+
1627+
files = new String[0];
1628+
try {
1629+
files = directory.listAll();
1630+
} catch (IOException e) {
1631+
store.logger.warn("Couldn't list Directory [{}]", e, directory);
1632+
}
1633+
1634+
for (String file : files) {
1635+
String extension = IndexFileNames.getExtension(file);
1636+
if (extension == null) {
1637+
continue;
1638+
}
1639+
1640+
try {
1641+
long length = directory.fileLength(file);
1642+
map.put(extension, length);
1643+
} catch (NoSuchFileException | FileNotFoundException e) {
1644+
store.logger.trace("Tried to query fileLength but file is gone, Directory [{}], file [{}]", e, directory, file);
1645+
} catch (IOException e) {
1646+
store.logger.warn("Error when trying to query fileLength in Directory [{}] and file [{}]", e, directory, file);
1647+
}
1648+
}
1649+
1650+
if (useCompoundFile && directory != null) {
1651+
try {
1652+
directory.close();
1653+
} catch (IOException e) {
1654+
store.logger.warn("Error when closing compound reader on Directory [{}]", e, directory);
1655+
}
1656+
}
1657+
}
1658+
1659+
return new StoreStats(estimatedSize, map.build(), throttleTimeInNanos);
15781660
}
15791661
}
15801662

‎core/src/main/java/org/elasticsearch/index/store/StoreStats.java‎

Lines changed: 71 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@
1919

2020
package org.elasticsearch.index.store;
2121

22+
import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
23+
24+
import org.elasticsearch.Version;
25+
import org.elasticsearch.common.collect.ImmutableOpenMap;
2226
import org.elasticsearch.common.io.stream.StreamInput;
2327
import org.elasticsearch.common.io.stream.StreamOutput;
2428
import org.elasticsearch.common.io.stream.Streamable;
@@ -29,21 +33,45 @@
2933
import org.elasticsearch.common.xcontent.XContentBuilderString;
3034

3135
import java.io.IOException;
36+
import java.util.Iterator;
3237

3338
/**
3439
*/
3540
public class StoreStats implements Streamable, ToXContent {
3641

3742
private long sizeInBytes;
3843

44+
private ImmutableOpenMap<String, Long> detail = ImmutableOpenMap.of();
45+
3946
private long throttleTimeInNanos;
4047

48+
private static ImmutableOpenMap<String, String> descriptions = ImmutableOpenMap.<String, String>builder()
49+
.fPut("si", "Segment Info")
50+
.fPut("fnm", "Fields")
51+
.fPut("fdx", "Field Index")
52+
.fPut("fdt", "Field Data")
53+
.fPut("tim", "Term Dictionary")
54+
.fPut("tip", "Term Index")
55+
.fPut("doc", "Frequencies")
56+
.fPut("pos", "Positions")
57+
.fPut("pay", "Payloads")
58+
.fPut("nvd", "Norms")
59+
.fPut("nvm", "Norms")
60+
.fPut("dvd", "DocValues")
61+
.fPut("dvm", "DocValues")
62+
.fPut("tvx", "Term Vector Index")
63+
.fPut("tvd", "Term Vector Documents")
64+
.fPut("tvf", "Term Vector Fields")
65+
.fPut("liv", "Live Documents")
66+
.build();
67+
4168
public StoreStats() {
4269

4370
}
4471

45-
public StoreStats(long sizeInBytes, long throttleTimeInNanos) {
72+
public StoreStats(long sizeInBytes, ImmutableOpenMap<String, Long> detail, long throttleTimeInNanos) {
4673
this.sizeInBytes = sizeInBytes;
74+
this.detail = detail;
4775
this.throttleTimeInNanos = throttleTimeInNanos;
4876
}
4977

@@ -52,10 +80,22 @@ public void add(StoreStats stats) {
5280
return;
5381
}
5482
sizeInBytes += stats.sizeInBytes;
83+
84+
ImmutableOpenMap.Builder<String, Long> map = ImmutableOpenMap.builder(this.detail);
85+
for (Iterator<ObjectObjectCursor<String, Long>> it = stats.detail.iterator(); it.hasNext();) {
86+
ObjectObjectCursor<String, Long> entry = it.next();
87+
if (map.containsKey(entry.key)) {
88+
long oldValue = map.get(entry.key);
89+
map.put(entry.key, oldValue + entry.value);
90+
} else {
91+
map.put(entry.key, entry.value);
92+
}
93+
}
94+
95+
this.detail = map.build();
5596
throttleTimeInNanos += stats.throttleTimeInNanos;
5697
}
5798

58-
5999
public long sizeInBytes() {
60100
return sizeInBytes;
61101
}
@@ -90,19 +130,46 @@ public static StoreStats readStoreStats(StreamInput in) throws IOException {
90130
public void readFrom(StreamInput in) throws IOException {
91131
sizeInBytes = in.readVLong();
92132
throttleTimeInNanos = in.readVLong();
133+
if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
134+
int size = in.readVInt();
135+
ImmutableOpenMap.Builder<String, Long> map = ImmutableOpenMap.builder(size);
136+
for (int i = 0; i < size; i++) {
137+
String key = in.readString();
138+
Long value = in.readLong();
139+
map.put(key, value);
140+
}
141+
detail = map.build();
142+
}
93143
}
94144

95145
@Override
96146
public void writeTo(StreamOutput out) throws IOException {
97147
out.writeVLong(sizeInBytes);
98148
out.writeVLong(throttleTimeInNanos);
149+
if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
150+
out.writeVInt(detail.size());
151+
for (Iterator<ObjectObjectCursor<String, Long>> it = detail.iterator(); it.hasNext();) {
152+
ObjectObjectCursor<String, Long> entry = it.next();
153+
out.writeString(entry.key);
154+
out.writeLong(entry.value);
155+
}
156+
}
99157
}
100158

101159
@Override
102160
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
103161
builder.startObject(Fields.STORE);
104162
builder.byteSizeField(Fields.SIZE_IN_BYTES, Fields.SIZE, sizeInBytes);
105163
builder.timeValueField(Fields.THROTTLE_TIME_IN_MILLIS, Fields.THROTTLE_TIME, throttleTime());
164+
builder.startObject(Fields.DETAIL);
165+
for (Iterator<ObjectObjectCursor<String, Long>> it = detail.iterator(); it.hasNext();) {
166+
ObjectObjectCursor<String, Long> entry = it.next();
167+
builder.startObject(entry.key);
168+
builder.byteSizeField(Fields.SIZE_IN_BYTES, Fields.SIZE, entry.value);
169+
builder.field(Fields.DESCRIPTION, descriptions.getOrDefault(entry.key, "Others"));
170+
builder.endObject();
171+
}
172+
builder.endObject();
106173
builder.endObject();
107174
return builder;
108175
}
@@ -113,5 +180,7 @@ static final class Fields {
113180
static final XContentBuilderString SIZE_IN_BYTES = new XContentBuilderString("size_in_bytes");
114181
static final XContentBuilderString THROTTLE_TIME = new XContentBuilderString("throttle_time");
115182
static final XContentBuilderString THROTTLE_TIME_IN_MILLIS = new XContentBuilderString("throttle_time_in_millis");
183+
static final XContentBuilderString DETAIL = new XContentBuilderString("detail");
184+
static final XContentBuilderString DESCRIPTION = new XContentBuilderString("description");
116185
}
117186
}

‎core/src/test/java/org/elasticsearch/cluster/DiskUsageTests.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,13 +98,13 @@ public void testFillShardLevelInfo() {
9898
ShardRoutingHelper.moveToStarted(test_0);
9999
Path test0Path = createTempDir().resolve("indices").resolve("test").resolve("0");
100100
CommonStats commonStats0 = new CommonStats();
101-
commonStats0.store = new StoreStats(100, 1);
101+
commonStats0.store = new StoreStats(100L, ImmutableOpenMap.of(), 1);
102102
ShardRouting test_1 = ShardRouting.newUnassigned(index, 1, null, false, new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
103103
ShardRoutingHelper.initialize(test_1, "node2");
104104
ShardRoutingHelper.moveToStarted(test_1);
105105
Path test1Path = createTempDir().resolve("indices").resolve("test").resolve("1");
106106
CommonStats commonStats1 = new CommonStats();
107-
commonStats1.store = new StoreStats(1000, 1);
107+
commonStats1.store = new StoreStats(1000L, ImmutableOpenMap.of(), 1);
108108
ShardStats[] stats = new ShardStats[] {
109109
new ShardStats(test_0, new ShardPath(false, test0Path, test0Path, "0xdeadbeef", test_0.shardId()), commonStats0 , null),
110110
new ShardStats(test_1, new ShardPath(false, test1Path, test1Path, "0xdeadbeef", test_1.shardId()), commonStats1 , null)

0 commit comments

Comments
 (0)