Skip to content

Commit 7dcac93

Browse files
anujmodi2021anujmodi2021
committed
Unit Tests
1 parent 2c51975 commit 7dcac93

File tree

2 files changed

+193
-12
lines changed

2 files changed

+193
-12
lines changed

‎hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azurebfs/services/SlidingWindowHdrHistogram.java‎

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,13 @@
2626
import java.util.concurrent.locks.ReentrantLock;
2727
import java.util.concurrent.atomic.AtomicInteger;
2828

29+
import org.apache.hadoop.classification.VisibleForTesting;
30+
2931
public class SlidingWindowHdrHistogram {
3032
private static final Logger LOG = LoggerFactory.getLogger(SlidingWindowHdrHistogram.class);
3133

3234
// Configuration
33-
private final long windowSizeSecondsMillis; // Total analysis window
35+
private final long windowSizeMillis; // Total analysis window
3436
private final long timeSegmentDurationMillis; // Subdivision on analysis window
3537
private final int numSegments;
3638
private final long highestTrackableValue;
@@ -68,29 +70,29 @@ public class SlidingWindowHdrHistogram {
6870
private double tailLatency = 0.0;
6971
private int deviation = 0;
7072

71-
public SlidingWindowHdrHistogram(long windowSizeSeconds,
73+
public SlidingWindowHdrHistogram(long windowSizeMillis,
7274
int numberOfSegments,
7375
int minSampleSize,
7476
int tailLatencyPercentile,
7577
int tailLatencyMinDeviation,
7678
long highestTrackableValue,
7779
int significantFigures,
7880
final AbfsRestOperationType operationType) {
79-
if (windowSizeSeconds <= 0) throw new IllegalArgumentException("windowSizeSeconds > 0");
81+
if (windowSizeMillis <= 0) throw new IllegalArgumentException("windowSizeMillis > 0");
8082
if (numberOfSegments <= 0) throw new IllegalArgumentException("bucketDurationMillis > 0");
8183
if (highestTrackableValue <= 0) throw new IllegalArgumentException("highestTrackableValue > 0");
8284
if (significantFigures < 1 || significantFigures > 5) throw new IllegalArgumentException("significantFigures in [1,5]");
8385

84-
this.windowSizeSecondsMillis = windowSizeSeconds;
85-
this.timeSegmentDurationMillis = windowSizeSeconds/numberOfSegments;
86+
this.windowSizeMillis = windowSizeMillis;
87+
this.numSegments = numberOfSegments;
88+
this.timeSegmentDurationMillis = windowSizeMillis/numberOfSegments;
8689
this.highestTrackableValue = highestTrackableValue;
8790
this.significantFigures = significantFigures;
8891
this.operationType = operationType;
8992
this.minSampleSize = minSampleSize;
9093
this.tailLatencyPercentile = tailLatencyPercentile;
9194
this.tailLatencyMinDeviation = tailLatencyMinDeviation; // 5ms
9295

93-
this.numSegments = (int) Math.ceil((double) this.windowSizeSecondsMillis / timeSegmentDurationMillis);
9496
this.completedSegments = new Histogram[numSegments];
9597
long now = System.currentTimeMillis();
9698
this.currentSegmentStartMillis = alignToSegmentDuration(now);
@@ -100,7 +102,7 @@ public SlidingWindowHdrHistogram(long windowSizeSeconds,
100102
this.tmpForDelta = new Histogram(highestTrackableValue, significantFigures);
101103
this.tmpForMerge = new Histogram(highestTrackableValue, significantFigures);
102104

103-
LOG.debug("[{}] Initialized SlidingWindowHdrHistogram with WindowSize {}, TimeSegmentDur: {}, NumOfSegments: {}", operationType, windowSizeSeconds, timeSegmentDurationMillis, numSegments);
105+
LOG.debug("[{}] Initialized SlidingWindowHdrHistogram with WindowSize {}, TimeSegmentDur: {}, NumOfSegments: {}", operationType, windowSizeMillis, timeSegmentDurationMillis, numSegments);
104106
}
105107

106108
/** Record a single latency value (in your chosen time unit). Thread-safe and lock-free. */
@@ -209,14 +211,11 @@ public void rotateIfNeeded() {
209211
}
210212
}
211213

212-
public long getCurrentTotalCount() {
213-
return currentTotalCount.get();
214-
}
215-
214+
@VisibleForTesting
216215
public double getTailLatency() {
217216
LOG.debug("[{}] Getting Tail Latency. Current total count: {}, Deviation: {}%, p50: {}, Tail Latency: {}, isAnalysisWindowFilled: {}",
218217
operationType, getCurrentTotalCount(), deviation, p50, tailLatency, isAnalysisWindowFilled);
219-
if (!isAnalysisWindowFilled) {
218+
if (!isAnalysisWindowFilled()) {
220219
LOG.debug("[{}] Analysis window not yet filled. Not reporting tail latency", operationType);
221220
return 0.0;
222221
}
@@ -227,4 +226,24 @@ public double getTailLatency() {
227226
}
228227
return tailLatency;
229228
}
229+
230+
@VisibleForTesting
231+
public long getCurrentTotalCount() {
232+
return currentTotalCount.get();
233+
}
234+
235+
@VisibleForTesting
236+
public int getCurrentIndex() {
237+
return currentIndex.get();
238+
}
239+
240+
@VisibleForTesting
241+
public double getP50() {
242+
return p50;
243+
}
244+
245+
@VisibleForTesting
246+
public boolean isAnalysisWindowFilled() {
247+
return isAnalysisWindowFilled;
248+
}
230249
}

‎hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestSlidingWindowHdrHistogram.java‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
/**
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing, software
13+
* distributed under the License is distributed on an "AS IS" BASIS,
14+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
* See the License for the specific language governing permissions and
16+
* limitations under the License.
17+
*/
18+
19+
package org.apache.hadoop.fs.azurebfs.services;
20+
21+
import org.junit.jupiter.api.Test;
22+
23+
import static org.assertj.core.api.Assertions.assertThat;
24+
25+
public class TestSlidingWindowHdrHistogram {
26+
27+
@Test
28+
public void testSlidingWindowHdrHistogram() throws Exception {
29+
SlidingWindowHdrHistogram histogram = new SlidingWindowHdrHistogram(
30+
100,
31+
5,
32+
7,
33+
99,
34+
0,
35+
100,
36+
3,
37+
AbfsRestOperationType.GetPathStatus);
38+
39+
// Verify that the histogram is created successfully with default values and
40+
// do not report any percentiles
41+
assertThat(histogram).isNotNull();
42+
assertThat(histogram.getCurrentTotalCount()).isEqualTo(0);
43+
assertThat(histogram.getCurrentIndex()).isEqualTo(0);
44+
assertThat(histogram.getP50()).isEqualTo(0.0);
45+
assertThat(histogram.getTailLatency()).isEqualTo(0.0);
46+
47+
// Verify that recording values works as expected
48+
addAndRotate(histogram, 10, 5); // Add 5 values of 10
49+
assertThat(histogram.getCurrentTotalCount()).isEqualTo(5);
50+
51+
// Verify that percentiles are not computed with insufficient samples
52+
assertThat(histogram.getP50()).isEqualTo(0.0);
53+
assertThat(histogram.getTailLatency()).isEqualTo(0.0);
54+
55+
// Record more values to exceed the minimum sample size
56+
addAndRotate(histogram, 20, 5); // Add 5 values of 20
57+
58+
// Verify that percentiles are now computed but tail Latency is still not reported
59+
assertThat(histogram.getP50()).isGreaterThan(0.0);
60+
assertThat(histogram.getTailLatency()).isEqualTo(0.0);
61+
62+
// Record more values and rotate histogram to fill whole analysis window
63+
addAndRotate(histogram, 30, 5); // Add 5 values of 30
64+
assertThat(histogram.getCurrentTotalCount()).isEqualTo(15);
65+
66+
// Verify that analysis window is not full until full rotation.
67+
assertThat(histogram.isAnalysisWindowFilled()).isFalse();
68+
69+
addAndRotate(histogram, 60, 5); // Add 5 values of 60
70+
assertThat(histogram.getCurrentTotalCount()).isEqualTo(20);
71+
72+
// Verify that analysis window is not full until full rotation.
73+
assertThat(histogram.isAnalysisWindowFilled()).isFalse();
74+
75+
// Verify that rotation is skipped if nothing new recorded and hence window not filled
76+
addAndRotate(histogram, 100, 0); // No new values added
77+
assertThat(histogram.isAnalysisWindowFilled()).isFalse();
78+
79+
// Verify that rotation does not happen if analysis window is not filled
80+
histogram.rotateIfNeeded();
81+
assertThat(histogram.isAnalysisWindowFilled()).isFalse();
82+
83+
addAndRotate(histogram, 80, 5); // Add 5 values of 80
84+
assertThat(histogram.getCurrentTotalCount()).isEqualTo(25);
85+
86+
// Verify that analysis window is full after full rotation.
87+
assertThat(histogram.isAnalysisWindowFilled()).isTrue();
88+
89+
// Verify that percentiles and tail latency are computed
90+
assertThat(histogram.getP50()).isGreaterThan(0.0);
91+
assertThat(histogram.getTailLatency()).isGreaterThan(0.0);
92+
93+
// Verify that sliding window works. Old values should be evicted
94+
addAndRotate(histogram, 90, 3); // Add 3 values of 90
95+
assertThat(histogram.getCurrentTotalCount()).isEqualTo(23);
96+
assertThat(histogram.isAnalysisWindowFilled()).isTrue();
97+
}
98+
99+
@Test
100+
public void testMinDeviationRequirementNotMet() throws Exception {
101+
SlidingWindowHdrHistogram histogram = new SlidingWindowHdrHistogram(
102+
100,
103+
5,
104+
7,
105+
99,
106+
100,
107+
100,
108+
3,
109+
AbfsRestOperationType.GetPathStatus);
110+
111+
// Add values with low deviation
112+
addAndRotate(histogram, 50, 5); // Add 5 values of 50
113+
addAndRotate(histogram, 51, 5); // Add 5 values of 52
114+
addAndRotate(histogram, 52, 5); // Add 5 values of 51
115+
addAndRotate(histogram, 80, 5); // Add 5 values of 53
116+
addAndRotate(histogram, 90, 5); // Add 5 values of 50
117+
118+
// Verify that analysis window is full after full rotation.
119+
assertThat(histogram.isAnalysisWindowFilled()).isTrue();
120+
121+
// Verify that percentiles are not computed due to low deviation
122+
assertThat(histogram.getP50()).isGreaterThan(0.0);
123+
assertThat(histogram.getTailLatency()).isEqualTo(0.0);
124+
}
125+
126+
@Test
127+
public void testMinDeviationRequirementMet() throws Exception {
128+
SlidingWindowHdrHistogram histogram = new SlidingWindowHdrHistogram(
129+
100,
130+
5,
131+
7,
132+
99,
133+
50,
134+
100,
135+
3,
136+
AbfsRestOperationType.GetPathStatus);
137+
138+
// Add values with low deviation
139+
addAndRotate(histogram, 50, 5); // Add 5 values of 50
140+
addAndRotate(histogram, 51, 5); // Add 5 values of 52
141+
addAndRotate(histogram, 52, 5); // Add 5 values of 51
142+
addAndRotate(histogram, 80, 5); // Add 5 values of 53
143+
addAndRotate(histogram, 90, 5); // Add 5 values of 50
144+
145+
// Verify that analysis window is full after full rotation.
146+
assertThat(histogram.isAnalysisWindowFilled()).isTrue();
147+
148+
// Verify that percentiles are not computed due to low deviation
149+
assertThat(histogram.getP50()).isGreaterThan(0.0);
150+
assertThat(histogram.getTailLatency()).isGreaterThan(0.0);
151+
}
152+
153+
private void addAndRotate(SlidingWindowHdrHistogram histogram, int value, int times)
154+
throws InterruptedException {
155+
for (int i = 0; i < times; i++) {
156+
histogram.recordValue(value);
157+
}
158+
Thread.sleep(200); // Sleep to allow rotation
159+
histogram.rotateIfNeeded();
160+
histogram.computeLatency();
161+
}
162+
}

0 commit comments

Comments
 (0)