Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Perf] Add test case name and description to the performance test chart #413

Merged
merged 3 commits into from
Apr 7, 2023
Merged

[Perf] Add test case name and description to the performance test chart #413

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0d3aa7a
[Perf] Add test case name and description to the perf test chart
taoran6 Apr 6, 2023
8004f88
Merge remote-tracking branch 'origin/main' into taoran/perf_case_name
taoran6 Apr 6, 2023
e2e5cce
Resolve comments
taoran6 Apr 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion agent/src/main/java/com/microsoft/hydralab/agent/runner/monkey/AdbMonkeyRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,7 @@ public long runMonkeyTestOnce(TestRun testRun, Logger logger,
ongoingMonkeyTest.setTestedClass(pkgName);
ongoingMonkeyTest.setDeviceTestResultId(testRun.getId());
ongoingMonkeyTest.setTestTaskId(testRun.getTestTaskId());
testRun.addNewTestUnit(ongoingMonkeyTest);

logger.info(ongoingMonkeyTest.getTitle());
testRunDeviceOrchestrator.addGifFrameAsyncDelay(testRunDevice, agentManagementService.getScreenshotDir(), 2, logger);
Expand Down Expand Up @@ -163,7 +164,6 @@ public long runMonkeyTestOnce(TestRun testRun, Logger logger,
logger.info(ongoingMonkeyTest.getTitle() + ".end");
ongoingMonkeyTest.setEndTimeMillis(System.currentTimeMillis());
testRunDeviceOrchestrator.setRunningTestName(testRunDevice, null);
testRun.addNewTestUnit(ongoingMonkeyTest);
testRun.addNewTimeTag(ongoingMonkeyTest.getTitle() + ".end",
System.currentTimeMillis() - recordingStartTimeMillis);
return checkTime;
Expand Down
2 changes: 1 addition & 1 deletion agent/src/main/java/com/microsoft/hydralab/agent/runner/monkey/AppiumMonkeyRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ protected File runAndGetGif(File appiumJarFile, String appiumCommand, TestRunDev
ongoingMonkeyTest.setTestedClass(pkgName);
ongoingMonkeyTest.setDeviceTestResultId(testRun.getId());
ongoingMonkeyTest.setTestTaskId(testRun.getTestTaskId());
testRun.addNewTestUnit(ongoingMonkeyTest);

logger.info(ongoingMonkeyTest.getTitle());

Expand Down Expand Up @@ -102,7 +103,6 @@ protected File runAndGetGif(File appiumJarFile, String appiumCommand, TestRunDev
performanceTestManagementService.testRunFinished();
ongoingMonkeyTest.setEndTimeMillis(System.currentTimeMillis());
testRunDeviceOrchestrator.setRunningTestName(testRunDevice, null);
testRun.addNewTestUnit(ongoingMonkeyTest);
testRun.addNewTimeTag(ongoingMonkeyTest.getTitle() + ".end",
System.currentTimeMillis() - recordingStartTimeMillis);
testRun.onTestEnded();
Expand Down
2 changes: 1 addition & 1 deletion agent/src/main/java/com/microsoft/hydralab/agent/runner/smart/SmartRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ public void runSmartTestOnce(int i, TestRunDevice testRunDevice, TestRun testRun
ongoingSmartTest.setTestedClass(pkgName);
ongoingSmartTest.setDeviceTestResultId(testRun.getId());
ongoingSmartTest.setTestTaskId(testRun.getTestTaskId());
testRun.addNewTestUnit(ongoingSmartTest);

testRun.addNewTimeTag(unitIndex + ". " + ongoingSmartTest.getTitle(), System.currentTimeMillis() - recordingStartTimeMillis);
testRunDeviceOrchestrator.setRunningTestName(testRunDevice, ongoingSmartTest.getTitle());
Expand Down Expand Up @@ -158,7 +159,6 @@ public void runSmartTestOnce(int i, TestRunDevice testRunDevice, TestRun testRun
ongoingSmartTest.setEndTimeMillis(System.currentTimeMillis());
logger.info(ongoingSmartTest.getTitle() + ".end");
testRunDeviceOrchestrator.setRunningTestName(testRunDevice, null);
testRun.addNewTestUnit(ongoingSmartTest);
testRun.addNewTimeTag(ongoingSmartTest.getTitle() + ".end",
System.currentTimeMillis() - recordingStartTimeMillis);
if (ongoingSmartTest.isSuccess()) {
Expand Down
14 changes: 13 additions & 1 deletion common/src/main/java/com/microsoft/hydralab/common/entity/common/TestRun.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.alibaba.fastjson.annotation.JSONField;
import com.microsoft.hydralab.agent.runner.ITestRun;
import com.microsoft.hydralab.common.util.Const;
import lombok.Data;
Expand All @@ -21,6 +22,7 @@
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.stream.Collectors;

@Data
Expand Down Expand Up @@ -72,7 +74,7 @@ public class TestRun implements Serializable, ITestRun {
@Transient
private List<StorageFileInfo> attachments;
@Transient
private List<PerformanceTestResultEntity> performanceTestResultEntities = new ArrayList<>();
private List<PerformanceTestResultEntity> performanceTestResultEntities = new CopyOnWriteArrayList<>();

@Transient
private transient List<CommandlineAndTime> commandlineAndTimeList = new ArrayList<>();
Expand Down Expand Up @@ -107,6 +109,16 @@ public String getSuccessRate() {
return String.format("%.2f", rate) + '%';
}

@Transient
taoran6 marked this conversation as resolved.
Show resolved Hide resolved
@JSONField(serialize = false)
public String getOngoingTestUnitName() {
if (testUnitList.size() == 0) {
return "";
}
return testUnitList.get(testUnitList.size() - 1).getTestName();
}


public void addNewTestUnit(AndroidTestUnit ongoingTestUnit) {
testUnitList.add(ongoingTestUnit);
}
Expand Down
3 changes: 2 additions & 1 deletion ...on/src/main/java/com/microsoft/hydralab/performance/PerformanceTestManagementService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@
import com.microsoft.hydralab.common.entity.common.PerformanceTestResultEntity;
import com.microsoft.hydralab.common.entity.common.TestRun;
import com.microsoft.hydralab.common.entity.common.TestRunDevice;
import com.microsoft.hydralab.common.management.device.DeviceType;
import com.microsoft.hydralab.common.entity.common.TestTask;
import com.microsoft.hydralab.common.management.device.DeviceType;
import com.microsoft.hydralab.common.util.FileUtil;
import com.microsoft.hydralab.common.util.ThreadPoolUtil;
import com.microsoft.hydralab.performance.inspectors.AndroidBatteryInfoInspector;
Expand Down Expand Up @@ -132,6 +132,7 @@ private PerformanceInspectionResult inspect(PerformanceInspection performanceIns
performanceInspection.resultFolder = inspectorFolder;

PerformanceInspectionResult result = performanceInspector.inspect(performanceInspection);
result.testCaseName = testRun.getOngoingTestUnitName();

testRunPerfResultMap.putIfAbsent(testRun.getId(), new HashMap<>());
Map<String, PerformanceTestResult> performanceTestResultMap = testRunPerfResultMap.get(testRun.getId());
Expand Down
27 changes: 26 additions & 1 deletion react/src/component/PerfTestDashboard.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,22 @@ export default class PerfTestDashboard extends React.Component {
const iosMemoryMetrics = [];
let perfHistoryList = this.state.perfHistoryList;

console.log("render history", perfHistoryList);
const CustomTooltip = ({ active, payload, label }) => {
if (active && payload && payload.length) {
return (
<div style={{ backgroundColor: 'white', border: '1px solid #ccc', whiteSpace: 'nowrap', margin: 0, padding: 10 }}>
<p style={{ margin: 0, padding: 5 }}>{"Time: " + label + "S"}</p>
{payload[0].payload.testCase && payload[0].payload.testCase.length && <p style={{ margin: 0, padding: 5 }}>{"Test case: " + payload[0].payload.testCase}</p>}
{payload.map((key) => (
<p style={{ margin: 0, padding: 5 }}><font color={key.color}>{key.dataKey + ": " + key.value}</font></p>
))}
{payload[0].payload.description && payload[0].payload.description.length && <p style={{ margin: 0, padding: 5 }}>{"Description: " + payload[0].payload.description}</p>}
</div>
)
}

return null;
}

/**
* Android Battery Info
Expand All @@ -128,6 +143,7 @@ export default class PerfTestDashboard extends React.Component {
let result = { ...inspectionResult.parsedData };
result.time = (inspectionResult.timestamp - startTime) / 1000;
result.ratio = inspectionResult.parsedData.ratio * 100;
result.testCase = inspectionResult.testCaseName;
isAndroidBatteryInfoEmpty = false;

androidBatteryMetrics.push(result);
Expand All @@ -152,6 +168,7 @@ export default class PerfTestDashboard extends React.Component {
<XAxis dataKey="time" label={{ value: 'Time', position: 'bottom' }} unit="s" type='number' />
<YAxis yAxisId="left" label={{ value: 'Battery usage (mAh)', angle: -90, position: 'left' }} />
<YAxis yAxisId="right" label={{ value: 'Ratio', angle: -90, position: 'right' }} unit="%" orientation="right" />
<Tooltip content={<CustomTooltip />} />
{this.state.selectedAndroidBatteryOptions.map((key, index) => (
<Line type="monotone" yAxisId={key.value == "ratio" ? "right" : "left"} dataKey={key.value} stroke={key.color} />
))}
Expand Down Expand Up @@ -182,6 +199,7 @@ export default class PerfTestDashboard extends React.Component {
isAndroidMemoryInfoEmpty = false;
}
})
result.testCase = inspectionResult.testCaseName;
androidMemoryMetrics.push(result);
}
})
Expand All @@ -204,6 +222,7 @@ export default class PerfTestDashboard extends React.Component {
<Legend verticalAlign="top" />
<XAxis dataKey="time" label={{ value: 'Time', position: 'bottom' }} unit="s" type='number' />
<YAxis yAxisId="left" label={{ value: 'Memory usage (MB)', angle: -90, position: 'left' }} />
<Tooltip content={<CustomTooltip />} />
{this.state.selectedAndroidMemoryOptions.map((key, index) => (
<Line type="monotone" yAxisId="left" dataKey={key.value} stroke={key.color} dot={false}/>
))}
Expand Down Expand Up @@ -247,6 +266,7 @@ export default class PerfTestDashboard extends React.Component {
}
})

result.testCase = inspectionResult.testCaseName;
windowsMemoryMetrics.push(result);
}
})
Expand All @@ -269,6 +289,7 @@ export default class PerfTestDashboard extends React.Component {
<Legend verticalAlign="top" />
<XAxis dataKey="time" label={{ value: 'Time', position: 'bottom' }} unit="s" type='number' />
<YAxis yAxisId="left" label={{ value: 'Memory usage (MB)', angle: -90, position: 'left' }} />
<Tooltip content={<CustomTooltip />} />
{this.state.selectedWindowsMemoryOptions.map((key, index) => (
<Line type="monotone" yAxisId="left" dataKey={key.value} stroke={key.color} dot={false}/>
))}
Expand All @@ -292,6 +313,7 @@ export default class PerfTestDashboard extends React.Component {
let parsedData = { ...inspectionResult.parsedData };
result.time = (inspectionResult.timestamp - startTime) / 1000;
isIosEnergyInfoEmpty = false;
result.testCase = inspectionResult.testCaseName;

iosEnergyMetrics.push(result);
}
Expand All @@ -315,6 +337,7 @@ export default class PerfTestDashboard extends React.Component {
<Legend verticalAlign="top" />
<XAxis dataKey="time" label={{ value: 'Time', position: 'bottom' }} unit="s" type='number' />
<YAxis yAxisId="left" label={{ value: 'Energy Usage (mW)', angle: -90, position: 'left' }} />
<Tooltip content={<CustomTooltip />} />
{this.state.selectedIosEnergyOptions.map((key, index) => (
<Line type="monotone" yAxisId="left" dataKey={key.value} stroke={key.color} dot={false}/>
))}
Expand All @@ -339,6 +362,7 @@ export default class PerfTestDashboard extends React.Component {
let parsedData = { ...inspectionResult.parsedData };
result.time = (inspectionResult.timestamp - startTime) / 1000;
isIosMemoryInfoEmpty = false;
result.testCase = inspectionResult.testCaseName;
iosMemoryMetrics.push(result);
}
})
Expand All @@ -349,6 +373,7 @@ export default class PerfTestDashboard extends React.Component {
<Legend verticalAlign="top" />
<XAxis dataKey="time" label={{ value: 'Time', position: 'bottom' }} unit="s" type='number' />
<YAxis yAxisId="left" label={{ value: 'Memory usage (MB)', angle: -90, position: 'left' }} />
<Tooltip content={<CustomTooltip />} />
{iosMemoryOptions.map((key, index) => (
<Line type="monotone" yAxisId="left" dataKey={key.value} stroke={key.color} dot={false} />
))}
Expand Down
2 changes: 2 additions & 0 deletions sdk/src/main/java/com/microsoft/hydralab/agent/runner/ITestRun.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,4 +8,6 @@ public interface ITestRun {
String getId();

String getDeviceSerialNumberByType(String type);

String getOngoingTestUnitName();
}
5 changes: 4 additions & 1 deletion sdk/src/main/java/com/microsoft/hydralab/performance/PerformanceInspectionResult.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ public class PerformanceInspectionResult {
@SuppressWarnings("visibilitymodifier")
public final long timestamp;
@SuppressWarnings("visibilitymodifier")
public String testCaseName;
@SuppressWarnings("visibilitymodifier")
public PerformanceInspection inspection;
@SuppressWarnings("visibilitymodifier")
public File rawResultFile;
Expand All @@ -33,8 +35,9 @@ public PerformanceInspectionResult(File rawResultFile, PerformanceInspection ins
public String toString() {
return "PerformanceInspectionResult{" +
"timestamp=" + timestamp +
", testCaseName='" + testCaseName + '\'' +
", inspection=" + inspection +
", rawResultFile=" + rawResultFile.getAbsolutePath() +
", rawResultFile=" + rawResultFile +
taoran6 marked this conversation as resolved.
Show resolved Hide resolved
", parsedData=" + parsedData +
'}';
}
Expand Down