Skip to content

Commit

Permalink
#593: send test status
Browse files Browse the repository at this point in the history
  • Loading branch information
@bhecquet
bhecquet committed Oct 16, 2023
1 parent b3b73d0 commit 4654123
Show file tree
Hide file tree
Showing 10 changed files with 136 additions and 75 deletions.
6 changes: 4 additions & 2 deletions ...main/java/com/seleniumtests/connectors/selenium/SeleniumRobotSnapshotServerConnector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ public class SeleniumRobotSnapshotServerConnector extends SeleniumRobotServerCon
private static final String FIELD_TEST_STEPS = "testSteps";
private static final String FIELD_SESSION = "session";
private static final String FIELD_TEST_CASE = "testCase";
private static final String FIELD_STATUS = "status";
public static final String SESSION_API_URL = "/snapshot/api/session/";
public static final String TESTCASEINSESSION_API_URL = "/snapshot/api/testcaseinsession/";
public static final String TESTSTEP_API_URL = "/snapshot/api/teststep/";
Expand Down Expand Up @@ -164,7 +165,7 @@ public Integer createSession(String sessionName) {
*/
@Deprecated
public Integer createTestCaseInSession(Integer sessionId, Integer testCaseId) {
return createTestCaseInSession(sessionId, testCaseId, "");
return createTestCaseInSession(sessionId, testCaseId, "", "UNKNOWN");
}

/**
Expand All @@ -174,7 +175,7 @@ public Integer createTestCaseInSession(Integer sessionId, Integer testCaseId) {
* @param name name of the test case in this session. This is to distinguish the test case (e.g: 'test1') and its full name (e.g: 'test1-1'), when executed with dataprovider
* @return the id of the created testCaseInSession
*/
public Integer createTestCaseInSession(Integer sessionId, Integer testCaseId, String name) {
public Integer createTestCaseInSession(Integer sessionId, Integer testCaseId, String name, String status) {
if (!active) {
return null;
}
Expand All @@ -191,6 +192,7 @@ public Integer createTestCaseInSession(Integer sessionId, Integer testCaseId, St
JSONObject testInSessionJson = getJSonResponse(buildPostRequest(url + TESTCASEINSESSION_API_URL)
.field(FIELD_TEST_CASE, testCaseId)
.field(FIELD_SESSION, sessionId.toString())
.field(FIELD_STATUS, status)
.field(FIELD_NAME, strippedName));
return testInSessionJson.getInt("id");
} catch (UnirestException | JSONException | SeleniumRobotServerException e) {
Expand Down
25 changes: 25 additions & 0 deletions core/src/main/java/com/seleniumtests/core/utils/TestNGResultUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -547,4 +547,29 @@ public static boolean isErrorCauseSearchedInReferencePicture(ITestResult testNGR
public static void setErrorCauseSearchedInReferencePicture(ITestResult testNGResult, Boolean errorCauseInReferencePicture) {
testNGResult.setAttribute(ERROR_CAUSE_IN_REFERENCE, errorCauseInReferencePicture);
}

/**
* Returns the string representation of the status: SUCCESS, ERROR, SKIPPED, ...
* @param testNGResult
* @return
*/
public static String getTestStatusString(ITestResult testNGResult) {

switch (testNGResult.getStatus()) {
case -1:
return "CREATED";
case 1:
return "SUCCESS";
case 2:
return "FAILURE";
case 3:
return "SKIP";
case 4:
return "SUCCESS_PERCENTAGE_FAILURE";
case 16:
return "STARTED";
default:
return "UNKNOWN";
}
}
}
2 changes: 1 addition & 1 deletion core/src/main/java/com/seleniumtests/reporter/logger/Snapshot.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ public JSONObject toJson() {
snapshotJson.put("snapshotCheckType", snapshotCheckType.getName());
snapshotJson.put("idImage", (screenshot.getImage() == null || screenshot.getImage().getId() == null) ? JSONObject.NULL: screenshot.getImage().getId());
snapshotJson.put("idHtml", (screenshot.getHtml() == null || screenshot.getHtml().getId() == null) ? JSONObject.NULL: screenshot.getHtml().getId());;
snapshotJson.put("name", screenshot.getImageName());
snapshotJson.put("name", name);
snapshotJson.put("title", screenshot.getTitle());
snapshotJson.put("url", screenshot.getLocation());
snapshotJson.put("displayInReport", displayInReport);
Expand Down
2 changes: 1 addition & 1 deletion core/src/main/java/com/seleniumtests/reporter/reporters/SeleniumRobotServerTestRecorder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ private void recordResults(SeleniumRobotSnapshotServerConnector serverConnector,

// record test case
Integer testCaseId = serverConnector.createTestCase(testName);
Integer testCaseInSessionId = serverConnector.createTestCaseInSession(sessionId, testCaseId, getVisualTestName(testResult));
Integer testCaseInSessionId = serverConnector.createTestCaseInSession(sessionId, testCaseId, getVisualTestName(testResult), TestNGResultUtils.getTestStatusString(testResult));
serverConnector.addLogsToTestCaseInSession(testCaseInSessionId, generateExecutionLogs(testResult).toString());

List<TestStep> testSteps = TestNGResultUtils.getSeleniumRobotTestContext(testResult).getTestStepManager().getTestSteps();
Expand Down
12 changes: 6 additions & 6 deletions ...ava/com/seleniumtests/it/connector/selenium/TestSeleniumRobotSnapshotServerConnector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ public void testCreateTestCase() {
public void testCreateTestCaseInSession() {
Integer sessionId = connector.createSession("Session1");
Integer testCaseId = connector.createTestCase("Test 1");
Integer testCaseInSessionId = connector.createTestCaseInSession(sessionId, testCaseId, "Test 1");
Integer testCaseInSessionId = connector.createTestCaseInSession(sessionId, testCaseId, "Test 1", "SUCCESS");
Assert.assertNotNull(sessionId);
Assert.assertNotNull(testCaseInSessionId);
Assert.assertNotNull(connector.getApplicationId());
Expand All @@ -115,7 +115,7 @@ public void testCreateTestCaseInSession() {
public void testCreateTestStep() {
Integer sessionId = connector.createSession("Session1");
Integer testCaseId = connector.createTestCase("Test 1");
Integer testCaseInSessionId = connector.createTestCaseInSession(sessionId, testCaseId, "Test 1");
Integer testCaseInSessionId = connector.createTestCaseInSession(sessionId, testCaseId, "Test 1", "SUCCESS");
Integer testStepId = connector.createTestStep("Step 1", testCaseInSessionId);
connector.createTestStep("Step 2", testCaseInSessionId);
Assert.assertNotNull(testStepId);
Expand All @@ -129,7 +129,7 @@ public void testCreateTestStep() {
public Integer testCreateStepReferenceSnapshot() throws IOException {
Integer sessionId = connector.createSession("Session1");
Integer testCaseId = connector.createTestCase("Test 2");
Integer testCaseInSessionId = connector.createTestCaseInSession(sessionId, testCaseId, "Test 2");
Integer testCaseInSessionId = connector.createTestCaseInSession(sessionId, testCaseId, "Test 2", "SUCCESS");
Integer testStepId = connector.createTestStep("Step 1", testCaseInSessionId);
Integer stepResultId = connector.recordStepResult(true, "logs", 1, sessionId, testCaseInSessionId, testStepId);

Expand Down Expand Up @@ -157,7 +157,7 @@ public void testCreateSnapshot() throws IOException {

Integer sessionId = connector.createSession("Session1");
Integer testCaseId = connector.createTestCase("Test 2");
Integer testCaseInSessionId = connector.createTestCaseInSession(sessionId, testCaseId, "Test 2");
Integer testCaseInSessionId = connector.createTestCaseInSession(sessionId, testCaseId, "Test 2", "SUCCESS");
Integer testStepId = connector.createTestStep("Step 1", testCaseInSessionId);
Integer stepResultId = connector.recordStepResult(true, "logs", 1, sessionId, testCaseInSessionId, testStepId);
File image = Paths.get(SeleniumTestsContextManager.getThreadContext().getOutputDirectory(), "img.png").toFile();
Expand All @@ -181,7 +181,7 @@ public void testCheckSnapshotHasNoDifferences() throws IOException {

Integer sessionId = connector.createSession("Session1");
Integer testCaseId = connector.createTestCase("Test 2");
Integer testCaseInSessionId = connector.createTestCaseInSession(sessionId, testCaseId, "Test 2");
Integer testCaseInSessionId = connector.createTestCaseInSession(sessionId, testCaseId, "Test 2", "SUCCESS");
Integer testStepId = connector.createTestStep("Step 1", testCaseInSessionId);
Integer stepResultId = connector.recordStepResult(true, "logs", 1, sessionId, testCaseInSessionId, testStepId);
File image = Paths.get(SeleniumTestsContextManager.getThreadContext().getOutputDirectory(), "img.png").toFile();
Expand All @@ -202,7 +202,7 @@ public void testCheckSnapshotHasNoDifferences() throws IOException {
public void testRecordStepResult() throws IOException {
Integer sessionId = connector.createSession("Session1");
Integer testCaseId = connector.createTestCase("Test 2");
Integer testCaseInSessionId = connector.createTestCaseInSession(sessionId, testCaseId, "Test 2");
Integer testCaseInSessionId = connector.createTestCaseInSession(sessionId, testCaseId, "Test 2", "SUCCESS");
Integer testStepId = connector.createTestStep("Step 1", testCaseInSessionId);
Integer stepResultId = connector.recordStepResult(true, "some logs", 1, sessionId, testCaseInSessionId, testStepId);

Expand Down
22 changes: 11 additions & 11 deletions core/src/test/java/com/seleniumtests/it/reporter/TestSeleniumRobotServerTestRecorder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,11 +103,11 @@ public void testReportGeneration() throws Exception {
verify(serverConnector).createTestCase("testWithException");
verify(serverConnector).createTestCase("testSkipped");
verify(serverConnector, times(5)).addLogsToTestCaseInSession(anyInt(), anyString());
verify(serverConnector).createTestCaseInSession(anyInt(), anyInt(), eq("testAndSubActions"));
verify(serverConnector).createTestCaseInSession(anyInt(), anyInt(), eq("testInError"));
verify(serverConnector).createTestCaseInSession(anyInt(), anyInt(), eq("testWithException"));
verify(serverConnector).createTestCaseInSession(anyInt(), anyInt(), eq("testSkipped"));
verify(serverConnector).createTestCaseInSession(anyInt(), anyInt(), eq("A test which is <OK> é&")); // a test with custom name
verify(serverConnector).createTestCaseInSession(anyInt(), anyInt(), eq("testAndSubActions"), eq("SUCCESS"));
verify(serverConnector).createTestCaseInSession(anyInt(), anyInt(), eq("testInError"), eq("FAILURE"));
verify(serverConnector).createTestCaseInSession(anyInt(), anyInt(), eq("testWithException"), eq("FAILURE"));
verify(serverConnector).createTestCaseInSession(anyInt(), anyInt(), eq("testSkipped"), eq("SKIP"));
verify(serverConnector).createTestCaseInSession(anyInt(), anyInt(), eq("A test which is <OK> é&"), eq("SUCCESS")); // a test with custom name
verify(serverConnector, times(4)).createTestStep(eq("step 1"), anyInt());
verify(serverConnector).createTestStep(eq("step 2"), anyInt());
verify(serverConnector).createSnapshot(any(Snapshot.class), anyInt(), anyInt(), anyInt(), eq(new ArrayList<>())); // two snapshots but only once is sent because the other has no name
Expand All @@ -122,8 +122,8 @@ public void testReportGeneration() throws Exception {
verify(serverConnector).uploadFile(eq(Paths.get(SeleniumTestsContextManager.getGlobalContext().getOutputDirectory(), "testWithException", "htmls", "testWithException_0-1_step_1--tened.html").toFile()), anyInt());
verify(serverConnector).uploadFile(eq(Paths.get(SeleniumTestsContextManager.getGlobalContext().getOutputDirectory(), "testWithException", "screenshots", "testWithException_0-1_step_1--rtened.png").toFile()), anyInt());
// check image id and html id has been updated once files has been uploaded, for tests where snapshots has been uploaded
verify(serverConnector).updateStepResult(contains("\"snapshots\":[{\"idHtml\":0,\"displayInReport\":true,\"name\":\"testWithException_0-1_step_1--rtened.png\",\"idImage\":0,\"failed\":false,\"position\":0,\"type\":\"snapshot\",\"snapshotCheckType\":\"NONE\""), eq(0));
verify(serverConnector).updateStepResult(contains("\"snapshots\":[{\"idHtml\":null,\"displayInReport\":true,\"name\":\"testAndSubActions_0-1_step_1--rtened.png\""), eq(0));
verify(serverConnector).updateStepResult(contains("\"snapshots\":[{\"idHtml\":0,\"displayInReport\":true,\"name\":\"a name\",\"idImage\":0,\"failed\":false,\"position\":0,\"type\":\"snapshot\",\"snapshotCheckType\":\"NONE\""), eq(0));
verify(serverConnector).updateStepResult(contains("\"snapshots\":[{\"idHtml\":null,\"displayInReport\":true,\"name\":\"main\""), eq(0));

// check logs has been uploaded (one upload for each test)
verify(serverConnector, times(5)).uploadLogs(any(File.class), eq(0));
Expand Down Expand Up @@ -158,7 +158,7 @@ public void testReportGenerationServerInactive() throws Exception {
verify(serverConnector, never()).createSession(anyString());
verify(serverConnector, never()).createTestCase(anyString());
verify(serverConnector, never()).addLogsToTestCaseInSession(anyInt(), anyString());
verify(serverConnector, never()).createTestCaseInSession(anyInt(), anyInt(), anyString());
verify(serverConnector, never()).createTestCaseInSession(anyInt(), anyInt(), anyString(), anyString());
verify(serverConnector, never()).createTestStep(anyString(), anyInt());
verify(serverConnector, never()).createSnapshot(any(Snapshot.class), anyInt(), anyInt(), anyInt(), eq(new ArrayList<>())); // two snapshots but only once is sent because the other has no name

Expand Down Expand Up @@ -195,7 +195,7 @@ public void testReportGenerationWithSnapshots() throws Exception {
// issue #331: check all test cases are created, call MUST be done only once to avoid result to be recorded several times
verify(serverConnector).createTestCase("testDriverCustomSnapshot");
verify(serverConnector).addLogsToTestCaseInSession(anyInt(), anyString());
verify(serverConnector).createTestCaseInSession(anyInt(), anyInt(), eq("testDriverCustomSnapshot"));
verify(serverConnector).createTestCaseInSession(anyInt(), anyInt(), eq("testDriverCustomSnapshot"), eq("SUCCESS"));
verify(serverConnector).createTestStep(eq("_captureSnapshot with args: (my snapshot, )"), anyInt());
verify(serverConnector).createSnapshot(any(Snapshot.class), anyInt(), anyInt(), anyInt(), listArgument.capture()); // 1 custom snapshot taken with name

Expand Down Expand Up @@ -235,7 +235,7 @@ public void testReportGenerationWithoutSnapshots() throws Exception {
// issue #331: check all test cases are created, call MUST be done only once to avoid result to be recorded several times
verify(serverConnector).createTestCase("testDriverCustomSnapshot");
verify(serverConnector).addLogsToTestCaseInSession(anyInt(), anyString());
verify(serverConnector).createTestCaseInSession(anyInt(), anyInt(), eq("testDriverCustomSnapshot"));
verify(serverConnector).createTestCaseInSession(anyInt(), anyInt(), eq("testDriverCustomSnapshot"), eq("SUCCESS"));
verify(serverConnector).createTestStep(eq("_captureSnapshot with args: (my snapshot, )"), anyInt());
verify(serverConnector, never()).createSnapshot(any(Snapshot.class), anyInt(), anyInt(), anyInt(), eq(new ArrayList<>())); // 1 custom snapshot taken with name
verify(serverConnector, never()).createExcludeZones(any(Rectangle.class), anyInt()); // one exclude zone created with that snapshot
Expand Down Expand Up @@ -281,7 +281,7 @@ public void testNoReportWhenServerIsOffline() throws Exception {

// check all test cases are created, in both test classes
verify(serverConnector, never()).createTestCase(anyString());
verify(serverConnector, never()).createTestCaseInSession(anyInt(), anyInt(), anyString());
verify(serverConnector, never()).createTestCaseInSession(anyInt(), anyInt(), anyString(), anyString());
verify(serverConnector, never()).createTestStep(anyString(), anyInt());

} finally {
Expand Down
2 changes: 1 addition & 1 deletion core/src/test/java/com/seleniumtests/it/stubclasses/StubTestClass.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,7 @@ public void testWithException() throws IOException {
File tmpImg2 = File.createTempFile("img", "_with_very_very_very_long_name_to_be_shortened.png");
File tmpHtml2 = File.createTempFile("html", "_with_very_very_very_long_name_to_be_shortened.html");
ScreenShot screenshot2 = new ScreenShot(tmpImg2, tmpHtml2);
step1.addSnapshot(new Snapshot(screenshot2, null, SnapshotCheckType.NONE), 1, null);
step1.addSnapshot(new Snapshot(screenshot2, "a name", SnapshotCheckType.NONE), 1, null);


step1.addAction(new TestAction(String.format("played %d times", count), false, new ArrayList<>()));
Expand Down
Loading

0 comments on commit 4654123

Please sign in to comment.