Skip to content

Commit

Permalink
Improve IT coverage of all AD & Alerting tools (opensearch-project#165)…
Browse files Browse the repository at this point in the history 
… (opensearch-project#167)

* Add search alert IT cases

* Add more IT

* Turn off returning ad job

* Add AD results IT; clean up constants

* Use sample monitor for ingestion

---------

(cherry picked from commit 21e964c)

Signed-off-by: Tyler Ohlsen <ohltyler@amazon.com>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Signed-off-by: yuye-aws <yuyezhu@amazon.com>
  • Loading branch information
@opensearch-trigger-bot @github-actions @yuye-aws
2 people authored and yuye-aws committed Apr 26, 2024
1 parent a013378 commit c66e109
Show file tree
Hide file tree
Showing 20 changed files with 2,177 additions and 115 deletions.
4 changes: 2 additions & 2 deletions src/main/java/org/opensearch/agent/tools/SearchAnomalyDetectorsTool.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
public class SearchAnomalyDetectorsTool implements Tool {
public static final String TYPE = "SearchAnomalyDetectorsTool";
private static final String DEFAULT_DESCRIPTION =
"This is a tool that searches anomaly detectors. It takes 12 optional arguments named detectorName which is the explicit name of the monitor (default is null), and detectorNamePattern which is a wildcard query to match detector name (default is null), and indices which defines the index being detected (default is null), and highCardinality which defines whether the anomaly detector is high cardinality (synonymous with multi-entity) of non-high-cardinality (synonymous with single-entity) (default is null, indicating both), and lastUpdateTime which defines the latest update time of the anomaly detector in epoch milliseconds (default is null), and sortOrder which defines the order of the results (options are asc or desc, and default is asc), and sortString which defines how to sort the results (default is name.keyword), and size which defines the size of the request to be returned (default is 20), and startIndex which defines the paginated index to start from (default is 0), and running which defines whether the anomaly detector is running (default is null, indicating both), and disabled which defines whether the anomaly detector is disabled (default is null, indicating both), and failed which defines whether the anomaly detector has failed (default is null, indicating both). The tool returns 2 values: a list of anomaly detectors (each containing the detector id, detector name, detector type indicating multi-entity or single-entity (where multi-entity also means high-cardinality), detector description, name of the configured index, last update time in epoch milliseconds), and the total number of anomaly detectors.";
"This is a tool that searches anomaly detectors. It takes 12 optional arguments named detectorName which is the explicit name of the monitor (default is null), and detectorNamePattern which is a wildcard query to match detector name (default is null), and indices which defines the index or index pattern the detector is detecting over (default is null), and highCardinality which defines whether the anomaly detector is high cardinality (synonymous with multi-entity) of non-high-cardinality (synonymous with single-entity) (default is null, indicating both), and lastUpdateTime which defines the latest update time of the anomaly detector in epoch milliseconds (default is null), and sortOrder which defines the order of the results (options are asc or desc, and default is asc), and sortString which defines how to sort the results (default is name.keyword), and size which defines the size of the request to be returned (default is 20), and startIndex which defines the paginated index to start from (default is 0), and running which defines whether the anomaly detector is running (default is null, indicating both), and disabled which defines whether the anomaly detector is disabled (default is null, indicating both), and failed which defines whether the anomaly detector has failed (default is null, indicating both). The tool returns 2 values: a list of anomaly detectors (each containing the detector id, detector name, detector type indicating multi-entity or single-entity (where multi-entity also means high-cardinality), detector description, name of the configured index, last update time in epoch milliseconds), and the total number of anomaly detectors.";

@Setter
@Getter
Expand Down Expand Up @@ -160,7 +160,7 @@ public <T> void run(Map<String, String> parameters, ActionListener<T> listener)
GetAnomalyDetectorRequest profileRequest = new GetAnomalyDetectorRequest(
hit.getId(),
Versions.MATCH_ANY,
true,
false,
true,
"",
"",
Expand Down
3 changes: 3 additions & 0 deletions src/main/java/org/opensearch/agent/tools/utils/ToolConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,10 @@ public static enum DetectorStateString {
// System indices constants are not cleanly exposed from the AD & Alerting plugins, so we persist our
// own constants here.
public static final String AD_RESULTS_INDEX_PATTERN = ".opendistro-anomaly-results*";
public static final String AD_RESULTS_INDEX = ".opendistro-anomaly-results";
public static final String AD_DETECTORS_INDEX = ".opendistro-anomaly-detectors";

public static final String ALERTING_CONFIG_INDEX = ".opendistro-alerting-config";
public static final String ALERTING_ALERTS_INDEX = ".opendistro-alerting-alerts";

}
13 changes: 13 additions & 0 deletions src/test/java/org/opensearch/integTest/BaseAgentToolsIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,13 @@ protected String indexMonitor(String monitorAsJsonString) {
return parseFieldFromResponse(response, "_id").toString();
}

protected String indexDetector(String detectorAsJsonString) {
Response response = makeRequest(client(), "POST", "_plugins/_anomaly_detection/detectors", null, detectorAsJsonString, null);

assertEquals(RestStatus.CREATED, RestStatus.fromCode(response.getStatusLine().getStatusCode()));
return parseFieldFromResponse(response, "_id").toString();
}

@SneakyThrows
protected Map<String, Object> waitResponseMeetingCondition(
String method,
Expand Down Expand Up @@ -242,6 +249,12 @@ protected void addDocToIndex(String indexName, String docId, List<String> fieldN
assertEquals(RestStatus.CREATED, RestStatus.fromCode(response.getStatusLine().getStatusCode()));
}

@SneakyThrows
protected void addDocToIndex(String indexName, String docId, String contents) {
Response response = makeRequest(client(), "POST", "/" + indexName + "/_doc/" + docId + "?refresh=true", null, contents, null);
assertEquals(RestStatus.CREATED, RestStatus.fromCode(response.getStatusLine().getStatusCode()));
}

public String createAgent(String requestBody) {
Response response = makeRequest(client(), "POST", "/_plugins/_ml/agents/_register", null, requestBody, null);
assertEquals(RestStatus.OK, RestStatus.fromCode(response.getStatusLine().getStatusCode()));
Expand Down
139 changes: 124 additions & 15 deletions src/test/java/org/opensearch/integTest/SearchAlertsToolIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,29 +10,43 @@

import org.junit.After;
import org.junit.Before;
import org.junit.jupiter.api.BeforeEach;
import org.opensearch.agent.tools.utils.ToolConstants;

import com.google.gson.Gson;
import com.google.gson.JsonObject;

import lombok.SneakyThrows;

public class SearchAlertsToolIT extends BaseAgentToolsIT {
private String registerAgentRequestBody;
private String alertsIndexMappings;
private String alertingConfigIndexMappings;
private String sampleAlert;
private static final String monitorId = "foo-id";
private static final String monitorName = "foo-name";
private static final String registerAgentFilepath =
"org/opensearch/agent/tools/alerting/register_flow_agent_of_search_alerts_tool_request_body.json";
private static final String alertsIndexMappingsFilepath = "org/opensearch/agent/tools/alerting/alert_index_mappings.json";
private static final String alertingConfigIndexMappingsFilepath =
"org/opensearch/agent/tools/alerting/alerting_config_index_mappings.json";
private static final String sampleAlertFilepath = "org/opensearch/agent/tools/alerting/sample_alert.json";

@Before
@SneakyThrows
public void setUp() {
super.setUp();
registerAgentRequestBody = Files
.readString(
Path
.of(
this
.getClass()
.getClassLoader()
.getResource("org/opensearch/agent/tools/register_flow_agent_of_search_alerts_tool_request_body.json")
.toURI()
)
);
registerAgentRequestBody = Files.readString(Path.of(this.getClass().getClassLoader().getResource(registerAgentFilepath).toURI()));
alertsIndexMappings = Files.readString(Path.of(this.getClass().getClassLoader().getResource(alertsIndexMappingsFilepath).toURI()));
alertingConfigIndexMappings = Files
.readString(Path.of(this.getClass().getClassLoader().getResource(alertingConfigIndexMappingsFilepath).toURI()));
sampleAlert = Files.readString(Path.of(this.getClass().getClassLoader().getResource(sampleAlertFilepath).toURI()));
}

@BeforeEach
@SneakyThrows
public void prepareTest() {
deleteSystemIndices();
}

@After
Expand All @@ -45,13 +59,108 @@ public void tearDown() {

@SneakyThrows
public void testSearchAlertsToolInFlowAgent_withNoSystemIndex() {
deleteSystemIndices();
String agentId = createAgent(registerAgentRequestBody);
String agentInput = "{\"parameters\":{\"monitorId\": \"" + monitorId + "\"}}";
String agentInput = "{\"parameters\":{}}";
String result = executeAgent(agentId, agentInput);
assertEquals("Alerts=[]TotalAlerts=0", result);
}

@SneakyThrows
public void testSearchAlertsToolInFlowAgent_withSystemIndex() {
setupAlertingSystemIndices();
String agentId = createAgent(registerAgentRequestBody);
String agentInput = "{\"parameters\":{}}";
String result = executeAgent(agentId, agentInput);
assertEquals("Alerts=[]TotalAlerts=0", result);
}

// TODO: Add IT to test against sample alerts data
// https://github.com/opensearch-project/skills/issues/136
@SneakyThrows
public void testSearchAlertsToolInFlowAgent_singleAlert_noFilter() {
setupAlertingSystemIndices();
ingestSampleAlert(monitorId, "1");
String agentId = createAgent(registerAgentRequestBody);
String agentInput = "{\"parameters\":{}}";
String result = executeAgent(agentId, agentInput);
assertTrue(result.contains("TotalAlerts=1"));
}

@SneakyThrows
public void testSearchAlertsToolInFlowAgent_singleAlert_filter_match() {
setupAlertingSystemIndices();
ingestSampleAlert(monitorId, "1");
String agentId = createAgent(registerAgentRequestBody);
String agentInput = "{\"parameters\":{\"monitorId\": \"" + monitorId + "\"}}";
String result = executeAgent(agentId, agentInput);
assertTrue(result.contains("TotalAlerts=1"));
}

@SneakyThrows
public void testSearchAlertsToolInFlowAgent_singleAlert_filter_noMatch() {
setupAlertingSystemIndices();
ingestSampleAlert(monitorId, "1");
String agentId = createAgent(registerAgentRequestBody);
String agentInput = "{\"parameters\":{\"monitorId\": \"" + monitorId + "foo" + "\"}}";
String result = executeAgent(agentId, agentInput);
assertTrue(result.contains("TotalAlerts=0"));
}

@SneakyThrows
public void testSearchAlertsToolInFlowAgent_multipleAlerts_noFilter() {
setupAlertingSystemIndices();
ingestSampleAlert(monitorId, "1");
ingestSampleAlert(monitorId + "foo", "2");
ingestSampleAlert(monitorId + "bar", "3");
String agentId = createAgent(registerAgentRequestBody);
String agentInput = "{\"parameters\":{}}";
String result = executeAgent(agentId, agentInput);
assertTrue(result.contains("TotalAlerts=3"));
}

@SneakyThrows
public void testSearchAlertsToolInFlowAgent_multipleAlerts_filter() {
setupAlertingSystemIndices();
ingestSampleAlert(monitorId, "1");
ingestSampleAlert(monitorId + "foo", "2");
ingestSampleAlert(monitorId + "bar", "3");
String agentId = createAgent(registerAgentRequestBody);
String agentInput = "{\"parameters\":{\"monitorId\": \"" + monitorId + "\"}}";
String result = executeAgent(agentId, agentInput);
assertTrue(result.contains("TotalAlerts=1"));
}

@SneakyThrows
public void testSearchAlertsToolInFlowAgent_multipleAlerts_complexParams() {
setupAlertingSystemIndices();
String monitorId2 = monitorId + "2";
String monitorId3 = monitorId + "3";
ingestSampleAlert(monitorId, "1");
ingestSampleAlert(monitorId2, "2");
ingestSampleAlert(monitorId3, "3");
String agentId = createAgent(registerAgentRequestBody);
String agentInput = "{\"parameters\":{\"monitorIds\": "
+ "[ \""
+ monitorId
+ "\", \""
+ monitorId2
+ "\", \""
+ monitorId3
+ "\" ], "
+ "\"sortOrder\": \"asc\", \"sortString\": \"monitor_name.keyword\", \"size\": 3, \"startIndex\": 0, \"severityLevel\": \"ALL\", \"alertState\": \"ALL\" } }";

String result = executeAgent(agentId, agentInput);
assertTrue(result.contains("TotalAlerts=3"));
}

@SneakyThrows
private void setupAlertingSystemIndices() {
createIndexWithConfiguration(ToolConstants.ALERTING_ALERTS_INDEX, alertsIndexMappings);
createIndexWithConfiguration(ToolConstants.ALERTING_CONFIG_INDEX, alertingConfigIndexMappings);
}

private void ingestSampleAlert(String monitorId, String docId) {
JsonObject sampleAlertJson = new Gson().fromJson(sampleAlert, JsonObject.class);
sampleAlertJson.addProperty("monitor_id", monitorId);
addDocToIndex(ToolConstants.ALERTING_ALERTS_INDEX, docId, sampleAlertJson.toString());
}

}
94 changes: 65 additions & 29 deletions src/test/java/org/opensearch/integTest/SearchAnomalyDetectorsToolIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,31 +11,43 @@

import org.junit.After;
import org.junit.Before;
import org.junit.jupiter.api.BeforeEach;
import org.opensearch.agent.tools.utils.ToolConstants;

import com.google.gson.Gson;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;

import lombok.SneakyThrows;

public class SearchAnomalyDetectorsToolIT extends BaseAgentToolsIT {
private String registerAgentRequestBody;
private static final String detectorId = "foo-id";
private String detectorsIndexMappings;
private String sampleDetector;
private String sampleIndexMappings;
private static final String detectorName = "foo-name";
private static final String registerAgentFilepath =
"org/opensearch/agent/tools/anomaly-detection/register_flow_agent_of_search_anomaly_detectors_tool_request_body.json";
private static final String detectorsIndexMappingsFilepath =
"org/opensearch/agent/tools/anomaly-detection/detectors_index_mappings.json";
private static final String sampleDetectorFilepath = "org/opensearch/agent/tools/anomaly-detection/sample_detector.json";
private static final String sampleIndexMappingsFilepath = "org/opensearch/agent/tools/anomaly-detection/sample_index_mappings.json";

@Before
@SneakyThrows
public void setUp() {
super.setUp();
registerAgentRequestBody = Files
.readString(
Path
.of(
this
.getClass()
.getClassLoader()
.getResource("org/opensearch/agent/tools/register_flow_agent_of_search_detectors_tool_request_body.json")
.toURI()
)
);
createDetectorsSystemIndex(detectorId, detectorName);
registerAgentRequestBody = Files.readString(Path.of(this.getClass().getClassLoader().getResource(registerAgentFilepath).toURI()));
detectorsIndexMappings = Files
.readString(Path.of(this.getClass().getClassLoader().getResource(detectorsIndexMappingsFilepath).toURI()));
sampleDetector = Files.readString(Path.of(this.getClass().getClassLoader().getResource(sampleDetectorFilepath).toURI()));
sampleIndexMappings = Files.readString(Path.of(this.getClass().getClassLoader().getResource(sampleIndexMappingsFilepath).toURI()));
}

@BeforeEach
@SneakyThrows
public void prepareTest() {
deleteSystemIndices();
}

@After
Expand All @@ -48,7 +60,6 @@ public void tearDown() {

@SneakyThrows
public void testSearchAnomalyDetectorsToolInFlowAgent_withNoSystemIndex() {
deleteSystemIndices();
String agentId = createAgent(registerAgentRequestBody);
String agentInput = "{\"parameters\":{\"detectorName\": \"" + detectorName + "\"}}";
String result = executeAgent(agentId, agentInput);
Expand All @@ -57,6 +68,9 @@ public void testSearchAnomalyDetectorsToolInFlowAgent_withNoSystemIndex() {

@SneakyThrows
public void testSearchAnomalyDetectorsToolInFlowAgent_noMatching() {
setupADSystemIndices();
setupTestDetectionIndex("test-index");
ingestSampleDetector(detectorName, "test-index");
String agentId = createAgent(registerAgentRequestBody);
String agentInput = "{\"parameters\":{\"detectorName\": \"" + detectorName + "foo" + "\"}}";
String result = executeAgent(agentId, agentInput);
Expand All @@ -65,6 +79,9 @@ public void testSearchAnomalyDetectorsToolInFlowAgent_noMatching() {

@SneakyThrows
public void testSearchAnomalyDetectorsToolInFlowAgent_matching() {
setupADSystemIndices();
setupTestDetectionIndex("test-index");
String detectorId = ingestSampleDetector(detectorName, "test-index");
String agentId = createAgent(registerAgentRequestBody);
String agentInput = "{\"parameters\":{\"detectorName\": \"" + detectorName + "\"}}";
String result = executeAgent(agentId, agentInput);
Expand All @@ -74,20 +91,39 @@ public void testSearchAnomalyDetectorsToolInFlowAgent_matching() {
}

@SneakyThrows
private void createDetectorsSystemIndex(String detectorId, String detectorName) {
createIndexWithConfiguration(
ToolConstants.AD_DETECTORS_INDEX,
"{\n"
+ " \"mappings\": {\n"
+ " \"properties\": {\n"
+ " \"name\": {\n"
+ " \"type\": \"text\",\n"
+ " \"fields\": { \"keyword\": { \"type\": \"keyword\", \"ignore_above\": 256 }}"
+ " }\n"
+ " }\n"
+ " }\n"
+ "}"
);
addDocToIndex(ToolConstants.AD_DETECTORS_INDEX, detectorId, List.of("name"), List.of(detectorName));
public void testSearchAnomalyDetectorsToolInFlowAgent_complexParams() {
setupADSystemIndices();
setupTestDetectionIndex("test-index");
String detectorId = ingestSampleDetector(detectorName, "test-index");
ingestSampleDetector(detectorName + "foo", "test-index");
String agentId = createAgent(registerAgentRequestBody);
String agentInput = "{\"parameters\":{\"detectorName\": \""
+ detectorName
+ "\", \"highCardinality\": false, \"sortOrder\": \"asc\", \"sortString\": \"name.keyword\", \"size\": 10, \"startIndex\": 0 }}";
String result = executeAgent(agentId, agentInput);
assertTrue(result.contains(String.format("id=%s", detectorId)));
assertTrue(result.contains(String.format("name=%s", detectorName)));
assertTrue(result.contains(String.format("TotalAnomalyDetectors=%d", 1)));
}

@SneakyThrows
private void setupADSystemIndices() {
createIndexWithConfiguration(ToolConstants.AD_DETECTORS_INDEX, detectorsIndexMappings);
}

@SneakyThrows
private void setupTestDetectionIndex(String indexName) {
createIndexWithConfiguration(indexName, sampleIndexMappings);
addDocToIndex(indexName, "foo-id", List.of("timestamp", "value"), List.of(1234, 1));
}

private String ingestSampleDetector(String detectorName, String detectionIndex) {
JsonObject sampleDetectorJson = new Gson().fromJson(sampleDetector, JsonObject.class);
JsonArray arr = new JsonArray(1);
arr.add(detectionIndex);
sampleDetectorJson.addProperty("name", detectorName);
sampleDetectorJson.remove("indices");
sampleDetectorJson.add("indices", arr);
return indexDetector(sampleDetectorJson.toString());
}
}
Loading

0 comments on commit c66e109

Please sign in to comment.