added secure rest test to verify that monitor clean up works when cre…

…ate request by user without delete permission fails partially

Signed-off-by: Surya Sashank Nistala <snistala@amazon.com>

alerting/src/test/kotlin/org/opensearch/alerting/MonitorDataSourcesIT.kt

@@ -24,6 +24,7 @@ import org.opensearch.alerting.action.SearchMonitorAction
 import org.opensearch.alerting.action.SearchMonitorRequest
 import org.opensearch.alerting.alerts.AlertIndices
 import org.opensearch.alerting.core.ScheduledJobIndices
+import org.opensearch.alerting.model.DocumentLevelTriggerRunResult
 import org.opensearch.alerting.transport.AlertingSingleNodeTestCase
 import org.opensearch.alerting.util.DocLevelMonitorQueries
 import org.opensearch.alerting.util.DocLevelMonitorQueries.Companion.INDEX_PATTERN_SUFFIX
@@ -808,6 +809,47 @@ class MonitorDataSourcesIT : AlertingSingleNodeTestCase() {
         }
     }
 
+    fun `test execute monitor without create when no monitors exists`() {
+        val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3")
+        val docLevelInput = DocLevelMonitorInput("description", listOf(index), listOf(docQuery))
+        val trigger = randomDocumentLevelTrigger(condition = ALWAYS_RUN)
+        val customQueryIndex = "custom_alerts_index"
+        val analyzer = "whitespace"
+        var monitor = randomDocumentLevelMonitor(
+            inputs = listOf(docLevelInput),
+            triggers = listOf(trigger),
+            dataSources = DataSources(
+                queryIndex = customQueryIndex,
+                queryIndexMappingsByType = mapOf(Pair("text", mapOf(Pair("analyzer", analyzer)))),
+            )
+        )
+        var executeMonitorResponse = executeMonitor(monitor, null)
+        val testTime = DateTimeFormatter.ISO_OFFSET_DATE_TIME.format(ZonedDateTime.now().truncatedTo(MILLIS))
+        val testDoc = """{
+            "message" : "This is an error from IAD region",
+            "test_strict_date_time" : "$testTime",
+            "test_field" : "us-west-2"
+        }"""
+
+        assertIndexNotExists(SCHEDULED_JOBS_INDEX)
+
+        val createMonitorResponse = createMonitor(monitor)
+
+        assertIndexExists(SCHEDULED_JOBS_INDEX)
+
+        indexDoc(index, "1", testDoc)
+
+        executeMonitorResponse = executeMonitor(monitor, createMonitorResponse?.id, dryRun = false)
+
+        Assert.assertEquals(executeMonitorResponse!!.monitorRunResult.monitorName, monitor.name)
+        Assert.assertEquals(executeMonitorResponse.monitorRunResult.triggerResults.size, 1)
+        Assert.assertEquals(
+            (executeMonitorResponse.monitorRunResult.triggerResults.iterator().next().value as DocumentLevelTriggerRunResult)
+                .triggeredDocs.size,
+            1
+        )
+    }
+
     fun `test execute monitor with custom query index and custom field mappings`() {
         val docQuery = DocLevelQuery(query = "test_field:\"us-west-2\"", name = "3")
         val docLevelInput = DocLevelMonitorInput("description", listOf(index), listOf(docQuery))

alerting/src/test/kotlin/org/opensearch/alerting/resthandler/SecureMonitorRestApiIT.kt

@@ -45,6 +45,7 @@ import org.opensearch.alerting.randomTemplateScript
 import org.opensearch.client.Response
 import org.opensearch.client.ResponseException
 import org.opensearch.client.RestClient
+import org.opensearch.common.settings.Settings
 import org.opensearch.common.xcontent.LoggingDeprecationHandler
 import org.opensearch.common.xcontent.XContentType
 import org.opensearch.common.xcontent.json.JsonXContent
@@ -1155,6 +1156,51 @@ class SecureMonitorRestApiIT : AlertingRestTestCase() {
         }
     }
 
+    /**
+     * We want to verify that user roles/permissions do not affect clean up of monitors during partial monitor creation failure
+     */
+    fun `test create monitor failure clean up with a user without delete monitor access`() {
+        createUserWithTestDataAndCustomRole(
+            user,
+            TEST_HR_INDEX,
+            TEST_HR_ROLE,
+            listOf(TEST_HR_BACKEND_ROLE),
+            getClusterPermissionsFromCustomRole(ALERTING_NO_ACCESS_ROLE)
+        )
+
+        val refresh = true
+        val docLevelQueryIndex = ".opensearch-alerting-queries-000001"
+        createIndex(
+            docLevelQueryIndex, Settings.EMPTY,
+            """
+                "properties" : {
+                  "query": {  "type": "percolator_ext" },
+                  "monitor_id": { "type" : "text" },
+                  "index": { "type": "text" }
+                }
+            """.trimIndent(),
+            ".opensearch-alerting-queries"
+        )
+        closeIndex(docLevelQueryIndex) // close index to simulate doc level query indexing failure
+        try {
+            createRandomDocumentMonitor(true)
+            fail("Monitor creation should have failed due to error in indexing doc level queries")
+        } catch (e: ResponseException) {
+            val search = SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).toString()
+            val searchResponse = client().makeRequest(
+                "GET", "$ALERTING_BASE_URI/_search",
+                emptyMap(),
+                StringEntity(search, ContentType.APPLICATION_JSON)
+            )
+            val xcp = createParser(XContentType.JSON.xContent(), searchResponse.entity.content)
+            val hits = xcp.map()["hits"]!! as Map<String, Map<String, Any>>
+            val numberDocsFound = hits["total"]?.get("value")
+            assertEquals("Monitors found. Clean up unsuccessful", 0, numberDocsFound)
+        } finally {
+            deleteRoleAndRoleMapping(TEST_HR_ROLE)
+        }
+    }
+
     fun `test query all alerts in all states with disabled filter by`() {
 
         disableFilterBy()