Update CI check for integ-test-with-security to run all integ tests w…

…ith security (opensearch-project#1243)

* Run all integ tests with security

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Use adminClient when deleting ISM index in RestStopRollupActionIT

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Update RolloverActionIT

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Fix RestStopSnapshotManagementIT

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Fix RestChangePolicyActionIT

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Fix more tests

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Fix more tests

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Fix more tests

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Fix RestExplainTransformActionIT

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Fix more tests

Signed-off-by: Craig Perkins <cwperx@amazon.com>

* Make sure read from system index also uses adminClient

Signed-off-by: Craig Perkins <cwperx@amazon.com>

---------

Signed-off-by: Craig Perkins <cwperx@amazon.com>

.github/workflows/security-test-workflow.yml

@@ -40,7 +40,7 @@ jobs:
  - name: Run integration tests
  run: | 
  chown -R 1000:1000 `pwd`
- su `id -un 1000` -c "./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*SecurityBehaviorIT'"
+ su `id -un 1000` -c "./gradlew integTest -Dsecurity=true -Dhttps=true --tests '*IT'"
  - name: Upload failed logs
  uses: actions/upload-artifact@v2
  if: failure()

src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/IndexStateManagementRestTestCase.kt

@@ -351,7 +351,7 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase()
  }
  """.trimIndent()
  val res =
- client().makeRequest(
+ adminClient().makeRequest(
  "PUT", "$index/_settings", emptyMap(),
  StringEntity(body, ContentType.APPLICATION_JSON),
  )
@@ -483,7 +483,7 @@ abstract class IndexStateManagementRestTestCase : IndexManagementRestTestCase()
 
  protected fun updateManagedIndexConfigPolicySeqNo(update: ManagedIndexConfig) {
  val response =
- client().makeRequest(
+ adminClient().makeRequest(
  "POST", "$INDEX_MANAGEMENT_INDEX/_update/${update.id}",
  StringEntity(
  "{\"doc\":{\"managed_index\":{\"policy_seq_no\":\"${update.policySeqNo}\"}}}",

src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/action/RolloverActionIT.kt

@@ -784,7 +784,7 @@ class RolloverActionIT : IndexStateManagementRestTestCase() {
  }
  // Manually produce transaction failure
  val response =
- client().makeRequest(
+ adminClient().makeRequest(
  "POST", "$INDEX_MANAGEMENT_INDEX/_update/${managedIndexConfig.id}%23metadata",
  StringEntity(
  "{\n" +

src/test/kotlin/org/opensearch/indexmanagement/indexstatemanagement/resthandler/RestChangePolicyActionIT.kt

@@ -6,6 +6,7 @@
 package org.opensearch.indexmanagement.indexstatemanagement.resthandler
 
 import org.junit.Before
+import org.opensearch.client.Request
 import org.opensearch.client.ResponseException
 import org.opensearch.common.settings.Settings
 import org.opensearch.core.rest.RestStatus
@@ -35,7 +36,7 @@ import org.opensearch.indexmanagement.spi.indexstatemanagement.model.StateMetaDa
 import org.opensearch.indexmanagement.waitFor
 import org.opensearch.rest.RestRequest
 import java.time.Instant
-import java.util.Locale
+import java.util.*
 
 class RestChangePolicyActionIT : IndexStateManagementRestTestCase() {
  private val testIndexName = javaClass.simpleName.lowercase(Locale.ROOT)
@@ -89,7 +90,10 @@ class RestChangePolicyActionIT : IndexStateManagementRestTestCase() {
  }
 
  fun `test nonexistent ism config index`() {
- if (indexExists(INDEX_MANAGEMENT_INDEX)) deleteIndex(INDEX_MANAGEMENT_INDEX)
+ if (indexExists(INDEX_MANAGEMENT_INDEX)) {
+ val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
+ adminClient().performRequest(deleteISMIndexRequest)
+ }
  try {
  val changePolicy = ChangePolicy("some_id", null, emptyList(), false)
  client().makeRequest(

src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStartRollupActionIT.kt

@@ -5,11 +5,13 @@
 
 package org.opensearch.indexmanagement.rollup.resthandler
 
+import org.opensearch.client.Request
 import org.opensearch.client.ResponseException
 import org.opensearch.common.settings.Settings
 import org.opensearch.core.rest.RestStatus
 import org.opensearch.indexmanagement.IndexManagementIndices
 import org.opensearch.indexmanagement.IndexManagementPlugin
+import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
 import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.ROLLUP_JOBS_BASE_URI
 import org.opensearch.indexmanagement.common.model.dimension.DateHistogram
 import org.opensearch.indexmanagement.indexstatemanagement.util.INDEX_HIDDEN
@@ -22,7 +24,7 @@ import org.opensearch.indexmanagement.waitFor
 import org.opensearch.jobscheduler.spi.schedule.IntervalSchedule
 import java.time.Instant
 import java.time.temporal.ChronoUnit
-import java.util.Locale
+import java.util.*
 
 class RestStartRollupActionIT : RollupRestAPITestCase() {
  private val testName = javaClass.simpleName.lowercase(Locale.ROOT)
@@ -200,7 +202,8 @@ class RestStartRollupActionIT : RollupRestAPITestCase() {
 
  fun `test start rollup when multiple shards configured for IM config index`() {
  // setup ism-config index with multiple primary shards
- deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
+ val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
+ adminClient().performRequest(deleteISMIndexRequest)
  val mapping = IndexManagementIndices.indexManagementMappings.trim().trimStart('{').trimEnd('}')
  val settings =
  Settings.builder()
@@ -253,7 +256,7 @@ class RestStartRollupActionIT : RollupRestAPITestCase() {
 
  // clearing the config index to prevent other tests using this multi shard index
  Thread.sleep(2000L)
- deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
+ adminClient().performRequest(deleteISMIndexRequest)
  Thread.sleep(2000L)
  }
 }

src/test/kotlin/org/opensearch/indexmanagement/rollup/resthandler/RestStopRollupActionIT.kt

@@ -5,11 +5,13 @@
 
 package org.opensearch.indexmanagement.rollup.resthandler
 
+import org.opensearch.client.Request
 import org.opensearch.client.ResponseException
 import org.opensearch.common.settings.Settings
 import org.opensearch.core.rest.RestStatus
 import org.opensearch.indexmanagement.IndexManagementIndices
 import org.opensearch.indexmanagement.IndexManagementPlugin
+import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
 import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.ROLLUP_JOBS_BASE_URI
 import org.opensearch.indexmanagement.common.model.dimension.DateHistogram
 import org.opensearch.indexmanagement.common.model.dimension.Terms
@@ -256,7 +258,8 @@ class RestStopRollupActionIT : RollupRestAPITestCase() {
 
  fun `test stop rollup when multiple shards configured for IM config index`() {
  // setup ism-config index with multiple primary shards
- deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
+ val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
+ adminClient().performRequest(deleteISMIndexRequest)
  val mapping = IndexManagementIndices.indexManagementMappings.trim().trimStart('{').trimEnd('}')
  val settings =
  Settings.builder()
@@ -313,7 +316,7 @@ class RestStopRollupActionIT : RollupRestAPITestCase() {
 
  // clearing the config index to prevent other tests using this multi shard index
  Thread.sleep(2000L)
- deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
+ adminClient().performRequest(deleteISMIndexRequest)
  Thread.sleep(2000L)
  }
 }

src/test/kotlin/org/opensearch/indexmanagement/rollup/runner/RollupRunnerIT.kt

@@ -1466,7 +1466,7 @@ class RollupRunnerIT : RollupRestTestCase() {
  // - Source index with pattern mapping to some closed indices
 
  private fun deleteRollupMetadata(metadataId: String) {
- val response = client().makeRequest("DELETE", "${IndexManagementPlugin.INDEX_MANAGEMENT_INDEX}/_doc/$metadataId")
+ val response = adminClient().makeRequest("DELETE", "${IndexManagementPlugin.INDEX_MANAGEMENT_INDEX}/_doc/$metadataId")
  assertEquals("Unable to delete rollup metadata $metadataId", RestStatus.OK, response.restStatus())
  }
 }

src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/SnapshotManagementRestTestCase.kt

@@ -139,7 +139,7 @@ abstract class SnapshotManagementRestTestCase : IndexManagementRestTestCase() {
  val startTimeMillis = desiredStartTimeMillis ?: (now().toEpochMilli() - millis)
  val waitForActiveShards = if (isMultiNode) "all" else "1"
  val response =
- client().makeRequest(
+ adminClient().makeRequest(
  "POST", "$INDEX_MANAGEMENT_INDEX/_update/${update.id}?wait_for_active_shards=$waitForActiveShards",
  StringEntity(
  "{\"doc\":{\"sm_policy\":{\"schedule\":{\"interval\":{\"start_time\":\"$startTimeMillis\"}}}}}",
@@ -171,7 +171,7 @@ abstract class SnapshotManagementRestTestCase : IndexManagementRestTestCase() {
  val startTimeMillis = desiredStartTimeMillis ?: (now().toEpochMilli() - millis)
  val waitForActiveShards = if (isMultiNode) "all" else "1"
  val response =
- client().makeRequest(
+ adminClient().makeRequest(
  "POST", "$INDEX_MANAGEMENT_INDEX/_update/${update.metadataID}?wait_for_active_shards=$waitForActiveShards",
  StringEntity(
  """

src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestDeleteSnapshotManagementIT.kt

@@ -5,6 +5,7 @@
 
 package org.opensearch.indexmanagement.snapshotmanagement.resthandler
 
+import org.opensearch.client.Request
 import org.opensearch.client.ResponseException
 import org.opensearch.core.rest.RestStatus
 import org.opensearch.indexmanagement.IndexManagementPlugin
@@ -38,7 +39,8 @@ class RestDeleteSnapshotManagementIT : SnapshotManagementRestTestCase() {
 
  fun `test deleting a snapshot management policy that doesn't exist and config index doesnt exist`() {
  try {
- deleteIndex(INDEX_MANAGEMENT_INDEX)
+ val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
+ adminClient().performRequest(deleteISMIndexRequest)
  client().makeRequest("DELETE", "${IndexManagementPlugin.SM_POLICIES_URI}/nonexistent_policy")
  fail("expected 404 ResponseException")
  } catch (e: ResponseException) {

src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestExplainSnapshotManagementIT.kt

@@ -5,6 +5,7 @@
 
 package org.opensearch.indexmanagement.snapshotmanagement.resthandler
 
+import org.opensearch.client.Request
 import org.opensearch.client.ResponseException
 import org.opensearch.common.xcontent.XContentType
 import org.opensearch.core.rest.RestStatus
@@ -141,7 +142,8 @@ class RestExplainSnapshotManagementIT : SnapshotManagementRestTestCase() {
 
  fun `test explain sm policy when config index doesn't exist`() {
  try {
- deleteIndex(INDEX_MANAGEMENT_INDEX)
+ val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
+ adminClient().performRequest(deleteISMIndexRequest)
  explainSMPolicy(randomAlphaOfLength(10))
  fail("expected response exception")
  } catch (e: ResponseException) {

src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestGetSnapshotManagementIT.kt

@@ -7,9 +7,11 @@ package org.opensearch.indexmanagement.snapshotmanagement.resthandler
 
 import org.apache.hc.core5.http.HttpHeaders
 import org.apache.hc.core5.http.message.BasicHeader
+import org.opensearch.client.Request
 import org.opensearch.client.ResponseException
 import org.opensearch.core.rest.RestStatus
 import org.opensearch.indexmanagement.IndexManagementPlugin
+import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
 import org.opensearch.indexmanagement.makeRequest
 import org.opensearch.indexmanagement.opensearchapi.convertToMap
 import org.opensearch.indexmanagement.snapshotmanagement.SnapshotManagementRestTestCase
@@ -48,7 +50,8 @@ class RestGetSnapshotManagementIT : SnapshotManagementRestTestCase() {
  @Throws(Exception::class)
  fun `test getting a snapshot management policy that doesn't exist and config index doesnt exist`() {
  try {
- deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
+ val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
+ adminClient().performRequest(deleteISMIndexRequest)
  getSMPolicy(randomAlphaOfLength(20))
  fail("expected response exception")
  } catch (e: ResponseException) {

src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestIndexSnapshotManagementIT.kt

@@ -5,6 +5,7 @@
 
 package org.opensearch.indexmanagement.snapshotmanagement.resthandler
 
+import org.opensearch.client.Request
 import org.opensearch.client.ResponseException
 import org.opensearch.common.xcontent.XContentType
 import org.opensearch.core.rest.RestStatus
@@ -130,7 +131,8 @@ class RestIndexSnapshotManagementIT : SnapshotManagementRestTestCase() {
  @Throws(Exception::class)
  @Suppress("UNCHECKED_CAST")
  fun `test mappings after sm policy creation`() {
- deleteIndex(INDEX_MANAGEMENT_INDEX)
+ val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
+ adminClient().performRequest(deleteISMIndexRequest)
  createSMPolicy(randomSMPolicy())
 
  val response = client().makeRequest("GET", "/$INDEX_MANAGEMENT_INDEX/_mapping")

src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestStartSnapshotManagementIT.kt

@@ -5,9 +5,11 @@
 
 package org.opensearch.indexmanagement.snapshotmanagement.resthandler
 
+import org.opensearch.client.Request
 import org.opensearch.client.ResponseException
 import org.opensearch.core.rest.RestStatus
 import org.opensearch.indexmanagement.IndexManagementPlugin
+import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
 import org.opensearch.indexmanagement.makeRequest
 import org.opensearch.indexmanagement.snapshotmanagement.SnapshotManagementRestTestCase
 import org.opensearch.indexmanagement.snapshotmanagement.randomSMPolicy
@@ -59,7 +61,8 @@ class RestStartSnapshotManagementIT : SnapshotManagementRestTestCase() {
 
  fun `test starting a snapshot management policy with no config index fails`() {
  try {
- deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
+ val request = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
+ adminClient().performRequest(request)
  client().makeRequest("POST", "${IndexManagementPlugin.SM_POLICIES_URI}/nonexistent_foo/_start")
  fail("expected response exception")
  } catch (e: ResponseException) {

src/test/kotlin/org/opensearch/indexmanagement/snapshotmanagement/resthandler/RestStopSnapshotManagementIT.kt

@@ -5,9 +5,11 @@
 
 package org.opensearch.indexmanagement.snapshotmanagement.resthandler
 
+import org.opensearch.client.Request
 import org.opensearch.client.ResponseException
 import org.opensearch.core.rest.RestStatus
 import org.opensearch.indexmanagement.IndexManagementPlugin
+import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
 import org.opensearch.indexmanagement.makeRequest
 import org.opensearch.indexmanagement.snapshotmanagement.SnapshotManagementRestTestCase
 import org.opensearch.indexmanagement.snapshotmanagement.randomSMPolicy
@@ -59,7 +61,8 @@ class RestStopSnapshotManagementIT : SnapshotManagementRestTestCase() {
 
  fun `test stopping a snapshot management policy with no config index fails`() {
  try {
- deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
+ val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
+ adminClient().performRequest(deleteISMIndexRequest)
  client().makeRequest("POST", "${IndexManagementPlugin.SM_POLICIES_URI}/nonexistent_foo/_stop")
  fail("expected response exception")
  } catch (e: ResponseException) {

src/test/kotlin/org/opensearch/indexmanagement/transform/TransformRestTestCase.kt

@@ -119,7 +119,7 @@ abstract class TransformRestTestCase : IndexManagementRestTestCase() {
 
  protected fun getTransformMetadata(metadataId: String): TransformMetadata {
  val response =
- client().makeRequest(
+ adminClient().makeRequest(
  "GET", "$INDEX_MANAGEMENT_INDEX/_doc/$metadataId", null, BasicHeader(HttpHeaders.CONTENT_TYPE, "application/json"),
  )
  assertEquals("Unable to get transform metadata $metadataId", RestStatus.OK, response.restStatus())

src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestDeleteTransformActionIT.kt

@@ -5,6 +5,7 @@
 
 package org.opensearch.indexmanagement.transform.resthandler
 
+import org.opensearch.client.Request
 import org.opensearch.client.ResponseException
 import org.opensearch.core.rest.RestStatus
 import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
@@ -76,7 +77,10 @@ class RestDeleteTransformActionIT : TransformRestTestCase() {
  @Throws(Exception::class)
  fun `test deleting a transform that doesn't exist and config index doesn't exist`() {
  try {
- if (indexExists(INDEX_MANAGEMENT_INDEX)) deleteIndex(INDEX_MANAGEMENT_INDEX)
+ if (indexExists(INDEX_MANAGEMENT_INDEX)) {
+ val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
+ adminClient().performRequest(deleteISMIndexRequest)
+ }
  val res = client().makeRequest("DELETE", "$TRANSFORM_BASE_URI/foobarbaz")
  fail("expected 404 ResponseException: ${res.asMap()}")
  } catch (e: ResponseException) {

src/test/kotlin/org/opensearch/indexmanagement/transform/resthandler/RestExplainTransformActionIT.kt

@@ -6,9 +6,10 @@
 package org.opensearch.indexmanagement.transform.resthandler
 
 import org.junit.Assert
+import org.opensearch.client.Request
 import org.opensearch.client.ResponseException
 import org.opensearch.core.rest.RestStatus
-import org.opensearch.indexmanagement.IndexManagementPlugin
+import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.INDEX_MANAGEMENT_INDEX
 import org.opensearch.indexmanagement.IndexManagementPlugin.Companion.TRANSFORM_BASE_URI
 import org.opensearch.indexmanagement.makeRequest
 import org.opensearch.indexmanagement.transform.TransformRestTestCase
@@ -160,7 +161,8 @@ class RestExplainTransformActionIT : TransformRestTestCase() {
 
  @Throws(Exception::class)
  fun `test explain transform when config doesnt exist`() {
- deleteIndex(IndexManagementPlugin.INDEX_MANAGEMENT_INDEX)
+ val deleteISMIndexRequest = Request("DELETE", "/$INDEX_MANAGEMENT_INDEX")
+ adminClient().performRequest(deleteISMIndexRequest)
  val responseExplicit = client().makeRequest("GET", "$TRANSFORM_BASE_URI/no_config_some_transform/_explain")
  val expectedResponse = mapOf("no_config_some_transform" to "Failed to search transform metadata")
  assertEquals("Non-existent transform didn't return null", expectedResponse, responseExplicit.asMap())