Adds support for ES 7.7.0 (opensearch-project#228)

* upgrade gradle to 6.4

* Routes replace usage of RestController.register
elastic/elasticsearch#51950

* add setting index.hidden=true to createIndex

* solve IT allowed action list problem

* try hard code jvmArgs

* add release note

* revert immutableList change, use listOf
jvmArgs put into afterEvaluate phase

.github/workflows/release-workflow.yml

@@ -9,7 +9,7 @@ jobs:
   build:
     strategy:
       matrix:
-        java: [13]
+        java: [14]
     # Job name
     name: Build Index Management with JDK ${{ matrix.java }}
     # This job runs on Linux

.github/workflows/test-and-build-workflow.yml

@@ -17,10 +17,10 @@ jobs:
       - name: Checkout Branch
         uses: actions/checkout@v2
       # This step uses the setup-java Github action: https://github.com/actions/setup-java
-      - name: Set Up JDK 13
+      - name: Set Up JDK 14
         uses: actions/setup-java@v1
         with:
-          java-version: 13
+          java-version: 14
       - name: Build with Gradle
         run: ./gradlew build
       - name: Create Artifact Path

README.md

@@ -41,7 +41,7 @@ Please see our [documentation](https://opendistro.github.io/for-elasticsearch-do
 
 1. Check out this package from version control.
 2. Launch Intellij IDEA, choose **Import Project**, and select the `settings.gradle` file in the root of this package. 
-3. To build from the command line, set `JAVA_HOME` to point to a JDK >= 13 before running `./gradlew`.
+3. To build from the command line, set `JAVA_HOME` to point to a JDK >= 14 before running `./gradlew`.
   - Unix System
     1. `export JAVA_HOME=jdk-install-dir`: Replace `jdk-install-dir` with the JAVA_HOME directory of your system.
     2. `export PATH=$JAVA_HOME/bin:$PATH`

build-tools/esplugin-coverage.gradle

@@ -27,11 +27,10 @@
  *  break if there are multiple nodes in the integTestCluster. But for now... it sorta works.
  */
 
-apply plugin: 'jacoco'
-
 // Get gradle to generate the required jvm agent arg for us using a dummy tasks of type Test. Unfortunately Elastic's
 // testing tasks don't derive from Test so the jacoco plugin can't do this automatically.
 def jacocoDir = "${buildDir}/jacoco"
+
 task dummyTest(type: Test) {
     enabled = false
     workingDir = file("/") // Force absolute path to jacoco agent jar
@@ -42,7 +41,6 @@ task dummyTest(type: Test) {
     }
 }
 
-
 task dummyIntegTest(type: Test) {
     enabled = false
     workingDir = file("/") // Force absolute path to jacoco agent jar
@@ -57,15 +55,6 @@ integTest.runner {
     systemProperty 'jacoco.dir', "${jacocoDir}"
 }
 
-testClusters.integTest {
-    jvmArgs " ${dummyIntegTest.jacoco.getAsJvmArg()}"
-    systemProperty 'com.sun.management.jmxremote', "true"
-    systemProperty 'com.sun.management.jmxremote.authenticate', "false"
-    systemProperty 'com.sun.management.jmxremote.port', "7777"
-    systemProperty 'com.sun.management.jmxremote.ssl', "false"
-    systemProperty 'java.rmi.server.hostname', "127.0.0.1"
-}
-
 jacocoTestReport {
     dependsOn integTest, test
     executionData dummyTest.jacoco.destinationFile, dummyIntegTest.jacoco.destinationFile
@@ -77,7 +66,17 @@ jacocoTestReport {
     }
 }
 
-project.gradle.projectsEvaluated {
-    jacocoTestReport.dependsOn integTest.runner
-}
+allprojects{
+    afterEvaluate {
+        jacocoTestReport.dependsOn integTest.runner
 
+        testClusters.integTest {
+            jvmArgs " ${dummyIntegTest.jacoco.getAsJvmArg()}"
+            systemProperty 'com.sun.management.jmxremote', "true"
+            systemProperty 'com.sun.management.jmxremote.authenticate', "false"
+            systemProperty 'com.sun.management.jmxremote.port', "7777"
+            systemProperty 'com.sun.management.jmxremote.ssl', "false"
+            systemProperty 'java.rmi.server.hostname', "127.0.0.1"
+        }
+    }
+}

build.gradle

@@ -15,8 +15,8 @@
 
 buildscript {
     ext {
-        es_version = System.getProperty("es.version", "7.6.1")
-        kotlin_version = System.getProperty("kotlin.version", "1.3.61")
+        es_version = System.getProperty("es.version", "7.7.0")
+        kotlin_version = System.getProperty("kotlin.version", "1.3.72")
     }
 
     repositories {
@@ -36,7 +36,7 @@ buildscript {
 }
 
 plugins {
-    id 'nebula.ospackage' version "8.2.0"
+    id 'nebula.ospackage' version "8.3.0"
     id "com.dorongold.task-tree" version "1.5"
 }
 
@@ -79,12 +79,12 @@ configurations.testCompile {
 
 dependencies {
     compileOnly "org.elasticsearch:elasticsearch:${es_version}"
-    compileOnly "com.amazon.opendistroforelasticsearch:opendistro-job-scheduler-spi:1.6.0.0"
+    compileOnly "com.amazon.opendistroforelasticsearch:opendistro-job-scheduler-spi:1.8.0.0"
     compile "org.jetbrains.kotlin:kotlin-stdlib:${kotlin_version}"
     compile "org.jetbrains.kotlin:kotlin-stdlib-common:${kotlin_version}"
     compile 'org.jetbrains.kotlinx:kotlinx-coroutines-core:1.2.1'
     compile "org.jetbrains:annotations:13.0"
-    compile "com.amazon.opendistroforelasticsearch:notification:1.6.0.0"
+    compile "com.amazon.opendistroforelasticsearch:notification:1.8.0.0"
 
     testCompile "org.elasticsearch.test:framework:${es_version}"
     testCompile "org.jetbrains.kotlin:kotlin-test:${kotlin_version}"
@@ -129,6 +129,15 @@ thirdPartyAudit.enabled = false
 def es_tmp_dir = rootProject.file('build/private/es_tmp').absoluteFile
 es_tmp_dir.mkdirs()
 
+afterEvaluate {
+    testClusters.integTest.nodes.each { node ->
+        def plugins = node.plugins
+        def firstPlugin = plugins.get(0)
+        plugins.remove(0)
+        plugins.add(firstPlugin)
+    }
+}
+
 test {
     systemProperty 'tests.security.manager', 'false'
 }

gradle.properties

@@ -13,4 +13,4 @@
 #   permissions and limitations under the License.
 #
 
-version = 1.7.0
+version = 1.8.0

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
-#Tue Mar 24 11:18:08 PDT 2020
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.2.2-all.zip
+#Tue May 19 22:58:14 PDT 2020
+distributionUrl=https\://services.gradle.org/distributions/gradle-6.4-all.zip
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStorePath=wrapper/dists

gradlew

@@ -82,6 +82,7 @@ esac
 
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
+
 # Determine the Java command to use to start the JVM.
 if [ -n "$JAVA_HOME" ] ; then
     if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
@@ -129,6 +130,7 @@ fi
 if [ "$cygwin" = "true" -o "$msys" = "true" ] ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+
     JAVACMD=`cygpath --unix "$JAVACMD"`
 
     # We build the pattern for arguments to be converted via cygpath

gradlew.bat

@@ -84,6 +84,7 @@ set CMD_LINE_ARGS=%*
 
 set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
 
+
 @rem Execute Gradle
 "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
 

release-notes/opendistro-elasticsearch-index-management-1.8.0.md

@@ -0,0 +1,8 @@
+
+## Version 1.8.0.0 (2020-5-20)
+
+Compatible with Elasticsearch 7.7.0, Adds support for ODFE 1.8.0
+
+### New Features
+* Snapshot implementation [PR #135](https://github.com/opendistro-for-elasticsearch/index-management/pull/135)
+

src/main/kotlin/com/amazon/opendistroforelasticsearch/indexstatemanagement/IndexStateManagementIndices.kt

@@ -32,6 +32,7 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse
 import org.elasticsearch.client.Client
 import org.elasticsearch.client.IndicesAdminClient
 import org.elasticsearch.cluster.service.ClusterService
+import org.elasticsearch.common.settings.Settings
 import org.elasticsearch.common.xcontent.XContentType
 
 @OpenForTesting
@@ -46,6 +47,7 @@ class IndexStateManagementIndices(
         if (!indexStateManagementIndexExists()) {
             val indexRequest = CreateIndexRequest(INDEX_STATE_MANAGEMENT_INDEX)
                     .mapping(_DOC, indexStateManagementMappings, XContentType.JSON)
+                    .settings(Settings.builder().put("index.hidden", true).build())
             client.create(indexRequest, object : ActionListener<CreateIndexResponse> {
                 override fun onFailure(e: Exception) {
                     actionListener.onFailure(e)
@@ -103,7 +105,9 @@ class IndexStateManagementIndices(
         }
         if (existsResponse.isExists) return true
 
-        val request = CreateIndexRequest(index).mapping(_DOC, indexStateManagementHistoryMappings, XContentType.JSON)
+        val request = CreateIndexRequest(index)
+                .mapping(_DOC, indexStateManagementHistoryMappings, XContentType.JSON)
+                .settings(Settings.builder().put("index.hidden", true).build())
         if (alias != null) request.alias(Alias(alias))
         return try {
             val createIndexResponse: CreateIndexResponse = client.suspendUntil { client.create(request, it) }

src/main/kotlin/com/amazon/opendistroforelasticsearch/indexstatemanagement/IndexStateManagementPlugin.kt

@@ -118,14 +118,14 @@ internal class IndexStateManagementPlugin : JobSchedulerExtension, ActionPlugin,
         nodesInCluster: Supplier<DiscoveryNodes>
     ): List<RestHandler> {
         return listOf(
-            RestIndexPolicyAction(settings, restController, clusterService, indexStateManagementIndices),
-            RestGetPolicyAction(restController),
-            RestDeletePolicyAction(restController),
-            RestExplainAction(restController),
-            RestRetryFailedManagedIndexAction(restController),
-            RestAddPolicyAction(restController),
-            RestRemovePolicyAction(restController),
-            RestChangePolicyAction(restController, clusterService)
+            RestIndexPolicyAction(settings, clusterService, indexStateManagementIndices),
+            RestGetPolicyAction(),
+            RestDeletePolicyAction(),
+            RestExplainAction(),
+            RestRetryFailedManagedIndexAction(),
+            RestAddPolicyAction(),
+            RestRemovePolicyAction(),
+            RestChangePolicyAction(clusterService)
         )
     }
 
@@ -138,7 +138,8 @@ internal class IndexStateManagementPlugin : JobSchedulerExtension, ActionPlugin,
         xContentRegistry: NamedXContentRegistry,
         environment: Environment,
         nodeEnvironment: NodeEnvironment,
-        namedWriteableRegistry: NamedWriteableRegistry
+        namedWriteableRegistry: NamedWriteableRegistry,
+        indexNameExpressionResolver: IndexNameExpressionResolver
     ): Collection<Any> {
         val settings = environment.settings()
         this.clusterService = clusterService

src/main/kotlin/com/amazon/opendistroforelasticsearch/indexstatemanagement/resthandler/RestAddPolicyAction.kt

@@ -38,9 +38,9 @@ import org.elasticsearch.common.unit.TimeValue
 import org.elasticsearch.common.xcontent.XContentHelper
 import org.elasticsearch.index.Index
 import org.elasticsearch.rest.BaseRestHandler
+import org.elasticsearch.rest.RestHandler.Route
 import org.elasticsearch.rest.BytesRestResponse
 import org.elasticsearch.rest.RestChannel
-import org.elasticsearch.rest.RestController
 import org.elasticsearch.rest.RestRequest
 import org.elasticsearch.rest.RestRequest.Method.POST
 import org.elasticsearch.rest.RestResponse
@@ -51,15 +51,17 @@ import java.io.IOException
 import java.time.Duration
 import java.time.Instant
 
-class RestAddPolicyAction(controller: RestController) : BaseRestHandler() {
-
-    init {
-        controller.registerHandler(POST, ADD_POLICY_BASE_URI, this)
-        controller.registerHandler(POST, "$ADD_POLICY_BASE_URI/{index}", this)
-    }
+class RestAddPolicyAction : BaseRestHandler() {
 
     override fun getName(): String = "add_policy_action"
 
+    override fun routes(): List<Route> {
+        return listOf(
+                Route(POST, ADD_POLICY_BASE_URI),
+                Route(POST, "$ADD_POLICY_BASE_URI/{index}")
+        )
+    }
+
     @Throws(IOException::class)
     @Suppress("SpreadOperator") // There is no way around dealing with java vararg without spread operator.
     override fun prepareRequest(request: RestRequest, client: NodeClient): RestChannelConsumer {

src/main/kotlin/com/amazon/opendistroforelasticsearch/indexstatemanagement/resthandler/RestChangePolicyAction.kt

@@ -55,9 +55,9 @@ import org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken
 import org.elasticsearch.common.xcontent.XContentType
 import org.elasticsearch.index.query.IdsQueryBuilder
 import org.elasticsearch.rest.BaseRestHandler
+import org.elasticsearch.rest.RestHandler.Route
 import org.elasticsearch.rest.BytesRestResponse
 import org.elasticsearch.rest.RestChannel
-import org.elasticsearch.rest.RestController
 import org.elasticsearch.rest.RestRequest
 import org.elasticsearch.rest.RestRequest.Method.POST
 import org.elasticsearch.rest.RestResponse
@@ -66,13 +66,15 @@ import org.elasticsearch.rest.action.RestResponseListener
 import org.elasticsearch.search.builder.SearchSourceBuilder
 import java.io.IOException
 
-class RestChangePolicyAction(controller: RestController, val clusterService: ClusterService) : BaseRestHandler() {
+class RestChangePolicyAction(val clusterService: ClusterService) : BaseRestHandler() {
 
     private val log = LogManager.getLogger(javaClass)
 
-    init {
-        controller.registerHandler(POST, CHANGE_POLICY_BASE_URI, this)
-        controller.registerHandler(POST, "$CHANGE_POLICY_BASE_URI/{index}", this)
+    override fun routes(): List<Route> {
+        return listOf(
+                Route(POST, CHANGE_POLICY_BASE_URI),
+                Route(POST, "$CHANGE_POLICY_BASE_URI/{index}")
+        )
     }
 
     override fun getName(): String = "change_policy_action"

src/main/kotlin/com/amazon/opendistroforelasticsearch/indexstatemanagement/resthandler/RestDeletePolicyAction.kt

@@ -22,16 +22,18 @@ import org.elasticsearch.action.delete.DeleteRequest
 import org.elasticsearch.action.support.WriteRequest.RefreshPolicy
 import org.elasticsearch.client.node.NodeClient
 import org.elasticsearch.rest.BaseRestHandler
-import org.elasticsearch.rest.RestController
+import org.elasticsearch.rest.RestHandler.Route
 import org.elasticsearch.rest.RestRequest
 import org.elasticsearch.rest.RestRequest.Method.DELETE
 import org.elasticsearch.rest.action.RestStatusToXContentListener
 import java.io.IOException
 
-class RestDeletePolicyAction(controller: RestController) : BaseRestHandler() {
+class RestDeletePolicyAction : BaseRestHandler() {
 
-    init {
-        controller.registerHandler(DELETE, "$POLICY_BASE_URI/{policyID}", this)
+    override fun routes(): List<Route> {
+        return listOf(
+                Route(DELETE, "$POLICY_BASE_URI/{policyID}")
+        )
     }
 
     override fun getName(): String = "delete_policy_action"

src/main/kotlin/com/amazon/opendistroforelasticsearch/indexstatemanagement/resthandler/RestExplainAction.kt

@@ -27,23 +27,26 @@ import org.elasticsearch.common.Strings
 import org.elasticsearch.common.xcontent.ToXContent
 import org.elasticsearch.common.xcontent.XContentBuilder
 import org.elasticsearch.rest.BaseRestHandler
+import org.elasticsearch.rest.RestHandler.Route
 import org.elasticsearch.rest.BytesRestResponse
 import org.elasticsearch.rest.RestChannel
-import org.elasticsearch.rest.RestController
 import org.elasticsearch.rest.RestRequest
+import org.elasticsearch.rest.RestRequest.Method.GET
 import org.elasticsearch.rest.RestResponse
 import org.elasticsearch.rest.RestStatus
 import org.elasticsearch.rest.action.RestBuilderListener
 
-class RestExplainAction(controller: RestController) : BaseRestHandler() {
+class RestExplainAction : BaseRestHandler() {
 
     companion object {
         const val EXPLAIN_BASE_URI = "${IndexStateManagementPlugin.ISM_BASE_URI}/explain"
     }
 
-    init {
-        controller.registerHandler(RestRequest.Method.GET, EXPLAIN_BASE_URI, this)
-        controller.registerHandler(RestRequest.Method.GET, "$EXPLAIN_BASE_URI/{index}", this)
+    override fun routes(): List<Route> {
+        return listOf(
+                Route(GET, EXPLAIN_BASE_URI),
+                Route(GET, "$EXPLAIN_BASE_URI/{index}")
+        )
     }
 
     override fun getName(): String {