Merge remote-tracking branch 'upstream/master' into async-metadata

.ci/bwcVersions

@@ -25,6 +25,7 @@ BWC_VERSION:
   - "7.9.1"
   - "7.9.2"
   - "7.9.3"
+  - "7.9.4"
   - "7.10.0"
   - "7.11.0"
   - "8.0.0"

TESTING.asciidoc

@@ -286,7 +286,7 @@ the elasticsearch official clients. The YAML based tests describe the
 operations to be executed and the obtained results that need to be tested.
 
 The YAML tests support various operators defined in the link:/rest-api-spec/src/main/resources/rest-api-spec/test/README.asciidoc[rest-api-spec] and adhere to the link:/rest-api-spec/README.markdown[Elasticsearch REST API JSON specification]
-In order to run the the YAML tests, the relevant API specification needs
+In order to run the YAML tests, the relevant API specification needs
 to be on the test classpath. Any gradle project that has support for REST
 tests will get the primary API on it's class path. However, to better support
 Gradle incremental builds, it is recommended to explicitly declare which

build.gradle

@@ -31,6 +31,7 @@ import org.gradle.plugins.ide.eclipse.model.SourceFolder
 import org.gradle.util.DistributionLocator
 import org.gradle.util.GradleVersion
 import static org.elasticsearch.gradle.util.GradleUtils.maybeConfigure
+import org.gradle.plugins.ide.eclipse.model.ProjectDependency
 
 plugins {
   id 'lifecycle-base'
@@ -330,6 +331,17 @@ allprojects {
         i++;
         folder.output = "build-eclipse/" + i
       }
+
+      // Starting with Gradle 6.7 test dependencies are not exposed by eclipse
+      // projects by default. This breaks project dependencies using the `java-test-fixtures` plugin
+      // or dependencies on other projects manually declared testArtifacts configurations
+      // This issue is tracked in gradle issue tracker.
+      // See https://github.com/gradle/gradle/issues/14932 for further details
+      classpath.entries.findAll {
+        it instanceof ProjectDependency
+      }.each {
+        it.entryAttributes.remove('without_test_code')
+      }
     }
   }
   /*

buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy

@@ -51,7 +51,7 @@ class PluginBuildPlugin implements Plugin<Project> {
     void apply(Project project) {
         project.pluginManager.apply(BuildPlugin)
         project.pluginManager.apply(RestTestBasePlugin)
-        project.pluginManager.apply(CompileOnlyResolvePlugin.class);
+        project.pluginManager.apply(CompileOnlyResolvePlugin.class)
 
         PluginPropertiesExtension extension = project.extensions.create(PLUGIN_EXTENSION_NAME, PluginPropertiesExtension, project)
         configureDependencies(project)
@@ -82,7 +82,7 @@ class PluginBuildPlugin implements Plugin<Project> {
             if (extension1.description == null) {
                 throw new InvalidUserDataException('description is a required setting for esplugin')
             }
-            if (extension1.classname == null) {
+            if (extension1.type != PluginType.BOOTSTRAP && extension1.classname == null) {
                 throw new InvalidUserDataException('classname is a required setting for esplugin')
             }
 
@@ -92,10 +92,12 @@ class PluginBuildPlugin implements Plugin<Project> {
                     'version'             : extension1.version,
                     'elasticsearchVersion': Version.fromString(VersionProperties.elasticsearch).toString(),
                     'javaVersion'         : project.targetCompatibility as String,
-                    'classname'           : extension1.classname,
+                    'classname'           : extension1.type == PluginType.BOOTSTRAP ? "" : extension1.classname,
                     'extendedPlugins'     : extension1.extendedPlugins.join(','),
                     'hasNativeController' : extension1.hasNativeController,
-                    'requiresKeystore'    : extension1.requiresKeystore
+                    'requiresKeystore'    : extension1.requiresKeystore,
+                    'type'                : extension1.type.toString(),
+                    'javaOpts'            : extension1.javaOpts,
             ]
             project.tasks.named('pluginProperties').configure {
                 expand(properties)

buildSrc/src/main/java/org/elasticsearch/gradle/internal/InternalDistributionBwcSetupPlugin.java

@@ -151,11 +151,13 @@ private void registerDistributionArchiveArtifact(Project bwcProject, Distributio
 
     private static List<DistributionProject> resolveArchiveProjects(File checkoutDir, Version bwcVersion) {
         List<String> projects = new ArrayList<>();
-        projects.addAll(asList("deb", "rpm"));
-        if (bwcVersion.onOrAfter("7.0.0")) {
+        // All active BWC branches publish default and oss variants of rpm and deb packages
+        projects.addAll(asList("deb", "rpm", "oss-deb", "oss-rpm"));
+
+        if (bwcVersion.onOrAfter("7.0.0")) { // starting with 7.0 we bundle a jdk which means we have platform-specific archives
             projects.addAll(asList("oss-windows-zip", "windows-zip", "oss-darwin-tar", "darwin-tar", "oss-linux-tar", "linux-tar"));
-        } else {
-            projects.addAll(asList("oss-zip", "zip", "oss-deb", "oss-rpm"));
+        } else { // prior to 7.0 we published only a single zip and tar archives for oss and default distributions
+            projects.addAll(asList("oss-zip", "zip", "tar", "oss-tar"));
         }
 
         return projects.stream().map(name -> {

buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginPropertiesExtension.java

@@ -43,6 +43,10 @@ public class PluginPropertiesExtension {
 
     private boolean hasNativeController;
 
+    private PluginType type = PluginType.ISOLATED;
+
+    private String javaOpts = "";
+
     /** True if the plugin requires the elasticsearch keystore to exist, false otherwise. */
     private boolean requiresKeystore;
 
@@ -105,6 +109,22 @@ public void setHasNativeController(boolean hasNativeController) {
         this.hasNativeController = hasNativeController;
     }
 
+    public PluginType getType() {
+        return type;
+    }
+
+    public void setType(PluginType type) {
+        this.type = type;
+    }
+
+    public String getJavaOpts() {
+        return javaOpts;
+    }
+
+    public void setJavaOpts(String javaOpts) {
+        this.javaOpts = javaOpts;
+    }
+
     public boolean isRequiresKeystore() {
         return requiresKeystore;
     }

buildSrc/src/main/java/org/elasticsearch/gradle/plugin/PluginType.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.gradle.plugin;
+
+import java.util.Locale;
+
+/**
+ * This mirrors org.elasticsearch.plugins.PluginType, which is not
+ * available to the Gradle plugin that actually builds plugins. See that
+ * class for further information.
+ */
+public enum PluginType {
+    ISOLATED,
+    BOOTSTRAP;
+
+    @Override
+    public String toString() {
+        return this.name().toLowerCase(Locale.ROOT);
+    }
+}

buildSrc/src/main/java/org/elasticsearch/gradle/test/DistroTestPlugin.java

@@ -78,6 +78,9 @@ public class DistroTestPlugin implements Plugin<Project> {
     private static final String BWC_DISTRIBUTION_SYSPROP = "tests.bwc-distribution";
     private static final String EXAMPLE_PLUGIN_SYSPROP = "tests.example-plugin";
 
+    private static final String QUOTA_AWARE_FS_PLUGIN_CONFIGURATION = "quotaAwareFsPlugin";
+    private static final String QUOTA_AWARE_FS_PLUGIN_SYSPROP = "tests.quota-aware-fs-plugin";
+
     @Override
     public void apply(Project project) {
         project.getRootProject().getPluginManager().apply(DockerSupportPlugin.class);
@@ -100,20 +103,23 @@ public void apply(Project project) {
         TaskProvider<Task> destructiveDistroTest = project.getTasks().register("destructiveDistroTest");
 
         Configuration examplePlugin = configureExamplePlugin(project);
+        Configuration quotaAwareFsPlugin = configureQuotaAwareFsPlugin(project);
 
         List<TaskProvider<Test>> windowsTestTasks = new ArrayList<>();
         Map<Type, List<TaskProvider<Test>>> linuxTestTasks = new HashMap<>();
         Map<String, List<TaskProvider<Test>>> upgradeTestTasks = new HashMap<>();
         Map<String, TaskProvider<?>> depsTasks = new HashMap<>();
+
         for (ElasticsearchDistribution distribution : testDistributions) {
             String taskname = destructiveDistroTestTaskName(distribution);
             TaskProvider<?> depsTask = project.getTasks().register(taskname + "#deps");
-            depsTask.configure(t -> t.dependsOn(distribution, examplePlugin));
+            depsTask.configure(t -> t.dependsOn(distribution, examplePlugin, quotaAwareFsPlugin));
             depsTasks.put(taskname, depsTask);
             TaskProvider<Test> destructiveTask = configureTestTask(project, taskname, distribution, t -> {
                 t.onlyIf(t2 -> distribution.isDocker() == false || dockerSupport.get().getDockerAvailability().isAvailable);
                 addDistributionSysprop(t, DISTRIBUTION_SYSPROP, distribution::getFilepath);
                 addDistributionSysprop(t, EXAMPLE_PLUGIN_SYSPROP, () -> examplePlugin.getSingleFile().toString());
+                addDistributionSysprop(t, QUOTA_AWARE_FS_PLUGIN_SYSPROP, () -> quotaAwareFsPlugin.getSingleFile().toString());
                 t.exclude("**/PackageUpgradeTests.class");
             }, depsTask);
 
@@ -316,6 +322,14 @@ private static Configuration configureExamplePlugin(Project project) {
         return examplePlugin;
     }
 
+    private static Configuration configureQuotaAwareFsPlugin(Project project) {
+        Configuration examplePlugin = project.getConfigurations().create(QUOTA_AWARE_FS_PLUGIN_CONFIGURATION);
+        DependencyHandler deps = project.getDependencies();
+        Map<String, String> quotaAwareFsPluginProject = Map.of("path", ":plugins:quota-aware-fs", "configuration", "zip");
+        deps.add(QUOTA_AWARE_FS_PLUGIN_CONFIGURATION, deps.project(quotaAwareFsPluginProject));
+        return examplePlugin;
+    }
+
     private static void configureVMWrapperTasks(
         Project project,
         List<TaskProvider<Test>> destructiveTasks,

buildSrc/src/main/resources/minimumGradleVersion

@@ -1 +1 @@
-6.7
+6.6.1

buildSrc/src/main/resources/plugin-descriptor.properties

@@ -16,6 +16,11 @@
 #
 ### mandatory elements for all plugins:
 #
+# 'type': the type of this plugin. 'isolated' indicated a typical sandboxed plugin,
+# whereas 'bootstrap' indicates a plugin whose jars are added to the JVM's boot
+# classpath.
+type=${type}
+#
 # 'description': simple summary of the plugin
 description=${description}
 #
@@ -24,9 +29,12 @@ version=${version}
 #
 # 'name': the plugin name
 name=${name}
+<% if (type != "bootstrap") { %>
 #
-# 'classname': the name of the class to load, fully-qualified.
+# 'classname': the name of the class to load, fully-qualified. Only applies to
+# "isolated" plugins
 classname=${classname}
+<% } %>
 #
 # 'java.version': version of java the code is built against
 # use the system property java.specification.version
@@ -43,3 +51,9 @@ extended.plugins=${extendedPlugins}
 #
 # 'has.native.controller': whether or not the plugin has a native controller
 has.native.controller=${hasNativeController}
+<% if (type == "bootstrap") { %>
+#
+# 'java.opts': any additional command line parameters to pass to the JVM when
+# Elasticsearch starts. Only applies to "bootstrap" plugins.
+java.opts=${javaOpts}
+<% } %>

buildSrc/version.properties

@@ -1,8 +1,8 @@
 elasticsearch     = 8.0.0
 lucene            = 8.7.0-snapshot-72d8528c3a6
 
-bundled_jdk_vendor = openjdk
-bundled_jdk = 15+36@779bf45e88a44cbd9ea6621d33e33db1
+bundled_jdk_vendor = adoptopenjdk
+bundled_jdk = 15.0.1+9
 
 checkstyle = 8.20
 

client/rest-high-level/src/main/java/org/elasticsearch/client/indices/DataStream.java

@@ -36,21 +36,27 @@ public final class DataStream {
     private final String timeStampField;
     private final List<String> indices;
     private final long generation;
+    private final boolean hidden;
     ClusterHealthStatus dataStreamStatus;
     @Nullable
     String indexTemplate;
     @Nullable
     String ilmPolicyName;
+    @Nullable
+    private final Map<String, Object> metadata;
 
     public DataStream(String name, String timeStampField, List<String> indices, long generation, ClusterHealthStatus dataStreamStatus,
-                      @Nullable String indexTemplate, @Nullable String ilmPolicyName) {
+                      @Nullable String indexTemplate, @Nullable String ilmPolicyName, @Nullable  Map<String, Object> metadata,
+                      boolean hidden) {
         this.name = name;
         this.timeStampField = timeStampField;
         this.indices = indices;
         this.generation = generation;
         this.dataStreamStatus = dataStreamStatus;
         this.indexTemplate = indexTemplate;
         this.ilmPolicyName = ilmPolicyName;
+        this.metadata = metadata;
+        this.hidden = hidden;
     }
 
     public String getName() {
@@ -81,13 +87,23 @@ public String getIlmPolicyName() {
         return ilmPolicyName;
     }
 
+    public Map<String, Object> getMetadata() {
+        return metadata;
+    }
+
+    public boolean isHidden() {
+        return hidden;
+    }
+
     public static final ParseField NAME_FIELD = new ParseField("name");
     public static final ParseField TIMESTAMP_FIELD_FIELD = new ParseField("timestamp_field");
     public static final ParseField INDICES_FIELD = new ParseField("indices");
     public static final ParseField GENERATION_FIELD = new ParseField("generation");
     public static final ParseField STATUS_FIELD = new ParseField("status");
     public static final ParseField INDEX_TEMPLATE_FIELD = new ParseField("template");
     public static final ParseField ILM_POLICY_FIELD = new ParseField("ilm_policy");
+    public static final ParseField METADATA_FIELD = new ParseField("_meta");
+    public static final ParseField HIDDEN_FIELD = new ParseField("hidden");
 
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<DataStream, Void> PARSER = new ConstructingObjectParser<>("data_stream",
@@ -101,7 +117,10 @@ public String getIlmPolicyName() {
             ClusterHealthStatus status = ClusterHealthStatus.fromString(statusStr);
             String indexTemplate = (String) args[5];
             String ilmPolicy = (String) args[6];
-            return new DataStream(dataStreamName, timeStampField, indices, generation, status, indexTemplate, ilmPolicy);
+            Map<String, Object> metadata = (Map<String, Object>) args[7];
+            Boolean hidden = (Boolean) args[8];
+            hidden = hidden != null && hidden;
+            return new DataStream(dataStreamName, timeStampField, indices, generation, status, indexTemplate, ilmPolicy, metadata, hidden);
         });
 
     static {
@@ -112,6 +131,8 @@ public String getIlmPolicyName() {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), STATUS_FIELD);
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), INDEX_TEMPLATE_FIELD);
         PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), ILM_POLICY_FIELD);
+        PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), METADATA_FIELD);
+        PARSER.declareBoolean(ConstructingObjectParser.optionalConstructorArg(), HIDDEN_FIELD);
     }
 
     public static DataStream fromXContent(XContentParser parser) throws IOException {
@@ -129,11 +150,12 @@ public boolean equals(Object o) {
             indices.equals(that.indices) &&
             dataStreamStatus == that.dataStreamStatus &&
             Objects.equals(indexTemplate, that.indexTemplate) &&
-            Objects.equals(ilmPolicyName, that.ilmPolicyName);
+            Objects.equals(ilmPolicyName, that.ilmPolicyName) &&
+            Objects.equals(metadata, that.metadata);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(name, timeStampField, indices, generation, dataStreamStatus, indexTemplate, ilmPolicyName);
+        return Objects.hash(name, timeStampField, indices, generation, dataStreamStatus, indexTemplate, ilmPolicyName, metadata);
     }
 }

client/rest-high-level/src/test/java/org/elasticsearch/client/IndicesClientIT.java

@@ -1579,7 +1579,7 @@ public void testDataStreams() throws Exception {
         CompressedXContent mappings = new CompressedXContent("{\"properties\":{\"@timestamp\":{\"type\":\"date\"}}}");
         Template template = new Template(null, mappings, null);
         ComposableIndexTemplate indexTemplate = new ComposableIndexTemplate(Collections.singletonList(dataStreamName), template,
-            Collections.emptyList(), 1L, 1L, new HashMap<>(), new ComposableIndexTemplate.DataStreamTemplate());
+            Collections.emptyList(), 1L, 1L, new HashMap<>(), new ComposableIndexTemplate.DataStreamTemplate(), null);
         PutComposableIndexTemplateRequest putComposableIndexTemplateRequest =
             new PutComposableIndexTemplateRequest().name("ds-template").create(true).indexTemplate(indexTemplate);
         AcknowledgedResponse response = execute(putComposableIndexTemplateRequest,
@@ -1658,7 +1658,7 @@ public void testIndexTemplates() throws Exception {
         Template template = new Template(settings, mappings, Map.of("alias", alias));
         List<String> pattern = List.of("pattern");
         ComposableIndexTemplate indexTemplate =
-            new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null);
+            new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null, null);
         PutComposableIndexTemplateRequest putComposableIndexTemplateRequest =
             new PutComposableIndexTemplateRequest().name(templateName).create(true).indexTemplate(indexTemplate);
 
@@ -1705,7 +1705,7 @@ public void testSimulateIndexTemplate() throws Exception {
         Template template = new Template(settings, mappings, Map.of("alias", alias));
         List<String> pattern = List.of("pattern");
         ComposableIndexTemplate indexTemplate =
-            new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null);
+            new ComposableIndexTemplate(pattern, template, Collections.emptyList(), 1L, 1L, new HashMap<>(), null, null);
         PutComposableIndexTemplateRequest putComposableIndexTemplateRequest =
             new PutComposableIndexTemplateRequest().name(templateName).create(true).indexTemplate(indexTemplate);
 
@@ -1716,7 +1716,7 @@ public void testSimulateIndexTemplate() throws Exception {
         SimulateIndexTemplateRequest simulateIndexTemplateRequest = new SimulateIndexTemplateRequest("pattern");
         AliasMetadata simulationAlias = AliasMetadata.builder("simulation-alias").writeIndex(true).build();
         ComposableIndexTemplate simulationTemplate = new ComposableIndexTemplate(pattern, new Template(null, null,
-            Map.of("simulation-alias", simulationAlias)), Collections.emptyList(), 2L, 1L, new HashMap<>(), null);
+            Map.of("simulation-alias", simulationAlias)), Collections.emptyList(), 2L, 1L, new HashMap<>(), null, null);
         PutComposableIndexTemplateRequest newIndexTemplateReq =
             new PutComposableIndexTemplateRequest().name("used-for-simulation").create(true).indexTemplate(indexTemplate);
         newIndexTemplateReq.indexTemplate(simulationTemplate);