SQL functions rework

.ci/os.ps1

@@ -27,10 +27,6 @@ New-Item -ItemType directory -Path \tmp
 
 $ErrorActionPreference="Continue"
 # TODO: remove the task exclusions once dependencies are set correctly and these don't run for Windows or buldiung the deb on windows is fixed
-& .\gradlew.bat -g "C:\Users\$env:username\.gradle" --parallel --scan --console=plain destructiveDistroTest `
-   -x :distribution:packages:buildOssDeb `
-   -x :distribution:packages:buildDeb `
-   -x :distribution:packages:buildOssRpm `
-   -x :distribution:packages:buildRpm `
+& .\gradlew.bat -g "C:\Users\$env:username\.gradle" --parallel --scan --console=plain destructiveDistroTest
 
 exit $LastExitCode

TESTING.asciidoc

@@ -43,6 +43,12 @@ supports a remote debugging option:
 ./gradlew run --debug-jvm
 ---------------------------------------------------------------------------
 
+This will instruct all JVMs (including any that run cli tools such as creating the keyring or adding users)
+to suspend and initiate a debug connection on port incrementing from `5005`.
+As such the IDE needs to be instructed to listen for connections on this port.
+Since we might run multiple JVMs as part of configuring and starting the cluster it's
+recommended to have the option to aut restart checked.
+
 ==== Distribution
 
 By default a node is started with the zip distribution.

buildSrc/reaper/src/main/java/org/elasticsearch/gradle/reaper/Reaper.java

@@ -37,7 +37,7 @@
  * Since how to reap a given service is platform and service dependent, this tool
  * operates on system commands to execute. It takes a single argument, a directory
  * that will contain files with reaping commands. Each line in each file will be
- * executed with {@link Runtime#getRuntime()#exec}.
+ * executed with {@link Runtime#exec(String)}.
  *
  * The main method will wait indefinitely on the parent process (Gradle) by
  * reading from stdin. When Gradle shuts down, whether normally or abruptly, the

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -676,6 +676,10 @@ class BuildPlugin implements Plugin<Project> {
              */
             (javadoc.options as CoreJavadocOptions).addBooleanOption('html5', true)
         }
+        // ensure javadoc task is run with 'check'
+        project.pluginManager.withPlugin('lifecycle-base') {
+            project.tasks.getByName(LifecycleBasePlugin.CHECK_TASK_NAME).dependsOn(project.tasks.withType(Javadoc))
+        }
         configureJavadocJar(project)
     }
 
@@ -891,7 +895,6 @@ class BuildPlugin implements Plugin<Project> {
                 test.systemProperty('io.netty.noUnsafe', 'true')
                 test.systemProperty('io.netty.noKeySetOptimization', 'true')
                 test.systemProperty('io.netty.recycler.maxCapacityPerThread', '0')
-                test.systemProperty('io.netty.allocator.numDirectArenas', '0')
 
                 test.testLogging { TestLoggingContainer logging ->
                     logging.showExceptions = true

buildSrc/src/main/java/org/elasticsearch/gradle/precommit/JarHellTask.java

@@ -42,7 +42,7 @@ public JarHellTask() {
     @TaskAction
     public void runJarHellCheck() {
         LoggedExec.javaexec(getProject(), spec -> {
-            spec.classpath(getClasspath());
+            spec.environment("CLASSPATH", getClasspath().getAsPath());
             spec.setMain("org.elasticsearch.bootstrap.JarHell");
         });
     }

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java

@@ -36,7 +36,7 @@ public Boolean getDebug() {
 
     @Override
     public void beforeStart() {
-        int debugPort = 8000;
+        int debugPort = 5005;
         int httpPort = 9200;
         int transportPort = 9300;
         Map<String, String> additionalSettings = System.getProperties().entrySet().stream()
@@ -57,7 +57,7 @@ public void beforeStart() {
                         "Running elasticsearch in debug mode, {} suspending until connected on debugPort {}",
                         node, debugPort
                     );
-                    node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=" + debugPort);
+                    node.jvmArgs("-agentlib:jdwp=transport=dt_socket,server=n,suspend=y,address=" + debugPort);
                     debugPort += 1;
                 }
             }

client/rest-high-level/src/main/java/org/elasticsearch/client/core/TermVectorsResponse.java

@@ -133,7 +133,7 @@ public boolean equals(Object obj) {
             && Objects.equals(id, other.id)
             && docVersion == other.docVersion
             && found == other.found
-            && tookInMillis == tookInMillis
+            && tookInMillis == other.tookInMillis
             && Objects.equals(termVectorList, other.termVectorList);
     }
 

client/rest-high-level/src/main/java/org/elasticsearch/client/enrich/NamedPolicy.java

@@ -20,7 +20,6 @@
 
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.ParsingException;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
 import org.elasticsearch.common.xcontent.XContentBuilder;
@@ -60,7 +59,7 @@ private static void declareParserOptions(ConstructingObjectParser<?, ?> parser)
         parser.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
             XContentBuilder builder = XContentBuilder.builder(p.contentType().xContent());
             builder.copyCurrentStructure(p);
-            return BytesArray.bytes(builder);
+            return BytesReference.bytes(builder);
         }, QUERY_FIELD);
         parser.declareStringArray(ConstructingObjectParser.constructorArg(), INDICES_FIELD);
         parser.declareString(ConstructingObjectParser.constructorArg(), MATCH_FIELD_FIELD);

client/rest-high-level/src/main/java/org/elasticsearch/client/ilm/IndexLifecycleExplainResponse.java

@@ -21,7 +21,6 @@
 
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.bytes.BytesArray;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
@@ -86,7 +85,7 @@ public class IndexLifecycleExplainResponse implements ToXContentObject {
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> {
             XContentBuilder builder = JsonXContent.contentBuilder();
             builder.copyCurrentStructure(p);
-            return BytesArray.bytes(builder);
+            return BytesReference.bytes(builder);
         }, STEP_INFO_FIELD);
         PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> PhaseExecutionInfo.parse(p, ""),
             PHASE_EXECUTION_INFO);

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/dataframe/evaluation/classification/MulticlassConfusionMatrixMetric.java

@@ -21,17 +21,17 @@
 import org.elasticsearch.client.ml.dataframe.evaluation.EvaluationMetric;
 import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 
 import java.io.IOException;
 import java.util.Collections;
-import java.util.Map;
+import java.util.List;
 import java.util.Objects;
-import java.util.TreeMap;
 
-import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
 
 /**
@@ -97,52 +97,52 @@ public int hashCode() {
     public static class Result implements EvaluationMetric.Result {
 
         private static final ParseField CONFUSION_MATRIX = new ParseField("confusion_matrix");
-        private static final ParseField OTHER_CLASSES_COUNT = new ParseField("_other_");
+        private static final ParseField OTHER_ACTUAL_CLASS_COUNT = new ParseField("other_actual_class_count");
 
         @SuppressWarnings("unchecked")
         private static final ConstructingObjectParser<Result, Void> PARSER =
             new ConstructingObjectParser<>(
-                "multiclass_confusion_matrix_result", true, a -> new Result((Map<String, Map<String, Long>>) a[0], (long) a[1]));
+                "multiclass_confusion_matrix_result", true, a -> new Result((List<ActualClass>) a[0], (Long) a[1]));
 
         static {
-            PARSER.declareObject(
-                constructorArg(),
-                (p, c) -> p.map(TreeMap::new, p2 -> p2.map(TreeMap::new, XContentParser::longValue)),
-                CONFUSION_MATRIX);
-            PARSER.declareLong(constructorArg(), OTHER_CLASSES_COUNT);
+            PARSER.declareObjectArray(optionalConstructorArg(), ActualClass.PARSER, CONFUSION_MATRIX);
+            PARSER.declareLong(optionalConstructorArg(), OTHER_ACTUAL_CLASS_COUNT);
         }
 
         public static Result fromXContent(XContentParser parser) {
             return PARSER.apply(parser, null);
         }
 
-        // Immutable
-        private final Map<String, Map<String, Long>> confusionMatrix;
-        private final long otherClassesCount;
+        private final List<ActualClass> confusionMatrix;
+        private final Long otherActualClassCount;
 
-        public Result(Map<String, Map<String, Long>> confusionMatrix, long otherClassesCount) {
-            this.confusionMatrix = Collections.unmodifiableMap(Objects.requireNonNull(confusionMatrix));
-            this.otherClassesCount = otherClassesCount;
+        public Result(@Nullable List<ActualClass> confusionMatrix, @Nullable Long otherActualClassCount) {
+            this.confusionMatrix = confusionMatrix != null ? Collections.unmodifiableList(Objects.requireNonNull(confusionMatrix)) : null;
+            this.otherActualClassCount = otherActualClassCount;
         }
 
         @Override
         public String getMetricName() {
             return NAME;
         }
 
-        public Map<String, Map<String, Long>> getConfusionMatrix() {
+        public List<ActualClass> getConfusionMatrix() {
             return confusionMatrix;
         }
 
-        public long getOtherClassesCount() {
-            return otherClassesCount;
+        public Long getOtherActualClassCount() {
+            return otherActualClassCount;
         }
 
         @Override
         public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
             builder.startObject();
-            builder.field(CONFUSION_MATRIX.getPreferredName(), confusionMatrix);
-            builder.field(OTHER_CLASSES_COUNT.getPreferredName(), otherClassesCount);
+            if (confusionMatrix != null) {
+                builder.field(CONFUSION_MATRIX.getPreferredName(), confusionMatrix);
+            }
+            if (otherActualClassCount != null) {
+                builder.field(OTHER_ACTUAL_CLASS_COUNT.getPreferredName(), otherActualClassCount);
+            }
             builder.endObject();
             return builder;
         }
@@ -153,12 +153,140 @@ public boolean equals(Object o) {
             if (o == null || getClass() != o.getClass()) return false;
             Result that = (Result) o;
             return Objects.equals(this.confusionMatrix, that.confusionMatrix)
-                && this.otherClassesCount == that.otherClassesCount;
+                && Objects.equals(this.otherActualClassCount, that.otherActualClassCount);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(confusionMatrix, otherClassesCount);
+            return Objects.hash(confusionMatrix, otherActualClassCount);
+        }
+    }
+
+    public static class ActualClass implements ToXContentObject {
+
+        private static final ParseField ACTUAL_CLASS = new ParseField("actual_class");
+        private static final ParseField ACTUAL_CLASS_DOC_COUNT = new ParseField("actual_class_doc_count");
+        private static final ParseField PREDICTED_CLASSES = new ParseField("predicted_classes");
+        private static final ParseField OTHER_PREDICTED_CLASS_DOC_COUNT = new ParseField("other_predicted_class_doc_count");
+
+        @SuppressWarnings("unchecked")
+        private static final ConstructingObjectParser<ActualClass, Void> PARSER =
+            new ConstructingObjectParser<>(
+                "multiclass_confusion_matrix_actual_class",
+                true,
+                a -> new ActualClass((String) a[0], (Long) a[1], (List<PredictedClass>) a[2], (Long) a[3]));
+
+        static {
+            PARSER.declareString(optionalConstructorArg(), ACTUAL_CLASS);
+            PARSER.declareLong(optionalConstructorArg(), ACTUAL_CLASS_DOC_COUNT);
+            PARSER.declareObjectArray(optionalConstructorArg(), PredictedClass.PARSER, PREDICTED_CLASSES);
+            PARSER.declareLong(optionalConstructorArg(), OTHER_PREDICTED_CLASS_DOC_COUNT);
+        }
+
+        private final String actualClass;
+        private final Long actualClassDocCount;
+        private final List<PredictedClass> predictedClasses;
+        private final Long otherPredictedClassDocCount;
+
+        public ActualClass(@Nullable String actualClass,
+                           @Nullable Long actualClassDocCount,
+                           @Nullable List<PredictedClass> predictedClasses,
+                           @Nullable Long otherPredictedClassDocCount) {
+            this.actualClass = actualClass;
+            this.actualClassDocCount = actualClassDocCount;
+            this.predictedClasses = predictedClasses != null ? Collections.unmodifiableList(predictedClasses) : null;
+            this.otherPredictedClassDocCount = otherPredictedClassDocCount;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            if (actualClass != null) {
+                builder.field(ACTUAL_CLASS.getPreferredName(), actualClass);
+            }
+            if (actualClassDocCount != null) {
+                builder.field(ACTUAL_CLASS_DOC_COUNT.getPreferredName(), actualClassDocCount);
+            }
+            if (predictedClasses != null) {
+                builder.field(PREDICTED_CLASSES.getPreferredName(), predictedClasses);
+            }
+            if (otherPredictedClassDocCount != null) {
+                builder.field(OTHER_PREDICTED_CLASS_DOC_COUNT.getPreferredName(), otherPredictedClassDocCount);
+            }
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            ActualClass that = (ActualClass) o;
+            return Objects.equals(this.actualClass, that.actualClass)
+                && Objects.equals(this.actualClassDocCount, that.actualClassDocCount)
+                && Objects.equals(this.predictedClasses, that.predictedClasses)
+                && Objects.equals(this.otherPredictedClassDocCount, that.otherPredictedClassDocCount);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(actualClass, actualClassDocCount, predictedClasses, otherPredictedClassDocCount);
+        }
+
+        @Override
+        public String toString() {
+            return Strings.toString(this);
+        }
+    }
+
+    public static class PredictedClass implements ToXContentObject {
+
+        private static final ParseField PREDICTED_CLASS = new ParseField("predicted_class");
+        private static final ParseField COUNT = new ParseField("count");
+
+        @SuppressWarnings("unchecked")
+        private static final ConstructingObjectParser<PredictedClass, Void> PARSER =
+            new ConstructingObjectParser<>(
+                "multiclass_confusion_matrix_predicted_class", true, a -> new PredictedClass((String) a[0], (Long) a[1]));
+
+        static {
+            PARSER.declareString(optionalConstructorArg(), PREDICTED_CLASS);
+            PARSER.declareLong(optionalConstructorArg(), COUNT);
+        }
+
+        private final String predictedClass;
+        private final Long count;
+
+        public PredictedClass(@Nullable String predictedClass, @Nullable Long count) {
+            this.predictedClass = predictedClass;
+            this.count = count;
+        }
+
+        @Override
+        public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+            builder.startObject();
+            if (predictedClass != null) {
+                builder.field(PREDICTED_CLASS.getPreferredName(), predictedClass);
+            }
+            if (count != null) {
+                builder.field(COUNT.getPreferredName(), count);
+            }
+            builder.endObject();
+            return builder;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            PredictedClass that = (PredictedClass) o;
+            return Objects.equals(this.predictedClass, that.predictedClass)
+                && Objects.equals(this.count, that.count);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(predictedClass, count);
         }
     }
 }