[ML] cleanup + adding description field to transforms

client/rest-high-level/src/main/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfig.java

@@ -20,6 +20,7 @@
 package org.elasticsearch.client.dataframe.transforms;
 
 import org.elasticsearch.client.dataframe.transforms.pivot.PivotConfig;
+import org.elasticsearch.common.Nullable;
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.xcontent.ConstructingObjectParser;
@@ -38,13 +39,15 @@ public class DataFrameTransformConfig implements ToXContentObject {
     public static final ParseField ID = new ParseField("id");
     public static final ParseField SOURCE = new ParseField("source");
     public static final ParseField DEST = new ParseField("dest");
+    public static final ParseField DESCRIPTION = new ParseField("description");
     // types of transforms
     public static final ParseField PIVOT_TRANSFORM = new ParseField("pivot");
 
     private final String id;
     private final SourceConfig source;
     private final DestConfig dest;
     private final PivotConfig pivotConfig;
+    private final String description;
 
     public static final ConstructingObjectParser<DataFrameTransformConfig, Void> PARSER =
             new ConstructingObjectParser<>("data_frame_transform", true,
@@ -53,29 +56,47 @@ public class DataFrameTransformConfig implements ToXContentObject {
                     SourceConfig source = (SourceConfig) args[1];
                     DestConfig dest = (DestConfig) args[2];
                     PivotConfig pivotConfig = (PivotConfig) args[3];
-                    return new DataFrameTransformConfig(id, source, dest, pivotConfig);
+                    String description = (String)args[4];
+                    return new DataFrameTransformConfig(id, source, dest, pivotConfig, description);
                 });
 
     static {
         PARSER.declareString(constructorArg(), ID);
         PARSER.declareObject(constructorArg(), (p, c) -> SourceConfig.PARSER.apply(p, null), SOURCE);
         PARSER.declareObject(constructorArg(), (p, c) -> DestConfig.PARSER.apply(p, null), DEST);
         PARSER.declareObject(optionalConstructorArg(), (p, c) -> PivotConfig.fromXContent(p), PIVOT_TRANSFORM);
+        PARSER.declareString(optionalConstructorArg(), DESCRIPTION);
     }
 
     public static DataFrameTransformConfig fromXContent(final XContentParser parser) {
         return PARSER.apply(parser, null);
     }
 
+    /**
+     * Helper method for previewing a data frame transform configuration
+     *
+     * The DataFrameTransformConfig returned from this method should only be used for previewing the resulting data.
+     *
+     * A new, valid, DataFrameTransformConfig with an appropriate destination and ID will have to be constructed to create
+     * the transform.
+     * @param source Source configuration for gathering the data
+     * @param pivotConfig Pivot config to preview
+     * @return A DataFrameTransformConfig to preview, NOTE it will have a {@code null} id, destination and index.
+     */
+    public static DataFrameTransformConfig forPreview(final SourceConfig source, final PivotConfig pivotConfig) {
+        return new DataFrameTransformConfig(null, source, null, pivotConfig, null);
+    }
 
     public DataFrameTransformConfig(final String id,
                                     final SourceConfig source,
                                     final DestConfig dest,
-                                    final PivotConfig pivotConfig) {
+                                    final PivotConfig pivotConfig,
+                                    final String description) {
         this.id = id;
         this.source = source;
         this.dest = dest;
         this.pivotConfig = pivotConfig;
+        this.description = description;
     }
 
     public String getId() {
@@ -94,6 +115,11 @@ public PivotConfig getPivotConfig() {
         return pivotConfig;
     }
 
+    @Nullable
+    public String getDescription() {
+        return description;
+    }
+
     @Override
     public XContentBuilder toXContent(final XContentBuilder builder, final Params params) throws IOException {
         builder.startObject();
@@ -109,6 +135,9 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
         if (pivotConfig != null) {
             builder.field(PIVOT_TRANSFORM.getPreferredName(), pivotConfig);
         }
+        if (description != null) {
+            builder.field(DESCRIPTION.getPreferredName(), description);
+        }
         builder.endObject();
         return builder;
     }
@@ -128,12 +157,13 @@ public boolean equals(Object other) {
         return Objects.equals(this.id, that.id)
                 && Objects.equals(this.source, that.source)
                 && Objects.equals(this.dest, that.dest)
+                && Objects.equals(this.description, that.description)
                 && Objects.equals(this.pivotConfig, that.pivotConfig);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(id, source, dest, pivotConfig);
+        return Objects.hash(id, source, dest, pivotConfig, description);
     }
 
     @Override

client/rest-high-level/src/test/java/org/elasticsearch/client/DataFrameTransformIT.java

@@ -312,7 +312,8 @@ private DataFrameTransformConfig validDataFrameTransformConfig(String id, String
         return new DataFrameTransformConfig(id,
                 new SourceConfig(new String[]{source}, queryConfig),
                 destConfig,
-                pivotConfig);
+                pivotConfig,
+            "this is a test transform");
     }
 
     public void testGetStats() throws Exception {
@@ -329,7 +330,10 @@ public void testGetStats() throws Exception {
 
         String id = "test-get-stats";
         DataFrameTransformConfig transform = new DataFrameTransformConfig(id,
-                new SourceConfig(new String[]{sourceIndex}, queryConfig), new DestConfig("pivot-dest"), pivotConfig);
+            new SourceConfig(new String[]{sourceIndex}, queryConfig),
+            new DestConfig("pivot-dest"),
+            pivotConfig,
+            "transform for testing stats");
 
         DataFrameClient client = highLevelClient().dataFrame();
         AcknowledgedResponse ack = execute(new PutDataFrameTransformRequest(transform), client::putDataFrameTransform,

client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PreviewDataFrameTransformRequestTests.java

@@ -65,13 +65,12 @@ public void testValidate() {
                 containsString("preview requires a non-null data frame config"));
 
         // null id and destination is valid
-        DataFrameTransformConfig config = new DataFrameTransformConfig(null, randomSourceConfig(), null,
-            PivotConfigTests.randomPivotConfig());
+        DataFrameTransformConfig config = DataFrameTransformConfig.forPreview(randomSourceConfig(), PivotConfigTests.randomPivotConfig());
 
         assertFalse(new PreviewDataFrameTransformRequest(config).validate().isPresent());
 
         // null source is not valid
-        config = new DataFrameTransformConfig(null, null, null, PivotConfigTests.randomPivotConfig());
+        config = new DataFrameTransformConfig(null, null, null, PivotConfigTests.randomPivotConfig(), null);
 
         Optional<ValidationException> error = new PreviewDataFrameTransformRequest(config).validate();
         assertTrue(error.isPresent());

client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/PutDataFrameTransformRequestTests.java

@@ -40,7 +40,7 @@ public class PutDataFrameTransformRequestTests extends AbstractXContentTestCase<
     public void testValidate() {
         assertFalse(createTestInstance().validate().isPresent());
 
-        DataFrameTransformConfig config = new DataFrameTransformConfig(null, null, null, PivotConfigTests.randomPivotConfig());
+        DataFrameTransformConfig config = new DataFrameTransformConfig(null, null, null, PivotConfigTests.randomPivotConfig(), null);
 
         Optional<ValidationException> error = new PutDataFrameTransformRequest(config).validate();
         assertTrue(error.isPresent());

client/rest-high-level/src/test/java/org/elasticsearch/client/dataframe/transforms/DataFrameTransformConfigTests.java

@@ -37,7 +37,7 @@ public class DataFrameTransformConfigTests extends AbstractXContentTestCase<Data
 
     public static DataFrameTransformConfig randomDataFrameTransformConfig() {
         return new DataFrameTransformConfig(randomAlphaOfLengthBetween(1, 10), randomSourceConfig(),
-                randomDestConfig(), PivotConfigTests.randomPivotConfig());
+                randomDestConfig(), PivotConfigTests.randomPivotConfig(), randomBoolean() ? null : randomAlphaOfLengthBetween(1, 100));
     }
 
     @Override

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/DataFrameTransformDocumentationIT.java

@@ -141,7 +141,8 @@ public void testPutDataFrameTransform() throws IOException, InterruptedException
                 new DataFrameTransformConfig("reviewer-avg-rating", // <1>
                 sourceConfig, // <2>
                 new DestConfig("pivot-destination"),  // <3>
-                pivotConfig);  // <4>
+                pivotConfig, // <4>
+                "This is my test transform");  // <5>
         // end::put-data-frame-transform-config
 
         {
@@ -161,7 +162,7 @@ public void testPutDataFrameTransform() throws IOException, InterruptedException
         {
             DataFrameTransformConfig configWithDifferentId = new DataFrameTransformConfig("reviewer-avg-rating2",
                     transformConfig.getSource(), transformConfig.getDestination(),
-                    transformConfig.getPivotConfig());
+                    transformConfig.getPivotConfig(), null);
             PutDataFrameTransformRequest request = new PutDataFrameTransformRequest(configWithDifferentId);
 
             // tag::put-data-frame-transform-execute-listener
@@ -205,7 +206,7 @@ public void testStartStop() throws IOException, InterruptedException {
         PivotConfig pivotConfig = new PivotConfig(groupConfig, aggConfig);
 
         DataFrameTransformConfig transformConfig = new DataFrameTransformConfig("mega-transform",
-                new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("pivot-dest"), pivotConfig);
+                new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("pivot-dest"), pivotConfig, null);
 
         client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(transformConfig), RequestOptions.DEFAULT);
         transformsToClean.add(transformConfig.getId());
@@ -320,9 +321,9 @@ public void testDeleteDataFrameTransform() throws IOException, InterruptedExcept
         PivotConfig pivotConfig = new PivotConfig(groupConfig, aggConfig);
 
         DataFrameTransformConfig transformConfig1 = new DataFrameTransformConfig("mega-transform",
-                new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("pivot-dest"), pivotConfig);
+                new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("pivot-dest"), pivotConfig, null);
         DataFrameTransformConfig transformConfig2 = new DataFrameTransformConfig("mega-transform2",
-                new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("pivot-dest2"), pivotConfig);
+                new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("pivot-dest2"), pivotConfig, null);
 
         client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(transformConfig1), RequestOptions.DEFAULT);
         client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(transformConfig2), RequestOptions.DEFAULT);
@@ -386,11 +387,9 @@ public void testPreview() throws IOException, InterruptedException {
 
         // tag::preview-data-frame-transform-request
         DataFrameTransformConfig transformConfig =
-                new DataFrameTransformConfig(null,  // <1>
-                new SourceConfig(new String[]{"source-data"}, queryConfig),
-                null,                               // <2>
-                pivotConfig);
-
+            DataFrameTransformConfig.forPreview(
+                new SourceConfig(new String[]{"source-data"}, queryConfig), // <1>
+                pivotConfig); // <2>
         PreviewDataFrameTransformRequest request =
                 new PreviewDataFrameTransformRequest(transformConfig); // <3>
         // end::preview-data-frame-transform-request
@@ -447,7 +446,7 @@ public void testGetStats() throws IOException, InterruptedException {
 
         String id = "statisitcal-transform";
         DataFrameTransformConfig transformConfig = new DataFrameTransformConfig(id,
-                new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("dest"), pivotConfig);
+                new SourceConfig(new String[]{"source-data"}, queryConfig), new DestConfig("dest"), pivotConfig, null);
         client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(transformConfig), RequestOptions.DEFAULT);
 
         // tag::get-data-frame-transform-stats-request
@@ -526,7 +525,7 @@ public void testGetDataFrameTransform() throws IOException, InterruptedException
 
         DataFrameTransformConfig putTransformConfig = new DataFrameTransformConfig("mega-transform",
                 new SourceConfig(new String[]{"source-data"}, queryConfig),
-                new DestConfig("pivot-dest"), pivotConfig);
+                new DestConfig("pivot-dest"), pivotConfig, null);
 
         RestHighLevelClient client = highLevelClient();
         client.dataFrame().putDataFrameTransform(new PutDataFrameTransformRequest(putTransformConfig), RequestOptions.DEFAULT);

docs/java-rest/high-level/dataframe/preview_data_frame.asciidoc

@@ -20,8 +20,8 @@ A +{request}+ takes a single argument: a valid data frame transform config.
 --------------------------------------------------
 include-tagged::{doc-tests-file}[{api}-request]
 --------------------------------------------------
-<1> The transform Id may be null for the preview
-<2> The destination may be null for the preview
+<1> The source config from which the data should be gathered
+<2> The pivot config used to transform the data
 <3> The configuration of the {dataframe-job} to preview
 
 include::../execution.asciidoc[]

docs/java-rest/high-level/dataframe/put_data_frame.asciidoc

@@ -35,6 +35,7 @@ include-tagged::{doc-tests-file}[{api}-config]
 <2> The source indices and query from which to gather data
 <3> The destination index
 <4> The PivotConfig
+<5> Optional free text description of the transform
 
 [id="{upid}-{api}-query-config"]
 

docs/reference/data-frames/apis/put-transform.asciidoc

@@ -33,6 +33,8 @@ a `query`.
 `pivot`:: Defines the pivot function `group by` fields and the aggregation to
 reduce the data.
 
+`description`:: Optional free text description of the data frame transform
+
 
 //==== Authorization
 
@@ -73,7 +75,8 @@ PUT _data_frame/transforms/ecommerce_transform
         }
       }
     }
-  }
+  },
+  "description": "Maximum priced ecommerce data by customer_id in Asia"
 }
 --------------------------------------------------
 // CONSOLE

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/DeleteDataFrameTransformAction.java

@@ -18,7 +18,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.Collections;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StartDataFrameTransformAction.java

@@ -16,7 +16,7 @@
 import org.elasticsearch.common.xcontent.ToXContentObject;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.Collections;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StartDataFrameTransformTaskAction.java

@@ -17,7 +17,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.Collections;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/action/StopDataFrameTransformAction.java

@@ -18,7 +18,7 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.tasks.Task;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
-import org.elasticsearch.xpack.core.ml.utils.ExceptionsHelper;
+import org.elasticsearch.xpack.core.dataframe.utils.ExceptionsHelper;
 
 import java.io.IOException;
 import java.util.Arrays;

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/dataframe/notifications/DataFrameAuditMessage.java

@@ -12,7 +12,7 @@
 import org.elasticsearch.xpack.core.common.notifications.AbstractAuditMessage;
 import org.elasticsearch.xpack.core.common.notifications.Level;
 import org.elasticsearch.xpack.core.dataframe.DataFrameField;
-import org.elasticsearch.xpack.core.ml.utils.time.TimeUtils;
+import org.elasticsearch.xpack.core.dataframe.utils.TimeUtils;
 
 import java.util.Date;
 
@@ -36,15 +36,10 @@ public class DataFrameAuditMessage extends AbstractAuditMessage {
             }
             throw new IllegalArgumentException("Unsupported token [" + p.currentToken() + "]");
         }, LEVEL, ObjectParser.ValueType.STRING);
-        PARSER.declareField(constructorArg(), parser -> {
-            if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
-                return new Date(parser.longValue());
-            } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
-                return new Date(TimeUtils.dateStringToEpoch(parser.text()));
-            }
-            throw new IllegalArgumentException(
-                "unexpected token [" + parser.currentToken() + "] for [" + TIMESTAMP.getPreferredName() + "]");
-        }, TIMESTAMP, ObjectParser.ValueType.VALUE);
+        PARSER.declareField(constructorArg(),
+            p -> TimeUtils.parseTimeField(p, TIMESTAMP.getPreferredName()),
+            TIMESTAMP,
+            ObjectParser.ValueType.VALUE);
         PARSER.declareString(optionalConstructorArg(), NODE_NAME);
     }