[7.x][Transform] add support for missing bucket (#59591) (#60390)

add support for "missing_bucket" in group_by

fixes #42941
fixes #55102
backport #59591

client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/DateHistogramGroupSource.java

@@ -188,9 +188,10 @@ public int hashCode() {
         (args) -> {
             String field = (String) args[0];
             Script script = (Script) args[1];
-            String fixedInterval = (String) args[2];
-            String calendarInterval = (String) args[3];
-            ZoneId zoneId = (ZoneId) args[4];
+            boolean missingBucket = args[2] == null ? false : (boolean) args[2];
+            String fixedInterval = (String) args[3];
+            String calendarInterval = (String) args[4];
+            ZoneId zoneId = (ZoneId) args[5];
 
             Interval interval = null;
 
@@ -204,13 +205,14 @@ public int hashCode() {
                 throw new IllegalArgumentException("You must specify either fixed_interval or calendar_interval, found none");
             }
 
-            return new DateHistogramGroupSource(field, script, interval, zoneId);
+            return new DateHistogramGroupSource(field, script, missingBucket, interval, zoneId);
         }
     );
 
     static {
         PARSER.declareString(optionalConstructorArg(), FIELD);
         Script.declareScript(PARSER, optionalConstructorArg(), SCRIPT);
+        PARSER.declareBoolean(optionalConstructorArg(), MISSING_BUCKET);
         PARSER.declareString(optionalConstructorArg(), new ParseField(FixedInterval.NAME));
         PARSER.declareString(optionalConstructorArg(), new ParseField(CalendarInterval.NAME));
 
@@ -231,7 +233,11 @@ public static DateHistogramGroupSource fromXContent(final XContentParser parser)
     private final ZoneId timeZone;
 
     DateHistogramGroupSource(String field, Script script, Interval interval, ZoneId timeZone) {
-        super(field, script);
+        this(field, script, false, interval, timeZone);
+    }
+
+    DateHistogramGroupSource(String field, Script script, boolean missingBucket, Interval interval, ZoneId timeZone) {
+        super(field, script, missingBucket);
         this.interval = interval;
         this.timeZone = timeZone;
     }
@@ -273,14 +279,16 @@ public boolean equals(Object other) {
 
         final DateHistogramGroupSource that = (DateHistogramGroupSource) other;
 
-        return Objects.equals(this.field, that.field)
+        return this.missingBucket == that.missingBucket
+            && Objects.equals(this.field, that.field)
+            && Objects.equals(this.script, that.script)
             && Objects.equals(this.interval, that.interval)
             && Objects.equals(this.timeZone, that.timeZone);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(field, interval, timeZone);
+        return Objects.hash(field, script, missingBucket, interval, timeZone);
     }
 
     @Override
@@ -298,6 +306,7 @@ public static class Builder {
         private Script script;
         private Interval interval;
         private ZoneId timeZone;
+        private boolean missingBucket;
 
         /**
          * The field with which to construct the date histogram grouping
@@ -339,8 +348,18 @@ public Builder setTimeZone(ZoneId timeZone) {
             return this;
         }
 
+        /**
+         * Sets the value of "missing_bucket"
+         * @param missingBucket value of "missing_bucket" to be set
+         * @return The {@link Builder} with "missing_bucket" set.
+         */
+        public Builder setMissingBucket(boolean missingBucket) {
+            this.missingBucket = missingBucket;
+            return this;
+        }
+
         public DateHistogramGroupSource build() {
-            return new DateHistogramGroupSource(field, script, interval, timeZone);
+            return new DateHistogramGroupSource(field, script, missingBucket, interval, timeZone);
         }
     }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/GeoTileGroupSource.java

@@ -40,16 +40,18 @@ public class GeoTileGroupSource extends SingleGroupSource implements ToXContentO
     private static final String NAME = "transform_geo_tile_group";
 
     private static final ParseField PRECISION = new ParseField("precision");
-    private static final ConstructingObjectParser<GeoTileGroupSource, Void> PARSER = new ConstructingObjectParser<>(NAME, true,  (args) -> {
+    private static final ConstructingObjectParser<GeoTileGroupSource, Void> PARSER = new ConstructingObjectParser<>(NAME, true, (args) -> {
         String field = (String) args[0];
-        Integer precision = (Integer) args[1];
-        GeoBoundingBox boundingBox = (GeoBoundingBox) args[2];
+        boolean missingBucket = args[1] == null ? false : (boolean) args[1];
+        Integer precision = (Integer) args[2];
+        GeoBoundingBox boundingBox = (GeoBoundingBox) args[3];
 
-        return new GeoTileGroupSource(field, precision, boundingBox);
+        return new GeoTileGroupSource(field, missingBucket, precision, boundingBox);
     });
 
     static {
         PARSER.declareString(optionalConstructorArg(), FIELD);
+        PARSER.declareBoolean(optionalConstructorArg(), MISSING_BUCKET);
         PARSER.declareInt(optionalConstructorArg(), PRECISION);
         PARSER.declareField(
             optionalConstructorArg(),
@@ -62,7 +64,11 @@ public class GeoTileGroupSource extends SingleGroupSource implements ToXContentO
     private final GeoBoundingBox geoBoundingBox;
 
     public GeoTileGroupSource(final String field, final Integer precision, final GeoBoundingBox boundingBox) {
-        super(field, null);
+        this(field, false, precision, boundingBox);
+    }
+
+    public GeoTileGroupSource(final String field, final boolean missingBucket, final Integer precision, final GeoBoundingBox boundingBox) {
+        super(field, null, missingBucket);
         if (precision != null) {
             GeoTileUtils.checkPrecisionRange(precision);
         }
@@ -113,14 +119,66 @@ public boolean equals(Object other) {
 
         final GeoTileGroupSource that = (GeoTileGroupSource) other;
 
-        return Objects.equals(this.field, that.field)
+        return this.missingBucket == that.missingBucket
+            && Objects.equals(this.field, that.field)
             && Objects.equals(this.precision, that.precision)
             && Objects.equals(this.geoBoundingBox, that.geoBoundingBox);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(field, precision, geoBoundingBox);
+        return Objects.hash(field, missingBucket, precision, geoBoundingBox);
     }
 
+    public static class Builder {
+
+        private String field;
+        private boolean missingBucket;
+        private Integer precision;
+        private GeoBoundingBox boundingBox;
+
+        /**
+         * The field with which to construct the geo tile grouping
+         * @param field The field name
+         * @return The {@link Builder} with the field set.
+         */
+        public Builder setField(String field) {
+            this.field = field;
+            return this;
+        }
+
+        /**
+         * Sets the value of "missing_bucket"
+         * @param missingBucket value of "missing_bucket" to be set
+         * @return The {@link Builder} with "missing_bucket" set.
+         */
+        public Builder setMissingBucket(boolean missingBucket) {
+            this.missingBucket = missingBucket;
+            return this;
+        }
+
+        /**
+         * The precision with which to construct the geo tile grouping
+         * @param precision The precision
+         * @return The {@link Builder} with the precision set.
+         */
+        public Builder setPrecission(Integer precision) {
+            this.precision = precision;
+            return this;
+        }
+
+        /**
+         * Set the bounding box for the geo tile grouping
+         * @param boundingBox The bounding box
+         * @return the {@link Builder} with the bounding box set.
+         */
+        public Builder setBoundingBox(GeoBoundingBox boundingBox) {
+            this.boundingBox = boundingBox;
+            return this;
+        }
+
+        public GeoTileGroupSource build() {
+            return new GeoTileGroupSource(field, missingBucket, precision, boundingBox);
+        }
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/HistogramGroupSource.java

@@ -41,12 +41,13 @@ public class HistogramGroupSource extends SingleGroupSource implements ToXConten
     private static final ConstructingObjectParser<HistogramGroupSource, Void> PARSER = new ConstructingObjectParser<>(
         "histogram_group_source",
         true,
-        args -> new HistogramGroupSource((String) args[0], (Script) args[1], (double) args[2])
+        args -> new HistogramGroupSource((String) args[0], (Script) args[1], args[2] == null ? false : (boolean) args[2], (double) args[3])
     );
 
     static {
         PARSER.declareString(optionalConstructorArg(), FIELD);
         Script.declareScript(PARSER, optionalConstructorArg(), SCRIPT);
+        PARSER.declareBoolean(optionalConstructorArg(), MISSING_BUCKET);
         PARSER.declareDouble(optionalConstructorArg(), INTERVAL);
     }
 
@@ -57,7 +58,11 @@ public static HistogramGroupSource fromXContent(final XContentParser parser) {
     private final double interval;
 
     HistogramGroupSource(String field, Script script, double interval) {
-        super(field, script);
+        this(field, script, false, interval);
+    }
+
+    HistogramGroupSource(String field, Script script, boolean missingBucket, double interval) {
+        super(field, script, missingBucket);
         if (interval <= 0) {
             throw new IllegalArgumentException("[interval] must be greater than 0.");
         }
@@ -94,12 +99,15 @@ public boolean equals(Object other) {
 
         final HistogramGroupSource that = (HistogramGroupSource) other;
 
-        return Objects.equals(this.field, that.field) && Objects.equals(this.interval, that.interval);
+        return this.missingBucket == that.missingBucket
+            && Objects.equals(this.field, that.field)
+            && Objects.equals(this.script, that.script)
+            && Objects.equals(this.interval, that.interval);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(field, interval);
+        return Objects.hash(field, script, interval, missingBucket);
     }
 
     public static Builder builder() {
@@ -110,6 +118,7 @@ public static class Builder {
 
         private String field;
         private Script script;
+        private boolean missingBucket;
         private double interval;
 
         /**
@@ -123,7 +132,7 @@ public Builder setField(String field) {
         }
 
         /**
-         * Set the interval for the histogram aggregation
+         * Set the interval for the histogram grouping
          * @param interval The numeric interval for the histogram grouping
          * @return The {@link Builder} with the interval set.
          */
@@ -142,8 +151,18 @@ public Builder setScript(Script script) {
             return this;
         }
 
+        /**
+         * Sets the value of "missing_bucket"
+         * @param missingBucket value of "missing_bucket" to be set
+         * @return The {@link Builder} with "missing_bucket" set.
+         */
+        public Builder setMissingBucket(boolean missingBucket) {
+            this.missingBucket = missingBucket;
+            return this;
+        }
+
         public HistogramGroupSource build() {
-            return new HistogramGroupSource(field, script, interval);
+            return new HistogramGroupSource(field, script, missingBucket, interval);
         }
     }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/SingleGroupSource.java

@@ -32,6 +32,7 @@ public abstract class SingleGroupSource implements ToXContentObject {
 
     protected static final ParseField FIELD = new ParseField("field");
     protected static final ParseField SCRIPT = new ParseField("script");
+    protected static final ParseField MISSING_BUCKET = new ParseField("missing_bucket");
 
     public enum Type {
         TERMS,
@@ -46,10 +47,12 @@ public String value() {
 
     protected final String field;
     protected final Script script;
+    protected final boolean missingBucket;
 
-    public SingleGroupSource(final String field, final Script script) {
+    public SingleGroupSource(final String field, final Script script, final boolean missingBucket) {
         this.field = field;
         this.script = script;
+        this.missingBucket = missingBucket;
     }
 
     public abstract Type getType();
@@ -62,13 +65,20 @@ public Script getScript() {
         return script;
     }
 
+    public boolean getMissingBucket() {
+        return missingBucket;
+    }
+
     protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
         if (field != null) {
             builder.field(FIELD.getPreferredName(), field);
         }
         if (script != null) {
             builder.field(SCRIPT.getPreferredName(), script);
         }
+        if (missingBucket) {
+            builder.field(MISSING_BUCKET.getPreferredName(), missingBucket);
+        }
     }
 
     @Override
@@ -83,11 +93,13 @@ public boolean equals(Object other) {
 
         final SingleGroupSource that = (SingleGroupSource) other;
 
-        return Objects.equals(this.field, that.field) && Objects.equals(this.script, that.script);
+        return this.missingBucket == that.missingBucket
+            && Objects.equals(this.field, that.field)
+            && Objects.equals(this.script, that.script);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(field, script);
+        return Objects.hash(field, script, missingBucket);
     }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/transform/transforms/pivot/TermsGroupSource.java

@@ -35,20 +35,25 @@ public class TermsGroupSource extends SingleGroupSource implements ToXContentObj
     private static final ConstructingObjectParser<TermsGroupSource, Void> PARSER = new ConstructingObjectParser<>(
         "terms_group_source",
         true,
-        args -> new TermsGroupSource((String) args[0], (Script) args[1])
+        args -> new TermsGroupSource((String) args[0], (Script) args[1], args[2] == null ? false : (boolean) args[2])
     );
 
     static {
         PARSER.declareString(optionalConstructorArg(), FIELD);
         Script.declareScript(PARSER, optionalConstructorArg(), SCRIPT);
+        PARSER.declareBoolean(optionalConstructorArg(), MISSING_BUCKET);
     }
 
     public static TermsGroupSource fromXContent(final XContentParser parser) {
         return PARSER.apply(parser, null);
     }
 
     TermsGroupSource(final String field, final Script script) {
-        super(field, script);
+        this(field, script, false);
+    }
+
+    TermsGroupSource(final String field, final Script script, final boolean missingBucket) {
+        super(field, script, missingBucket);
     }
 
     @Override
@@ -72,9 +77,10 @@ public static class Builder {
 
         private String field;
         private Script script;
+        private boolean missingBucket;
 
         /**
-         * The field with which to construct the date histogram grouping
+         * The field with which to construct the terms grouping
          * @param field The field name
          * @return The {@link Builder} with the field set.
          */
@@ -93,8 +99,18 @@ public Builder setScript(Script script) {
             return this;
         }
 
+        /**
+         * Sets the value of "missing_bucket"
+         * @param missingBucket value of "missing_bucket" to be set
+         * @return The {@link Builder} with "missing_bucket" set.
+         */
+        public Builder setMissingBucket(boolean missingBucket) {
+            this.missingBucket = missingBucket;
+            return this;
+        }
+
         public TermsGroupSource build() {
-            return new TermsGroupSource(field, script);
+            return new TermsGroupSource(field, script, missingBucket);
         }
     }
 }