Merge branch 'main' of https://github.com/elastic/elasticsearch into …

…fix_non_snapshot_error_messages

docs/changelog/112063.yaml

@@ -0,0 +1,32 @@
+pr: 112063
+summary: Spatial search functions support multi-valued fields in compute engine
+area: ES|QL
+type: bug
+issues:
+ - 112102
+ - 112505
+ - 110830
+highlight:
+  title: "ESQL: Multi-value fields supported in Geospatial predicates"
+  body: |-
+    Supporting multi-value fields in `WHERE` predicates is a challenge due to not knowing whether `ALL` or `ANY`
+    of the values in the field should pass the predicate.
+    For example, should the field `age:[10,30]` pass the predicate `WHERE age>20` or not?
+    This ambiguity does not exist with the spatial predicates
+    `ST_INTERSECTS` and `ST_DISJOINT`, because the choice between `ANY` or `ALL`
+    is implied by the predicate itself.
+    Consider a predicate checking a field named `location` against a test geometry named `shape`:
+
+    * `ST_INTERSECTS(field, shape)` - true if `ANY` value can intersect the shape
+    * `ST_DISJOINT(field, shape)` - true only if `ALL` values are disjoint from the shape
+
+    This works even if the shape argument is itself a complex or compound geometry.
+
+    Similar logic exists for `ST_CONTAINS` and `ST_WITHIN` predicates, but these are not as easily solved
+    with `ANY` or `ALL`, because a collection of geometries contains another collection if each of the contained
+    geometries is within at least one of the containing geometries. Evaluating this requires that the multi-value
+    field is first combined into a single geometry before performing the predicate check.
+
+    * `ST_CONTAINS(field, shape)` - true if the combined geometry contains the shape
+    * `ST_WITHIN(field, shape)` - true if the combined geometry is within the shape
+  notable: false

updatecli-compose.yaml

@@ -6,9 +6,8 @@ policies:
     values:
       - .github/updatecli/values.d/scm.yml
       - .github/updatecli/values.d/ironbank.yml
-
   - name: Update Updatecli policies
-    policy: ghcr.io/updatecli/policies/autodiscovery/updatecli:0.4.0@sha256:254367f5b1454fd6032b88b314450cd3b6d5e8d5b6c953eb242a6464105eb869
+    policy: ghcr.io/updatecli/policies/autodiscovery/updatecli:0.6.0@sha256:6bd6999620674b2fbb1d374f7a1a5e9740d042667f0592900b44259f3e1ae98f
     values:
       - .github/updatecli/values.d/scm.yml
-      - .github/updatecli/values.d/updatecli-compose.yml
+      - .github/updatecli/values.d/updatecli-compose.yml

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java

@@ -122,8 +122,19 @@ public static Tuple<Page, List<String>> loadPageFromCsv(URL source, Map<String,
 
         record CsvColumn(String name, Type type, BuilderWrapper builderWrapper) implements Releasable {
             void append(String stringValue) {
-                if (stringValue.startsWith("\"") && stringValue.endsWith("\"")) { // string value
-                    stringValue = stringValue.substring(1, stringValue.length() - 1).replace(ESCAPED_COMMA_SEQUENCE, ",");
+                if (stringValue.startsWith("\"") && stringValue.endsWith("\"")) {
+                    // string value
+                    String[] mvStrings = stringValue.substring(1, stringValue.length() - 1).split("\",\\s*\"");
+                    if (mvStrings.length > 1) {
+                        builderWrapper().builder().beginPositionEntry();
+                        for (String mvString : mvStrings) {
+                            mvString = mvString.replace(ESCAPED_COMMA_SEQUENCE, ",");
+                            builderWrapper().append().accept(mvString.length() == 0 ? null : type.convert(mvString));
+                        }
+                        builderWrapper().builder().endPositionEntry();
+                        return;
+                    }
+                    stringValue = mvStrings[0].replace(ESCAPED_COMMA_SEQUENCE, ",");
                 } else if (stringValue.contains(",")) {// multi-value field
                     builderWrapper().builder().beginPositionEntry();
 
@@ -376,7 +387,20 @@ public static ExpectedResults loadCsvSpecValues(String csv) {
                         }
                         List<Object> listOfMvValues = new ArrayList<>();
                         for (String mvValue : multiValues) {
-                            listOfMvValues.add(columnTypes.get(i).convert(mvValue.trim().replace(ESCAPED_COMMA_SEQUENCE, ",")));
+                            try {
+                                listOfMvValues.add(columnTypes.get(i).convert(mvValue.trim().replace(ESCAPED_COMMA_SEQUENCE, ",")));
+                            } catch (IllegalArgumentException e) {
+                                throw new IllegalArgumentException(
+                                    "Error parsing multi-value field ["
+                                        + columnNames.get(i)
+                                        + "] with value ["
+                                        + mvValue
+                                        + "] on row "
+                                        + values.size(),
+                                    e
+                                );
+
+                            }
                         }
                         rowValues.add(listOfMvValues);
                     } else {

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java

@@ -75,6 +75,8 @@ public class CsvTestsDataLoader {
     private static final TestsDataset COUNTRIES_BBOX_WEB = new TestsDataset("countries_bbox_web");
     private static final TestsDataset AIRPORT_CITY_BOUNDARIES = new TestsDataset("airport_city_boundaries");
     private static final TestsDataset CARTESIAN_MULTIPOLYGONS = new TestsDataset("cartesian_multipolygons");
+    private static final TestsDataset MULTIVALUE_GEOMETRIES = new TestsDataset("multivalue_geometries");
+    private static final TestsDataset MULTIVALUE_POINTS = new TestsDataset("multivalue_points");
     private static final TestsDataset DISTANCES = new TestsDataset("distances");
     private static final TestsDataset K8S = new TestsDataset("k8s", "k8s-mappings.json", "k8s.csv").withSetting("k8s-settings.json");
     private static final TestsDataset ADDRESSES = new TestsDataset("addresses");
@@ -104,6 +106,8 @@ public class CsvTestsDataLoader {
         Map.entry(COUNTRIES_BBOX_WEB.indexName, COUNTRIES_BBOX_WEB),
         Map.entry(AIRPORT_CITY_BOUNDARIES.indexName, AIRPORT_CITY_BOUNDARIES),
         Map.entry(CARTESIAN_MULTIPOLYGONS.indexName, CARTESIAN_MULTIPOLYGONS),
+        Map.entry(MULTIVALUE_GEOMETRIES.indexName, MULTIVALUE_GEOMETRIES),
+        Map.entry(MULTIVALUE_POINTS.indexName, MULTIVALUE_POINTS),
         Map.entry(DATE_NANOS.indexName, DATE_NANOS),
         Map.entry(K8S.indexName, K8S),
         Map.entry(DISTANCES.indexName, DISTANCES),

x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-airports-no-doc-values.json

@@ -0,0 +1,30 @@
+{
+  "properties": {
+    "abbrev": {
+      "type": "keyword"
+    },
+    "name": {
+      "type": "text"
+    },
+    "scalerank": {
+      "type": "integer"
+    },
+    "type": {
+      "type": "keyword"
+    },
+    "location": {
+      "type": "geo_point",
+      "index": false,
+      "doc_values": false
+    },
+    "country": {
+      "type": "keyword"
+    },
+    "city": {
+      "type": "keyword"
+    },
+    "city_location": {
+      "type": "geo_point"
+    }
+  }
+}

x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-multivalue_geometries.json

@@ -0,0 +1,19 @@
+{
+  "properties": {
+    "id": {
+      "type": "long"
+    },
+    "intersects": {
+      "type": "boolean"
+    },
+    "contains": {
+      "type": "boolean"
+    },
+    "shape": {
+      "type": "geo_shape"
+    },
+    "smaller": {
+      "type": "geo_shape"
+    }
+  }
+}

x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-multivalue_points.json

@@ -0,0 +1,22 @@
+{
+  "properties": {
+    "id": {
+      "type": "long"
+    },
+    "intersects": {
+      "type": "boolean"
+    },
+    "within": {
+      "type": "boolean"
+    },
+    "centroid": {
+      "type": "geo_point"
+    },
+    "location": {
+      "type": "geo_point"
+    },
+    "subset": {
+      "type": "geo_point"
+    }
+  }
+}

x-pack/plugin/esql/qa/testFixtures/src/main/resources/multivalue_geometries.csv

@@ -0,0 +1,13 @@
+id:l, intersects:boolean, contains:boolean, shape:geo_shape,                                                                                                           smaller:geo_shape
+0,    true,               true,             ["GEOMETRYCOLLECTION(POLYGON ((-10 -10\, 0 -10\, 0 0\, -10 0\, -10 -10))\, POLYGON ((0 0\, 10 0\, 10 10\, 0 10\, 0 0)))"], ["GEOMETRYCOLLECTION(POLYGON ((-9 -9\, -1 -9\, -1 -1\, -9 -1\, -9 -9))\, POLYGON ((1 1\, 9 1\, 9 9\, 1 9\, 1 1)))"]
+1,    true,               true,             ["MULTIPOLYGON( ((-10 -10\, 0 -10\, 0 0\, -10 0\, -10 -10))\, ((0 0\, 10 0\, 10 10\, 0 10\, 0 0)))"],                      ["MULTIPOLYGON( ((-9 -9\, -1 -9\, -1 -1\, -9 -1\, -9 -9))\, ((1 1\, 9 1\, 9 9\, 1 9\, 1 1)))"]
+2,    true,               true,             ["POLYGON ((-15 -15\, 15 -15\, 15 15\, -15 15\, -15 -15))"],                                                               ["POLYGON ((-14 -14\, 14 -14\, 14 14\, -14 14\, -14 -14))"]
+3,    true,               true,             ["POLYGON ((-15 -15\, 15 -15\, 15 15\, -15 15\, -15 -15))", "POLYGON ((15 15\, 25 15\, 25 25\, 15 25\, 15 15))"],          ["POLYGON ((-14 -14\, 14 -14\, 14 14\, -14 14\, -14 -14))", "POLYGON ((16 16\, 24 16\, 24 24\, 16 24\, 16 16))"]
+4,    true,               true,             ["POLYGON ((-10 -10\, 0 -10\, 0 0\, -10 0\, -10 -10))", "POLYGON ((0 0\, 10 0\, 10 10\, 0 10\, 0 0))"],                    ["POLYGON ((-9 -9\, -1 -9\, -1 -1\, -9 -1\, -9 -9))", "POLYGON ((1 1\, 9 1\, 9 9\, 1 9\, 1 1))"]
+5,    true,               false,            ["POLYGON ((-5 -5\, 5 -5\, 5 5\, -5 5\, -5 -5))"],                                                                         ["POLYGON ((-4 -4\, 4 -4\, 4 4\, -4 4\, -4 -4))"]
+6,    true,               false,            ["POLYGON ((-5 -5\, 5 -5\, 5 5\, -5 5\, -5 -5))", "POLYGON ((15 15\, 25 15\, 25 25\, 15 25\, 15 15))"],                    ["POLYGON ((-4 -4\, 4 -4\, 4 4\, -4 4\, -4 -4))", "POLYGON ((16 16\, 24 16\, 24 24\, 16 24\, 16 16))"]
+7,    true,               false,            ["POLYGON ((-9 -9\, -1 -9\, -1 -1\, -9 -1\, -9 -9))", "POLYGON ((1 1\, 9 1\, 9 9\, 1 9\, 1 1))"],                          ["POLYGON ((-8 -8\, -2 -8\, -2 -2\, -8 -2\, -8 -8))", "POLYGON ((2 2\, 8 2\, 8 8\, 2 8\, 2 2))"]
+8,    false,              false,            ["POLYGON ((15 15\, 25 15\, 25 25\, 15 25\, 15 15))"],                                                                     ["POLYGON ((16 16\, 24 16\, 24 24\, 16 24\, 16 16))"]
+9,    false,              false,            ["POLYGON ((-25 -25\, -15 -25\, -15 -15\, -25 -15\, -25 -25))", "POLYGON ((15 15\, 25 15\, 25 25\, 15 25\, 15 15))"],      ["POLYGON ((-24 -24\, -16 -24\, -16 -16\, -24 -16\, -24 -24))", "POLYGON ((16 16\, 24 16\, 24 24\, 16 24\, 16 16))"]
+10,   true,               false,            ["POLYGON ((-15 -15\, 15 -15\, 15 15\, -15 15\, -15 -15))", "POLYGON ((5 5\, 15 5\, 15 15\, 5 15\, 5 5))"],                ["POLYGON ((-14 -14\, 14 -14\, 14 14\, -14 14\, -14 -14))", "POLYGON ((6 6\, 14 6\, 14 14\, 6 14\, 6 6))"]
+11,   true,               false,            ["POLYGON ((-11 -11\, 1 -11\, 1 1\, -11 1\, -11 -11))", "POLYGON ((-1 -1\, 11 -1\, 11 11\, -1 11\, -1 -1))"],              ["POLYGON ((-10 -10\, 0 -10\, 0 0\, -10 0\, -10 -10))", "POLYGON ((0 0\, 10 0\, 10 10\, 0 10\, 0 0))"]