Merge branch '8.x' into ES-9378_8.18

alexey-ivanov-es · Dec 13, 2024 · 7abd18e · 7abd18e
2 parents 9859dce + 9e2d44b
commit 7abd18e
Show file tree

Hide file tree

Showing 210 changed files with 4,197 additions and 1,629 deletions.
diff --git a/.buildkite/pipelines/intake.yml b/.buildkite/pipelines/intake.yml
@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.27", "8.15.6", "8.16.2", "8.17.0", "8.18.0"]
+            BWC_VERSION: ["7.17.27", "8.16.2", "8.17.0", "8.18.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.buildkite/pipelines/periodic-packaging.yml b/.buildkite/pipelines/periodic-packaging.yml
@@ -561,8 +561,8 @@ steps:
         env:
           BWC_VERSION: 8.14.3
 
-      - label: "{{matrix.image}} / 8.15.6 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.6
+      - label: "{{matrix.image}} / 8.15.5 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.15.5
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -575,7 +575,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.15.6
+          BWC_VERSION: 8.15.5
 
       - label: "{{matrix.image}} / 8.16.2 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.2

diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml
@@ -629,8 +629,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.15.6 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.6#bwcTest
+      - label: 8.15.5 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.15.5#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -639,7 +639,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.15.6
+          BWC_VERSION: 8.15.5
         retry:
           automatic:
             - exit_status: "-1"
@@ -771,7 +771,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.27", "8.15.6", "8.16.2", "8.17.0", "8.18.0"]
+            BWC_VERSION: ["7.17.27", "8.16.2", "8.17.0", "8.18.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -819,7 +819,7 @@ steps:
               - openjdk21
               - openjdk22
               - openjdk23
-            BWC_VERSION: ["7.17.27", "8.15.6", "8.16.2", "8.17.0", "8.18.0"]
+            BWC_VERSION: ["7.17.27", "8.16.2", "8.17.0", "8.18.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -32,7 +32,7 @@ BWC_VERSION:
   - "8.12.2"
   - "8.13.4"
   - "8.14.3"
-  - "8.15.6"
+  - "8.15.5"
   - "8.16.2"
   - "8.17.0"
   - "8.18.0"
diff --git a/.ci/snapshotBwcVersions b/.ci/snapshotBwcVersions
@@ -1,6 +1,5 @@
 BWC_VERSION:
   - "7.17.27"
-  - "8.15.6"
   - "8.16.2"
   - "8.17.0"
   - "8.18.0"
diff --git a/branches.json b/branches.json
@@ -13,9 +13,6 @@
     {
       "branch": "8.x"
     },
-    {
-      "branch": "8.15"
-    },
     {
       "branch": "7.17"
     }

diff --git a/...va/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java b/...va/org/elasticsearch/gradle/internal/conventions/precommit/FormattingPrecommitPlugin.java
@@ -17,8 +17,6 @@
 import org.gradle.api.Project;
 
 import java.io.File;
-import java.util.Arrays;
-import java.util.Map;
 
 /**
  * This plugin configures formatting for Java source using Spotless
@@ -66,8 +64,7 @@ public void apply(Project project) {
                 java.importOrderFile(new File(elasticsearchWorkspace, importOrderPath));
 
                 // Most formatting is done through the Eclipse formatter
-                java.eclipse().withP2Mirrors(Map.of("https://download.eclipse.org/", "https://mirror.umd.edu/eclipse/"))
-                    .configFile(new File(elasticsearchWorkspace, formatterConfigPath));
+                java.eclipse().configFile(new File(elasticsearchWorkspace, formatterConfigPath));
 
                 // Ensure blank lines are actually empty. Since formatters are applied in
                 // order, apply this one last, otherwise non-empty blank lines can creep

diff --git a/build.gradle b/build.gradle
@@ -290,7 +290,10 @@ allprojects {
     if (project.path.contains(":distribution:docker")) {
       enabled = false
     }
-
+    if (project.path.contains(":libs:cli")) {
+      // ensure we resolve p2 dependencies for the spotless eclipse formatter
+      dependsOn "spotlessJavaCheck"
+    }
   }
 
   plugins.withId('lifecycle-base') {

diff --git a/docs/changelog/117839.yaml b/docs/changelog/117839.yaml
@@ -0,0 +1,5 @@
+pr: 117839
+summary: Add match support for `semantic_text` fields
+area: "Search"
+type: enhancement
+issues: []
diff --git a/docs/changelog/118035.yaml b/docs/changelog/118035.yaml
@@ -0,0 +1,6 @@
+pr: 118035
+summary: Include hidden indices in `DeprecationInfoAction`
+area: Indices APIs
+type: bug
+issues:
+ - 118020
diff --git a/docs/changelog/118102.yaml b/docs/changelog/118102.yaml
@@ -0,0 +1,5 @@
+pr: 118102
+summary: "ESQL: Enterprise license enforcement for CCS"
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/118114.yaml b/docs/changelog/118114.yaml
@@ -0,0 +1,5 @@
+pr: 118114
+summary: Enable physical plan verification
+area: ES|QL
+type: enhancement
+issues: []
diff --git a/docs/changelog/118166.yaml b/docs/changelog/118166.yaml
@@ -0,0 +1,5 @@
+pr: 118166
+summary: Update minimum supported snapshot version for Machine Learning jobs to 8.3.0
+area: Machine Learning
+type: upgrade
+issues: []
diff --git a/docs/changelog/118173.yaml b/docs/changelog/118173.yaml
@@ -0,0 +1,5 @@
+pr: 118173
+summary: ES|QL categorize with multiple groupings
+area: Machine Learning
+type: feature
+issues: []
diff --git a/docs/changelog/118194.yaml b/docs/changelog/118194.yaml
@@ -0,0 +1,5 @@
+pr: 118194
+summary: Retry on `ClusterBlockException` on transform destination index
+area: Machine Learning
+type: enhancement
+issues: []
diff --git a/docs/changelog/118435.yaml b/docs/changelog/118435.yaml
@@ -0,0 +1,6 @@
+pr: 118435
+summary: '`_score` should not be a reserved attribute in ES|QL'
+area: ES|QL
+type: enhancement
+issues:
+ - 118460
diff --git a/docs/reference/connector/docs/connectors-release-notes.asciidoc b/docs/reference/connector/docs/connectors-release-notes.asciidoc
@@ -9,8 +9,76 @@
 Prior to version *8.16.0*, the connector release notes were published as part of the {enterprise-search-ref}/changelog.html[Enterprise Search documentation].
 ====
 
-*Release notes*:
+[discrete]
+[[es-connectors-release-notes-8-17-0]]
+=== 8.17.0
 
-* <<es-connectors-release-notes-8-16-0>>
+No notable changes in this release.
 
-include::release-notes/connectors-release-notes-8.16.0.asciidoc[]
+[discrete]
+[[es-connectors-release-notes-8-16-1]]
+=== 8.16.1
+
+[discrete]
+[[es-connectors-release-notes-8-16-1-bug-fixes]]
+==== Bug fixes
+
+* Fixed a bug in the Outlook Connector where having deactivated users could cause the sync to fail.
+See https://github.com/elastic/connectors/pull/2967[*PR 2967*].
+* Fixed a bug where the Confluence connector was not downloading some blog post documents due to unexpected response format.
+See https://github.com/elastic/connectors/pull/2984[*PR 2984*].
+
+[discrete]
+[[es-connectors-release-notes-8-16-0]]
+=== 8.16.0
+
+[discrete]
+[[es-connectors-release-notes-deprecation-notice]]
+==== Deprecation notices
+
+* *Direct index access for connectors and sync jobs*
++
+IMPORTANT: Directly accessing connector and sync job state through `.elastic-connectors*` indices is deprecated, and will be disallowed entirely in a future release.
+
+* Instead, the Elasticsearch Connector APIs should be used. Connectors framework code now uses the <<connector-apis,Connector APIs>> by default.
+See https://github.com/elastic/connectors/pull/2884[*PR 2902*].
+
+* *Docker `enterprise-search` namespace deprecation*
++
+IMPORTANT: The `enterprise-search` Docker namespace is deprecated and will be discontinued in a future release. 
++
+Starting in `8.16.0`, Docker images are being transitioned to the new `integrations` namespace, which will become the sole location for future releases. This affects the https://github.com/elastic/connectors[Elastic Connectors] and https://github.com/elastic/data-extraction-service[Elastic Data Extraction Service].
++
+During this transition period, images are published to both namespaces:
++
+** *Example*:
++
+Deprecated namespace::
+`docker.elastic.co/enterprise-search/elastic-connectors:v8.16.0`
++
+New namespace::
+`docker.elastic.co/integrations/elastic-connectors:v8.16.0`
++
+Users should migrate to the new `integrations` namespace as soon as possible to ensure continued access to future releases.
+
+[discrete]
+[[es-connectors-release-notes-8-16-0-enhancements]]
+==== Enhancements
+
+* Docker images now use Chainguard's Wolfi base image (`docker.elastic.co/wolfi/jdk:openjdk-11-dev`), replacing the previous `ubuntu:focal` base.
+
+* The Sharepoint Online connector now works with the `Sites.Selected` permission instead of the broader permission `Sites.Read.All`.
+See https://github.com/elastic/connectors/pull/2762[*PR 2762*].
+
+* Starting in 8.16.0, connectors will start using proper SEMVER, with `MAJOR.MINOR.PATCH`, which aligns with Elasticsearch/Kibana versions. This drops the previous `.BUILD` suffix, which we used to release connectors between Elastic stack releases. Going forward, these inter-stack-release releases will be suffixed instead with `+<timestamp>`, aligning with Elastic Agent and conforming to SEMVER. 
+See https://github.com/elastic/connectors/pull/2749[*PR 2749*].
+
+* Connector logs now use UTC timestamps, instead of machine-local timestamps. This only impacts logging output.
+See https://github.com/elastic/connectors/pull/2695[*PR 2695*].
+
+[discrete]
+[[es-connectors-release-notes-8-16-0-bug-fixes]]
+==== Bug fixes
+
+* The Dropbox connector now fetches the files from team shared folders.
+See https://github.com/elastic/connectors/pull/2718[*PR 2718*].
diff --git a/...reference/connector/docs/release-notes/connectors-release-notes-8.16.0.asciidoc b/...reference/connector/docs/release-notes/connectors-release-notes-8.16.0.asciidoc
diff --git a/docs/reference/docs/update.asciidoc b/docs/reference/docs/update.asciidoc
@@ -71,7 +71,7 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=refresh]
 include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=routing]
 
 `_source`::
-(Optional, list) Set to `false` to disable source retrieval (default: `true`).
+(Optional, list) Set to `true` to enable source retrieval (default: `false`).
 You can also specify a comma-separated list of the fields you want to retrieve.
 
 `_source_excludes`::

diff --git a/docs/reference/index-modules.asciidoc b/docs/reference/index-modules.asciidoc
@@ -113,7 +113,7 @@ Index mode supports the following values:
 
 `standard`::: Standard indexing with default settings.
 
-`tsds`::: _(data streams only)_ Index mode optimized for storage of metrics. For more information, see <<tsds-index-settings>>.
+`time_series`::: _(data streams only)_ Index mode optimized for storage of metrics. For more information, see <<tsds-index-settings>>.
 
 `logsdb`::: _(data streams only)_ Index mode optimized for <<logs-data-stream,logs>>.
 

diff --git a/docs/reference/mapping/params/index-prefixes.asciidoc b/docs/reference/mapping/params/index-prefixes.asciidoc
@@ -54,3 +54,30 @@ PUT my-index-000001
   }
 }
 --------------------------------
+
+`index_prefixes` parameter instructs {ES} to create a subfield "._index_prefix". This
+field will be used to do fast prefix queries. When doing highlighting, add "._index_prefix"
+subfield to the `matched_fields` parameter to highlight the main field based on the
+found matches of the prefix field, like in the request below:
+
+[source,console]
+--------------------------------
+GET my-index-000001/_search
+{
+  "query": {
+    "prefix": {
+      "full_name": {
+        "value": "ki"
+      }
+    }
+  },
+  "highlight": {
+    "fields": {
+      "full_name": {
+        "matched_fields": ["full_name._index_prefix"]
+      }
+    }
+  }
+}
+--------------------------------
+// TEST[continued]
diff --git a/docs/reference/mapping/types/search-as-you-type.asciidoc b/docs/reference/mapping/types/search-as-you-type.asciidoc
@@ -97,11 +97,21 @@ GET my-index-000001/_search
         "my_field._3gram"
       ]
     }
+  },
+  "highlight": {
+    "fields": {
+      "my_field": {
+        "matched_fields": ["my_field._index_prefix"] <1>
+      }
+    }
   }
 }
 --------------------------------------------------
 // TEST[continued]
 
+<1> Adding "my_field._index_prefix" to the `matched_fields` allows to highlight
+    "my_field" also based on matches from "my_field._index_prefix" field.
+
 [source,console-result]
 --------------------------------------------------
 {
@@ -126,6 +136,11 @@ GET my-index-000001/_search
         "_score" : 0.8630463,
         "_source" : {
           "my_field" : "quick brown fox jump lazy dog"
+        },
+        "highlight": {
+          "my_field": [
+            "quick <em>brown fox jump lazy</em> dog"
+          ]
         }
       }
     ]
-Original file line number
+Diff line change
@@ Expand Up / @@ -13,9 +13,6 @@ @@
         {
           "branch": "8.x"
         },
-        {
-          "branch": "8.15"
-        },
         {
           "branch": "7.17"
         }
@@ Expand Down @@