Merge branch 'main' into retrievers

.buildkite/pipelines/intake.yml

@@ -48,7 +48,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.18", "8.12.1", "8.13.0"]
+            BWC_VERSION: ["7.17.19", "8.12.2", "8.13.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -1121,6 +1121,22 @@ steps:
         env:
           BWC_VERSION: 7.17.18
 
+      - label: "{{matrix.image}} / 7.17.19 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.19
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.19
+
       - label: "{{matrix.image}} / 8.0.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.0
         timeout_in_minutes: 300
@@ -1841,6 +1857,22 @@ steps:
         env:
           BWC_VERSION: 8.12.1
 
+      - label: "{{matrix.image}} / 8.12.2 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.12.2
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2004
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.12.2
+
       - label: "{{matrix.image}} / 8.13.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.13.0
         timeout_in_minutes: 300

.buildkite/pipelines/periodic.yml

@@ -682,6 +682,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 7.17.18
+      - label: 7.17.19 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.19#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 7.17.19
       - label: 8.0.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.0.0#bwcTest
         timeout_in_minutes: 300
@@ -1132,6 +1142,16 @@ steps:
           buildDirectory: /dev/shm/bk
         env:
           BWC_VERSION: 8.12.1
+      - label: 8.12.2 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.12.2#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 8.12.2
       - label: 8.13.0 / bwc
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.13.0#bwcTest
         timeout_in_minutes: 300

.buildkite/pull-requests.json

@@ -11,7 +11,7 @@
       "set_commit_status": false,
       "build_on_commit": true,
       "build_on_comment": true,
-      "trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*(buildkite\\s*)?test\\s+this(\\s+please)?)",
+      "trigger_comment_regex": "(run\\W+elasticsearch-ci.+)|(^\\s*((buildkite|@elastic(search)?machine)\\s*)?test\\s+this(\\s+please)?)",
       "cancel_intermediate_builds": true,
       "cancel_intermediate_builds_on_comment": false
     },

.buildkite/scripts/pull-request/__snapshots__/pipeline.test.ts.snap

@@ -201,3 +201,111 @@ exports[`generatePipelines should generate correct pipeline when using a trigger
   },
 ]
 `;
+
+exports[`generatePipelines should generate correct pipelines with a non-docs change and @elasticmachine 1`] = `
+[
+  {
+    "name": "bwc-snapshots",
+    "pipeline": {
+      "steps": [
+        {
+          "group": "bwc-snapshots",
+          "steps": [
+            {
+              "agents": {
+                "buildDirectory": "/dev/shm/bk",
+                "image": "family/elasticsearch-ubuntu-2004",
+                "machineType": "custom-32-98304",
+                "provider": "gcp",
+              },
+              "command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest",
+              "env": {
+                "BWC_VERSION": "{{matrix.BWC_VERSION}}",
+              },
+              "label": "{{matrix.BWC_VERSION}} / bwc-snapshots",
+              "matrix": {
+                "setup": {
+                  "BWC_VERSION": [
+                    "7.17.14",
+                    "8.10.3",
+                    "8.11.0",
+                  ],
+                },
+              },
+              "timeout_in_minutes": 300,
+            },
+          ],
+        },
+      ],
+    },
+  },
+  {
+    "name": "using-defaults",
+    "pipeline": {
+      "env": {
+        "CUSTOM_ENV_VAR": "value",
+      },
+      "steps": [
+        {
+          "command": "echo 'hello world'",
+          "label": "test-step",
+        },
+      ],
+    },
+  },
+]
+`;
+
+exports[`generatePipelines should generate correct pipelines with a non-docs change and @elasticsearchmachine 1`] = `
+[
+  {
+    "name": "bwc-snapshots",
+    "pipeline": {
+      "steps": [
+        {
+          "group": "bwc-snapshots",
+          "steps": [
+            {
+              "agents": {
+                "buildDirectory": "/dev/shm/bk",
+                "image": "family/elasticsearch-ubuntu-2004",
+                "machineType": "custom-32-98304",
+                "provider": "gcp",
+              },
+              "command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest",
+              "env": {
+                "BWC_VERSION": "{{matrix.BWC_VERSION}}",
+              },
+              "label": "{{matrix.BWC_VERSION}} / bwc-snapshots",
+              "matrix": {
+                "setup": {
+                  "BWC_VERSION": [
+                    "7.17.14",
+                    "8.10.3",
+                    "8.11.0",
+                  ],
+                },
+              },
+              "timeout_in_minutes": 300,
+            },
+          ],
+        },
+      ],
+    },
+  },
+  {
+    "name": "using-defaults",
+    "pipeline": {
+      "env": {
+        "CUSTOM_ENV_VAR": "value",
+      },
+      "steps": [
+        {
+          "command": "echo 'hello world'",
+          "label": "test-step",
+        },
+      ],
+    },
+  },
+]
+`;

.buildkite/scripts/pull-request/pipeline.test.ts

@@ -13,11 +13,11 @@ describe("generatePipelines", () => {
   });
 
   // Helper for testing pipeline generations that should be the same when using the overall ci trigger comment "buildkite test this"
-  const testWithTriggerCheck = (directory: string, changedFiles?: string[]) => {
+  const testWithTriggerCheck = (directory: string, changedFiles?: string[], comment = "buildkite test this") => {
     const pipelines = generatePipelines(directory, changedFiles);
     expect(pipelines).toMatchSnapshot();
 
-    process.env["GITHUB_PR_TRIGGER_COMMENT"] = "buildkite test this";
+    process.env["GITHUB_PR_TRIGGER_COMMENT"] = comment;
     const pipelinesWithTriggerComment = generatePipelines(directory, changedFiles);
     expect(pipelinesWithTriggerComment).toEqual(pipelines);
   };
@@ -42,4 +42,20 @@ describe("generatePipelines", () => {
     const pipelines = generatePipelines(`${import.meta.dir}/mocks/pipelines`, ["build.gradle"]);
     expect(pipelines).toMatchSnapshot();
   });
+
+  test("should generate correct pipelines with a non-docs change and @elasticmachine", () => {
+    testWithTriggerCheck(
+      `${import.meta.dir}/mocks/pipelines`,
+      ["build.gradle", "docs/README.asciidoc"],
+      "@elasticmachine test this please"
+    );
+  });
+
+  test("should generate correct pipelines with a non-docs change and @elasticsearchmachine", () => {
+    testWithTriggerCheck(
+      `${import.meta.dir}/mocks/pipelines`,
+      ["build.gradle", "docs/README.asciidoc"],
+      "@elasticsearchmachine test this please"
+    );
+  });
 });

.buildkite/scripts/pull-request/pipeline.ts

@@ -148,7 +148,9 @@ export const generatePipelines = (
   // However, if we're using the overall CI trigger "[buildkite] test this [please]", we should use the regular filters above
   if (
     process.env["GITHUB_PR_TRIGGER_COMMENT"] &&
-    !process.env["GITHUB_PR_TRIGGER_COMMENT"].match(/^\s*(buildkite\s*)?test\s+this(\s+please)?/i)
+    !process.env["GITHUB_PR_TRIGGER_COMMENT"].match(
+      /^\s*((@elastic(search)?machine|buildkite)\s*)?test\s+this(\s+please)?/i
+    )
   ) {
     filters = [triggerCommentCheck];
   }

.ci/bwcVersions

@@ -67,6 +67,7 @@ BWC_VERSION:
   - "7.17.16"
   - "7.17.17"
   - "7.17.18"
+  - "7.17.19"
   - "8.0.0"
   - "8.0.1"
   - "8.1.0"
@@ -112,4 +113,5 @@ BWC_VERSION:
   - "8.11.4"
   - "8.12.0"
   - "8.12.1"
+  - "8.12.2"
   - "8.13.0"

.ci/snapshotBwcVersions

@@ -1,4 +1,4 @@
 BWC_VERSION:
-  - "7.17.18"
-  - "8.12.1"
+  - "7.17.19"
+  - "8.12.2"
   - "8.13.0"

benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java

@@ -26,7 +26,6 @@
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
-import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.script.ScriptContext;
@@ -72,8 +71,7 @@ public static MapperService create(String mappings) {
                 public <T> T compile(Script script, ScriptContext<T> scriptContext) {
                     throw new UnsupportedOperationException();
                 }
-            },
-            () -> DocumentParsingObserver.EMPTY_INSTANCE
+            }
         );
 
         try {

benchmarks/src/main/java/org/elasticsearch/benchmark/search/QueryParserHelperBenchmark.java

@@ -40,7 +40,6 @@
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
 import org.elasticsearch.indices.breaker.NoneCircuitBreakerService;
-import org.elasticsearch.plugins.internal.DocumentParsingObserver;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.script.ScriptContext;
@@ -187,8 +186,7 @@ protected final MapperService createMapperService(String mappings) {
                 public <T> T compile(Script script, ScriptContext<T> scriptContext) {
                     throw new UnsupportedOperationException();
                 }
-            },
-            () -> DocumentParsingObserver.EMPTY_INSTANCE
+            }
         );
 
         try {

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/util/Util.java

@@ -119,29 +119,29 @@ public static SourceSetContainer getJavaSourceSets(Project project) {
         return project.getExtensions().getByType(JavaPluginExtension.class).getSourceSets();
     }
 
-        public static File locateElasticsearchWorkspace(Gradle gradle) {
-            if(gradle.getRootProject().getName().startsWith("build-tools")) {
-                File buildToolsParent = gradle.getRootProject().getRootDir().getParentFile();
-                if(versionFileExists(buildToolsParent)) {
-                    return buildToolsParent;
-                }
+    public static File locateElasticsearchWorkspace(Gradle gradle) {
+        if (gradle.getRootProject().getName().startsWith("build-tools")) {
+            File buildToolsParent = gradle.getRootProject().getRootDir().getParentFile();
+            if (versionFileExists(buildToolsParent)) {
                 return buildToolsParent;
             }
-            if (gradle.getParent() == null) {
-                // See if any of these included builds is the Elasticsearch gradle
-                for (IncludedBuild includedBuild : gradle.getIncludedBuilds()) {
-                    if (versionFileExists(includedBuild.getProjectDir())) {
-                        return includedBuild.getProjectDir();
-                    }
+            return buildToolsParent;
+        }
+        if (gradle.getParent() == null) {
+            // See if any of these included builds is the Elasticsearch gradle
+            for (IncludedBuild includedBuild : gradle.getIncludedBuilds()) {
+                if (versionFileExists(includedBuild.getProjectDir())) {
+                    return includedBuild.getProjectDir();
                 }
-
-                // Otherwise assume this gradle is the root elasticsearch workspace
-                return gradle.getRootProject().getRootDir();
-            } else {
-                // We're an included build, so keep looking
-                return locateElasticsearchWorkspace(gradle.getParent());
             }
+
+            // Otherwise assume this gradle is the root elasticsearch workspace
+            return gradle.getRootProject().getRootDir();
+        } else {
+            // We're an included build, so keep looking
+            return locateElasticsearchWorkspace(gradle.getParent());
         }
+    }
 
     private static boolean versionFileExists(File rootDir) {
         return new File(rootDir, "build-tools-internal/version.properties").exists();

build-tools-internal/build.gradle

@@ -143,6 +143,10 @@ gradlePlugin {
       id = 'elasticsearch.mrjar'
       implementationClass = 'org.elasticsearch.gradle.internal.MrjarPlugin'
     }
+    embeddedProvider {
+      id = 'elasticsearch.embedded-providers'
+      implementationClass = 'org.elasticsearch.gradle.internal.EmbeddedProviderPlugin'
+    }
     releaseTools {
       id = 'elasticsearch.release-tools'
       implementationClass = 'org.elasticsearch.gradle.internal.release.ReleaseToolsPlugin'