Merge branch 'opensearch-project:main' into main-opentelemetry

CHANGELOG.md

@@ -9,6 +9,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add getter for path field in NestedQueryBuilder ([#4636](https://github.com/opensearch-project/OpenSearch/pull/4636))
 - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
 - Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854))
+- Add support for ignoring missing Javadoc on generated code using annotation ([#7604](https://github.com/opensearch-project/OpenSearch/pull/7604))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
@@ -97,6 +98,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Support to clear filecache using clear indices cache API ([#7498](https://github.com/opensearch-project/OpenSearch/pull/7498))
 - Create NamedRoute to map extension routes to a shortened name ([#6870](https://github.com/opensearch-project/OpenSearch/pull/6870))
 - Added @dbwiddis as on OpenSearch maintainer ([#7665](https://github.com/opensearch-project/OpenSearch/pull/7665))
+- [Extensions] Add ExtensionAwarePlugin extension point to add custom settings for extensions ([#7526](https://github.com/opensearch-project/OpenSearch/pull/7526))
+- Add new cluster setting to set default index replication type ([#7420](https://github.com/opensearch-project/OpenSearch/pull/7420))
 
 ### Dependencies
 - Bump `com.netflix.nebula:gradle-info-plugin` from 12.0.0 to 12.1.3 (#7564)
@@ -124,13 +127,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `io.projectreactor.netty:reactor-netty-core` from 1.1.5 to 1.1.7 (#7657)
 - Bump `org.apache.maven:maven-model` from 3.9.1 to 3.9.2 (#7655)
 - Bump `com.google.api:gax` from 2.17.0 to 2.27.0 (#7697)
+- Bump `io.projectreactor.netty:reactor-netty` from 1.1.4 to 1.1.7 ([#7725](https://github.com/opensearch-project/OpenSearch/pull/7725))
+- Bump `io.projectreactor.netty:reactor-netty-http` from 1.1.4 to 1.1.7 ([#7725](https://github.com/opensearch-project/OpenSearch/pull/7725))
 
 ### Changed
 - Enable `./gradlew build` on MacOS by disabling bcw tests ([#7303](https://github.com/opensearch-project/OpenSearch/pull/7303))
 - Moved concurrent-search from sandbox plugin to server module behind feature flag ([#7203](https://github.com/opensearch-project/OpenSearch/pull/7203))
 - Allow access to indices cache clear APIs for read only indexes ([#7303](https://github.com/opensearch-project/OpenSearch/pull/7303))
 - Changed concurrent-search threadpool type to be resizable and support task resource tracking ([#7502](https://github.com/opensearch-project/OpenSearch/pull/7502))
 - Default search preference to _primary for searchable snapshot indices ([#7628](https://github.com/opensearch-project/OpenSearch/pull/7628))
+- [Segment Replication] Remove codec name string match check for checkpoints ([#7741](https://github.com/opensearch-project/OpenSearch/pull/7741))
 
 ### Deprecated
 
@@ -140,6 +146,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add more index blocks check for resize APIs ([#6774](https://github.com/opensearch-project/OpenSearch/pull/6774))
 - Replaces ZipInputStream with ZipFile to fix Zip Slip vulnerability ([#7230](https://github.com/opensearch-project/OpenSearch/pull/7230))
 - Add missing validation/parsing of SearchBackpressureMode of SearchBackpressureSettings ([#7541](https://github.com/opensearch-project/OpenSearch/pull/7541))
+- [Search Pipelines] Better exception handling in search pipelines ([#7735](https://github.com/opensearch-project/OpenSearch/pull/7735))
+- Fix input validation in segments and delete pit request ([#6645](https://github.com/opensearch-project/OpenSearch/pull/6645))
 
 ### Security
 

buildSrc/version.properties

@@ -23,6 +23,7 @@ kotlin            = 1.7.10
 antlr4            = 4.11.1
 guava             = 31.1-jre
 protobuf          = 3.22.3
+jboss_annotation  = 1.0.2.Final
 
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.5.0

distribution/src/config/opensearch.yml

@@ -102,10 +102,9 @@ ${path.logs}
 #
 # ---------------------------------- Experimental Features -----------------------------------
 #
-# Gates the visibility of the index setting that allows changing of replication type.
-# Once the feature is ready for production release, this feature flag can be removed.
+# Gates the visibility of the experimental segment replication features until they are production ready.
 #
-#opensearch.experimental.feature.replication_type.enabled: false
+#opensearch.experimental.feature.segment_replication_experimental.enabled: false
 #
 #
 # Gates the visibility of the index setting that allows persisting data to remote store along with local disk.

doc-tools/missing-doclet/build.gradle

@@ -5,10 +5,6 @@ plugins {
 group 'org.opensearch'
 version '1.0.0-SNAPSHOT'
 
-repositories {
-    mavenCentral()
-}
-
 tasks.withType(JavaCompile) {
   options.compilerArgs += ["--release", targetCompatibility.toString()]
   options.encoding = "UTF-8"

doc-tools/missing-doclet/src/main/java/org/opensearch/missingdoclet/MissingDoclet.java

@@ -13,6 +13,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -291,6 +292,22 @@ private void checkComment(Element element) {
         if (ignored.contains(element.toString())) {
             return;
         }
+        // Ignore classes annotated with @Generated and all enclosed elements in them.
+        if (isGenerated(element)) {
+            return;
+        }
+        Element enclosing = element.getEnclosingElement();
+        if (enclosing != null && isGenerated(enclosing)) {
+            return;
+        }
+        // If a package contains only generated classes, ignore the package as well.
+        if (element.getKind() == ElementKind.PACKAGE) {
+            List<? extends Element> enclosedElements = element.getEnclosedElements();
+            Optional<?> elm = enclosedElements.stream().findFirst().filter(e -> ((e.getKind() != ElementKind.CLASS) || !isGenerated(e)));
+            if (elm.isEmpty()) {
+                return;
+            }
+        }
         var tree = docTrees.getDocCommentTree(element);
         if (tree == null || tree.getFirstSentence().isEmpty()) {
             // Check for methods that override other stuff and perhaps inherit their Javadocs.
@@ -313,6 +330,17 @@ private void checkComment(Element element) {
         }
     }
 
+    // Ignore classes annotated with @Generated and all enclosed elements in them.
+    private boolean isGenerated(Element element) {
+        return element
+            .getAnnotationMirrors()
+            .stream()
+            .anyMatch(m -> m
+                .getAnnotationType()
+                .toString() /* ClassSymbol.toString() returns class name */
+                .equalsIgnoreCase("javax.annotation.Generated"));
+    }
+
     private boolean hasInheritedJavadocs(Element element) {
         boolean hasOverrides = element.getAnnotationMirrors().stream()
             .anyMatch(ann -> ann.getAnnotationType().toString().equals(Override.class.getName()));

modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/30_filter_query.yml

@@ -122,3 +122,27 @@ teardown:
         index: test
         body: { }
   - match: { hits.total.value: 2 }
+---
+"Test invalid inline query":
+  - do:
+      catch: bad_request
+      search:
+        index: test
+        body: {
+          search_pipeline: {
+            "request_processors": [
+              {
+                "filter_query": {
+                  "query": {
+                    "woozlewuzzle": {
+                      "field": "foo"
+                    }
+                  }
+                }
+              }
+            ]
+          }
+        }
+  - match: { status: 400 }
+  - match: { error.type: "parsing_exception"}
+  - match: { error.reason: "unknown query [woozlewuzzle]"}

modules/search-pipeline-common/src/yamlRestTest/resources/rest-api-spec/test/search_pipeline/50_script_processor.yml

@@ -83,10 +83,6 @@ teardown:
           "profile": true
         }
   - length: { hits.hits: 2 }
-  - match: { _shards.total: 1 }
-  - match: { hits.total.value: 1 }
-  - match: { hits.hits.0._score: 1.0 }
-  - match: { hits.hits.1._score: 1.0 }
   - is_false: hits.hits.0._explanation
   - is_false: hits.hits.1._explanation
   - is_false: hits.hits.0._seq_no

plugins/repository-azure/build.gradle

@@ -58,9 +58,9 @@ dependencies {
   api 'com.azure:azure-storage-blob:12.21.1'
   api 'org.reactivestreams:reactive-streams:1.0.4'
   api 'io.projectreactor:reactor-core:3.5.1'
-  api 'io.projectreactor.netty:reactor-netty:1.1.4'
+  api 'io.projectreactor.netty:reactor-netty:1.1.7'
   api 'io.projectreactor.netty:reactor-netty-core:1.1.7'
-  api 'io.projectreactor.netty:reactor-netty-http:1.1.4'
+  api 'io.projectreactor.netty:reactor-netty-http:1.1.7'
   api "org.slf4j:slf4j-api:${versions.slf4j}"
   api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"

plugins/repository-azure/licenses/reactor-netty-1.1.7.jar.sha1

@@ -0,0 +1 @@
+c16497c29f96ea7b1db538cb0ddde55d9be173fe

plugins/repository-azure/licenses/reactor-netty-http-1.1.7.jar.sha1

@@ -0,0 +1 @@
+39d7c0a13afa471b426a30bcf82664496ad34723

qa/rolling-upgrade/src/test/java/org/opensearch/upgrades/IndexingIT.java

@@ -245,6 +245,7 @@ public void testIndexing() throws IOException, ParseException {
      *
      * @throws Exception
      */
+    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/7679")
     public void testIndexingWithSegRep() throws Exception {
         final String indexName = "test-index-segrep";
         final int shardCount = 3;

server/build.gradle

@@ -165,6 +165,7 @@ dependencies {
 
   // protobuf
   api "com.google.protobuf:protobuf-java:${versions.protobuf}"
+  implementation "org.jboss.spec.javax.annotation:jboss-annotations-api_1.2_spec:${versions.jboss_annotation}"
 
   testImplementation(project(":test:framework")) {
     // tests use the locally compiled version of server
@@ -193,6 +194,7 @@ tasks.named("forbiddenPatterns").configure {
     exclude '**/*.dic'
     exclude '**/*.binary'
     exclude '**/*.st'
+    exclude '**/*.meta'
 }
 
 tasks.named("testingConventions").configure {
@@ -238,6 +240,16 @@ protobuf {
   protoc {
     artifact = "com.google.protobuf:protoc:${versions.protobuf}"
   }
+
+  generateProtoTasks {
+    all().each { task ->
+      task.builtins {
+        java {
+          option "annotate_code"
+        }
+      }
+    }
+  }
 }
 
 tasks.named("processResources").configure {
@@ -424,29 +436,6 @@ tasks.named("dependencyLicenses").configure {
     }
 }
 
-tasks.named("missingJavadoc").configure {
-  /*
-   * Generated code doesn't follow javadocs formats.
-   * Unfortunately the missingJavadoc task doesnt take *.
-   * TODO: Add support to missingJavadoc task to ignore all generated source code
-   *  https://github.com/opensearch-project/OpenSearch/issues/7264
-   */
-  dependsOn("generateProto")
-  javadocMissingIgnore = [
-    "org.opensearch.extensions.proto.ExtensionIdentityProto",
-    "org.opensearch.extensions.proto.ExtensionIdentityProto.ExtensionIdentityOrBuilder",
-    "org.opensearch.extensions.proto.RegisterRestActionsProto",
-    "org.opensearch.extensions.proto.RegisterRestActionsProto.RegisterRestActionsOrBuilder",
-    "org.opensearch.extensions.proto.ExtensionRequestProto",
-    "org.opensearch.extensions.proto.ExtensionRequestProto.ExtensionRequestOrBuilder",
-    "org.opensearch.extensions.proto.RegisterTransportActionsProto",
-    "org.opensearch.extensions.proto.RegisterTransportActionsProto.RegisterTransportActionsOrBuilder",
-    "org.opensearch.extensions.proto.ExtensionTransportMessageProto",
-    "org.opensearch.extensions.proto.ExtensionTransportMessageProto.ExtensionTransportMessageOrBuilder",
-    "org.opensearch.extensions.proto"
-  ]
-}
-
 tasks.named("filepermissions").configure {
   mustRunAfter("generateProto")
 }

server/licenses/jboss-annotations-api_1.2_spec-1.0.2.Final.jar.sha1

@@ -0,0 +1 @@
+0d6b20c0e95c5b38f313cc2ab1a55560cedabe1f