Merge first batch of feature/extensions into main

CHANGELOG.md

@@ -22,6 +22,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Test] Add IAE test for deprecated edgeNGram analyzer name ([#5040](https://github.com/opensearch-project/OpenSearch/pull/5040))
 - Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
 - Add feature flag for extensions ([#5211](https://github.com/opensearch-project/OpenSearch/pull/5211))
+- Merge first batch of feature/extensions into main ([#5347](https://github.com/opensearch-project/OpenSearch/pull/5347))
 
 ### Dependencies
 - Bumps `log4j-core` from 2.18.0 to 2.19.0

modules/ingest-geoip/build.gradle

@@ -41,8 +41,6 @@ opensearchplugin {
 dependencies {
   api('com.maxmind.geoip2:geoip2:3.0.2')
   // geoip2 dependencies:
-  api("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
-  api("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}")
   api('com.maxmind.db:maxmind-db:2.1.0')
 
   testImplementation 'org.elasticsearch:geolite2-databases:20191119'

plugins/discovery-ec2/build.gradle

@@ -46,8 +46,6 @@ dependencies {
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
-  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
 }
 
 restResources {

plugins/repository-azure/build.gradle

@@ -44,30 +44,28 @@ opensearchplugin {
 }
 
 dependencies {
-  api 'com.azure:azure-core:1.34.0'
+  api 'com.azure:azure-core:1.33.0'
   api 'com.azure:azure-storage-common:12.18.1'
-  api 'com.azure:azure-core-http-netty:1.12.7'
+  api 'com.azure:azure-core-http-netty:1.12.4'
   api "io.netty:netty-codec-dns:${versions.netty}"
   api "io.netty:netty-codec-socks:${versions.netty}"
   api "io.netty:netty-codec-http2:${versions.netty}"
   api "io.netty:netty-handler-proxy:${versions.netty}"
   api "io.netty:netty-resolver-dns:${versions.netty}"
   api "io.netty:netty-transport-native-unix-common:${versions.netty}"
   implementation project(':modules:transport-netty4')
-  api 'com.azure:azure-storage-blob:12.20.0'
+  api 'com.azure:azure-storage-blob:12.16.1'
   api 'org.reactivestreams:reactive-streams:1.0.4'
   api 'io.projectreactor:reactor-core:3.4.23'
   api 'io.projectreactor.netty:reactor-netty:1.0.18'
   api 'io.projectreactor.netty:reactor-netty-core:1.0.24'
   api 'io.projectreactor.netty:reactor-netty-http:1.0.24'
   api "org.slf4j:slf4j-api:${versions.slf4j}"
-  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
   api "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.jackson}"
   api "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${versions.jackson}"
   api "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:${versions.jackson}"
   api 'org.codehaus.woodstox:stax2-api:4.2.1'
-  implementation "com.fasterxml.woodstox:woodstox-core:${versions.woodstox}"
+  implementation 'com.fasterxml.woodstox:woodstox-core:6.2.8'
   runtimeOnly 'com.google.guava:guava:31.1-jre'
   api 'org.apache.commons:commons-lang3:3.12.0'
   testImplementation project(':test:fixtures:azure-fixture')

plugins/repository-azure/licenses/azure-core-1.33.0.jar.sha1

@@ -0,0 +1 @@
+93f105c2e923f0ab90521cc0e6e729b9c8304ad8

plugins/repository-azure/licenses/azure-core-http-netty-1.12.4.jar.sha1

@@ -0,0 +1 @@
+70dcc08887f2d70a8f812bf00d4fa10390fab3fd

plugins/repository-azure/licenses/azure-storage-blob-12.16.1.jar.sha1

@@ -0,0 +1 @@
+84054ca8a6660eb77910925d71f70330fd3d83aa

plugins/repository-azure/licenses/woodstox-core-6.2.8.jar.sha1

@@ -0,0 +1 @@
+670748292899c53b1963730d9eb7f8ab71314e90

plugins/repository-hdfs/build.gradle

@@ -66,7 +66,6 @@ dependencies {
   api 'org.apache.htrace:htrace-core4:4.2.0-incubating'
   api "org.apache.logging.log4j:log4j-core:${versions.log4j}"
   api 'org.apache.avro:avro:1.11.1'
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
   api 'com.google.code.gson:gson:2.10'
   runtimeOnly 'com.google.guava:guava:31.1-jre'
   api 'com.google.protobuf:protobuf-java:3.21.7'

plugins/repository-s3/build.gradle

@@ -54,9 +54,6 @@ dependencies {
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
-  api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
-  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   api "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
   api "joda-time:joda-time:${versions.joda}"
 

server/build.gradle

@@ -135,6 +135,9 @@ dependencies {
   // jna
   api "net.java.dev.jna:jna:${versions.jna}"
 
+  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
+  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+
   testImplementation(project(":test:framework")) {
     // tests use the locally compiled version of server
     exclude group: 'org.opensearch', module: 'server'
@@ -208,31 +211,12 @@ tasks.named("processResources").configure {
 
 tasks.named("thirdPartyAudit").configure {
     ignoreMissingClasses(
-            // from com.fasterxml.jackson.dataformat.yaml.YAMLMapper (jackson-dataformat-yaml)
-            'com.fasterxml.jackson.databind.ObjectMapper',
 
             // from log4j
             'com.conversantmedia.util.concurrent.SpinPolicy',
-            'com.fasterxml.jackson.annotation.JsonInclude$Include',
-            'com.fasterxml.jackson.databind.DeserializationContext',
-            'com.fasterxml.jackson.databind.DeserializationFeature',
-            'com.fasterxml.jackson.databind.JsonMappingException',
-            'com.fasterxml.jackson.databind.JsonNode',
-            'com.fasterxml.jackson.databind.Module$SetupContext',
-            'com.fasterxml.jackson.databind.ObjectReader',
-            'com.fasterxml.jackson.databind.ObjectWriter',
-            'com.fasterxml.jackson.databind.SerializerProvider',
-            'com.fasterxml.jackson.databind.deser.std.StdDeserializer',
-            'com.fasterxml.jackson.databind.deser.std.StdScalarDeserializer',
-            'com.fasterxml.jackson.databind.module.SimpleModule',
-            'com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter',
-            'com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider',
-            'com.fasterxml.jackson.databind.ser.std.StdScalarSerializer',
-            'com.fasterxml.jackson.databind.ser.std.StdSerializer',
             'com.fasterxml.jackson.dataformat.xml.JacksonXmlModule',
             'com.fasterxml.jackson.dataformat.xml.XmlMapper',
             'com.fasterxml.jackson.dataformat.xml.util.DefaultXmlPrettyPrinter',
-            'com.fasterxml.jackson.databind.node.ObjectNode',
             'org.fusesource.jansi.Ansi',
             'org.fusesource.jansi.AnsiRenderer$Code',
             'com.lmax.disruptor.EventFactory',
@@ -292,9 +276,6 @@ tasks.named("thirdPartyAudit").configure {
             'org.noggit.JSONParser',
 
             // from lucene-spatial
-            'com.fasterxml.jackson.databind.JsonSerializer',
-            'com.fasterxml.jackson.databind.JsonDeserializer',
-            'com.fasterxml.jackson.databind.node.ArrayNode',
             'com.google.common.geometry.S2Cell',
             'com.google.common.geometry.S2CellId',
             'com.google.common.geometry.S2Projections',

server/src/main/java/org/opensearch/action/admin/cluster/state/ClusterStateResponse.java

@@ -96,6 +96,11 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(waitForTimedOut);
     }
 
+    @Override
+    public String toString() {
+        return "ClusterStateResponse{" + "clusterState=" + clusterState + '}';
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;

server/src/main/java/org/opensearch/bootstrap/Security.java

@@ -36,6 +36,7 @@
 import org.opensearch.common.SuppressForbidden;
 import org.opensearch.common.io.PathUtils;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.env.Environment;
 import org.opensearch.http.HttpTransportSettings;
 import org.opensearch.plugins.PluginInfo;
@@ -316,6 +317,9 @@ static void addFilePermissions(Permissions policy, Environment environment) thro
         addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.libDir(), "read,readlink", false);
         addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.modulesDir(), "read,readlink", false);
         addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.pluginsDir(), "read,readlink", false);
+        if (FeatureFlags.isEnabled(FeatureFlags.EXTENSIONS)) {
+            addDirectoryPath(policy, Environment.PATH_HOME_SETTING.getKey(), environment.extensionDir(), "read,readlink", false);
+        }
         addDirectoryPath(policy, "path.conf'", environment.configDir(), "read,readlink", false);
         // read-write dirs
         addDirectoryPath(policy, "java.io.tmpdir", environment.tmpDir(), "read,readlink,write,delete", false);

server/src/main/java/org/opensearch/cluster/ClusterSettingsResponse.java

@@ -0,0 +1,60 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster;
+
+import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.io.stream.StreamInput;
+import org.opensearch.common.io.stream.StreamOutput;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.transport.TransportResponse;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * PluginSettings Response for Extensibility
+ *
+ * @opensearch.internal
+ */
+public class ClusterSettingsResponse extends TransportResponse {
+    private final Settings clusterSettings;
+
+    public ClusterSettingsResponse(ClusterService clusterService) {
+        this.clusterSettings = clusterService.getSettings();
+    }
+
+    public ClusterSettingsResponse(StreamInput in) throws IOException {
+        super(in);
+        this.clusterSettings = Settings.readSettingsFromStream(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        Settings.writeSettingsToStream(clusterSettings, out);
+    }
+
+    @Override
+    public String toString() {
+        return "ClusterSettingsResponse{" + "clusterSettings=" + clusterSettings + '}';
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ClusterSettingsResponse that = (ClusterSettingsResponse) o;
+        return Objects.equals(clusterSettings, that.clusterSettings);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(clusterSettings);
+    }
+
+}