[Feature/extensions] Read from extensions.yml (#3381)

* Fixed git errors Signed-off-by: Ryan Bogan <rbogan@amazon.com> * Added ExtensionsSettings file and corresponding method in ExtensionsOrchestrator to read from extensions.yml Signed-off-by: Ryan Bogan <rbogan@amazon.com> * Removed unnecessary files Signed-off-by: Ryan Bogan <rbogan@amazon.com> * Fixed lint errors Signed-off-by: Ryan Bogan <rbogan@amazon.com> * Added more asserts to tests and removed unnecessary dependencies Signed-off-by: Ryan Bogan <rbogan@amazon.com> * Fixed gradle check errors Signed-off-by: Ryan Bogan <rbogan@amazon.com> * Removed unused import Signed-off-by: Ryan Bogan <rbogan@amazon.com>
opensearch-project · May 18, 2022 · 3b58b05 · 3b58b05
1 parent 84a61bc
commit 3b58b05
Show file tree

Hide file tree

Showing 36 changed files with 212 additions and 153 deletions.
diff --git a/modules/ingest-geoip/build.gradle b/modules/ingest-geoip/build.gradle
@@ -41,8 +41,6 @@ opensearchplugin {
 dependencies {
   api('com.maxmind.geoip2:geoip2:3.0.1')
   // geoip2 dependencies:
-  api("com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}")
-  api("com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}")
   api('com.maxmind.db:maxmind-db:2.0.0')
 
   testImplementation 'org.elasticsearch:geolite2-databases:20191119'

diff --git a/modules/ingest-geoip/licenses/jackson-annotations-2.13.2.jar.sha1 b/modules/ingest-geoip/licenses/jackson-annotations-2.13.2.jar.sha1
diff --git a/modules/ingest-geoip/licenses/jackson-databind-2.13.2.2.jar.sha1 b/modules/ingest-geoip/licenses/jackson-databind-2.13.2.2.jar.sha1
diff --git a/modules/ingest-geoip/licenses/jackson-databind-LICENSE b/modules/ingest-geoip/licenses/jackson-databind-LICENSE
diff --git a/modules/ingest-geoip/licenses/jackson-databind-NOTICE b/modules/ingest-geoip/licenses/jackson-databind-NOTICE
diff --git a/plugins/discovery-ec2/build.gradle b/plugins/discovery-ec2/build.gradle
@@ -50,8 +50,6 @@ dependencies {
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
-  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
 }
 
 restResources {

diff --git a/plugins/discovery-ec2/licenses/jackson-LICENSE b/plugins/discovery-ec2/licenses/jackson-LICENSE
diff --git a/plugins/discovery-ec2/licenses/jackson-NOTICE b/plugins/discovery-ec2/licenses/jackson-NOTICE
diff --git a/plugins/discovery-ec2/licenses/jackson-annotations-2.13.2.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-annotations-2.13.2.jar.sha1
diff --git a/plugins/discovery-ec2/licenses/jackson-databind-2.13.2.2.jar.sha1 b/plugins/discovery-ec2/licenses/jackson-databind-2.13.2.2.jar.sha1
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
@@ -61,8 +61,6 @@ dependencies {
   api 'io.projectreactor.netty:reactor-netty-core:1.0.16'
   api 'io.projectreactor.netty:reactor-netty-http:1.0.18'
   api "org.slf4j:slf4j-api:${versions.slf4j}"
-  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
   api "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:${versions.jackson}"
   api "com.fasterxml.jackson.dataformat:jackson-dataformat-xml:${versions.jackson}"
   api "com.fasterxml.jackson.module:jackson-module-jaxb-annotations:${versions.jackson}"

diff --git a/plugins/repository-azure/licenses/jackson-LICENSE b/plugins/repository-azure/licenses/jackson-LICENSE
diff --git a/...ools/upgrade-cli/licenses/jackson-LICENSE → ...sitory-azure/licenses/jackson-LICENSE.txt b/...ools/upgrade-cli/licenses/jackson-LICENSE → ...sitory-azure/licenses/jackson-LICENSE.txt
diff --git a/plugins/repository-azure/licenses/jackson-NOTICE b/plugins/repository-azure/licenses/jackson-NOTICE
diff --git a/...tools/upgrade-cli/licenses/jackson-NOTICE → ...ository-azure/licenses/jackson-NOTICE.txt b/...tools/upgrade-cli/licenses/jackson-NOTICE → ...ository-azure/licenses/jackson-NOTICE.txt
diff --git a/plugins/repository-azure/licenses/jackson-annotations-2.13.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-annotations-2.13.2.jar.sha1
diff --git a/plugins/repository-azure/licenses/jackson-databind-2.13.2.2.jar.sha1 b/plugins/repository-azure/licenses/jackson-databind-2.13.2.2.jar.sha1
diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
@@ -64,7 +64,6 @@ dependencies {
   api 'org.apache.htrace:htrace-core4:4.2.0-incubating'
   api "org.apache.logging.log4j:log4j-core:${versions.log4j}"
   api 'org.apache.avro:avro:1.10.2'
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
   api 'com.google.code.gson:gson:2.9.0'
   runtimeOnly 'com.google.guava:guava:31.1-jre'
   api 'com.google.protobuf:protobuf-java:3.20.1'

diff --git a/plugins/repository-hdfs/licenses/jackson-databind-2.13.2.2.jar.sha1 b/plugins/repository-hdfs/licenses/jackson-databind-2.13.2.2.jar.sha1
diff --git a/plugins/repository-s3/build.gradle b/plugins/repository-s3/build.gradle
@@ -57,9 +57,6 @@ dependencies {
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
-  api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
-  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
   api "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
   api "joda-time:joda-time:${versions.joda}"
 

diff --git a/plugins/repository-s3/licenses/jackson-LICENSE b/plugins/repository-s3/licenses/jackson-LICENSE
diff --git a/plugins/repository-s3/licenses/jackson-NOTICE b/plugins/repository-s3/licenses/jackson-NOTICE
diff --git a/plugins/repository-s3/licenses/jackson-annotations-2.13.2.jar.sha1 b/plugins/repository-s3/licenses/jackson-annotations-2.13.2.jar.sha1
diff --git a/plugins/repository-s3/licenses/jackson-databind-2.13.2.2.jar.sha1 b/plugins/repository-s3/licenses/jackson-databind-2.13.2.2.jar.sha1
diff --git a/server/build.gradle b/server/build.gradle
@@ -134,6 +134,9 @@ dependencies {
   // jna
   api "net.java.dev.jna:jna:${versions.jna}"
 
+  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
+  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+
   testImplementation(project(":test:framework")) {
     // tests use the locally compiled version of server
     exclude group: 'org.opensearch', module: 'server'
@@ -223,31 +226,12 @@ tasks.named("processResources").configure {
 
 tasks.named("thirdPartyAudit").configure {
     ignoreMissingClasses(
-            // from com.fasterxml.jackson.dataformat.yaml.YAMLMapper (jackson-dataformat-yaml)
-            'com.fasterxml.jackson.databind.ObjectMapper',
 
             // from log4j
             'com.conversantmedia.util.concurrent.SpinPolicy',
-            'com.fasterxml.jackson.annotation.JsonInclude$Include',
-            'com.fasterxml.jackson.databind.DeserializationContext',
-            'com.fasterxml.jackson.databind.DeserializationFeature',
-            'com.fasterxml.jackson.databind.JsonMappingException',
-            'com.fasterxml.jackson.databind.JsonNode',
-            'com.fasterxml.jackson.databind.Module$SetupContext',
-            'com.fasterxml.jackson.databind.ObjectReader',
-            'com.fasterxml.jackson.databind.ObjectWriter',
-            'com.fasterxml.jackson.databind.SerializerProvider',
-            'com.fasterxml.jackson.databind.deser.std.StdDeserializer',
-            'com.fasterxml.jackson.databind.deser.std.StdScalarDeserializer',
-            'com.fasterxml.jackson.databind.module.SimpleModule',
-            'com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter',
-            'com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider',
-            'com.fasterxml.jackson.databind.ser.std.StdScalarSerializer',
-            'com.fasterxml.jackson.databind.ser.std.StdSerializer',
             'com.fasterxml.jackson.dataformat.xml.JacksonXmlModule',
             'com.fasterxml.jackson.dataformat.xml.XmlMapper',
             'com.fasterxml.jackson.dataformat.xml.util.DefaultXmlPrettyPrinter',
-            'com.fasterxml.jackson.databind.node.ObjectNode',
             'org.fusesource.jansi.Ansi',
             'org.fusesource.jansi.AnsiRenderer$Code',
             'com.lmax.disruptor.EventFactory',
@@ -307,9 +291,6 @@ tasks.named("thirdPartyAudit").configure {
             'org.noggit.JSONParser',
 
             // from lucene-spatial
-            'com.fasterxml.jackson.databind.JsonSerializer',
-            'com.fasterxml.jackson.databind.JsonDeserializer',
-            'com.fasterxml.jackson.databind.node.ArrayNode',
             'com.google.common.geometry.S2Cell',
             'com.google.common.geometry.S2CellId',
             'com.google.common.geometry.S2Projections',

diff --git a/...enses/jackson-annotations-2.13.2.jar.sha1 → ...enses/jackson-annotations-2.13.2.jar.sha1 b/...enses/jackson-annotations-2.13.2.jar.sha1 → ...enses/jackson-annotations-2.13.2.jar.sha1
diff --git a/...eoip/licenses/jackson-annotations-LICENSE → .../licenses/jackson-annotations-LICENSE.txt b/...eoip/licenses/jackson-annotations-LICENSE → .../licenses/jackson-annotations-LICENSE.txt
diff --git a/...geoip/licenses/jackson-annotations-NOTICE → ...r/licenses/jackson-annotations-NOTICE.txt b/...geoip/licenses/jackson-annotations-NOTICE → ...r/licenses/jackson-annotations-NOTICE.txt
diff --git a/...censes/jackson-databind-2.13.2.2.jar.sha1 → ...censes/jackson-databind-2.13.2.2.jar.sha1 b/...censes/jackson-databind-2.13.2.2.jar.sha1 → ...censes/jackson-databind-2.13.2.2.jar.sha1
diff --git a/...dfs/licenses/jackson-databind-LICENSE.txt → server/licenses/jackson-databind-LICENSE.txt b/...dfs/licenses/jackson-databind-LICENSE.txt → server/licenses/jackson-databind-LICENSE.txt
diff --git a/...hdfs/licenses/jackson-databind-NOTICE.txt → server/licenses/jackson-databind-NOTICE.txt b/...hdfs/licenses/jackson-databind-NOTICE.txt → server/licenses/jackson-databind-NOTICE.txt
diff --git a/server/src/main/java/org/opensearch/extensions/ExtensionsOrchestrator.java b/server/src/main/java/org/opensearch/extensions/ExtensionsOrchestrator.java
@@ -8,6 +8,9 @@
 
 package org.opensearch.extensions;
 
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
+
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
@@ -36,6 +39,7 @@
 import org.opensearch.transport.TransportService;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.InetAddress;
 import java.net.UnknownHostException;
 import java.nio.file.Path;
@@ -98,6 +102,32 @@ private void extensionsDiscovery() throws IOException {
         for (final Path plugin : PluginsService.findPluginDirs(extensionsPath)) {
             try {
                 PluginInfo pluginInfo = PluginInfo.readFromProperties(plugin);
+
+                // TODO: Add unit tests for commented section below
+
+                /**
+                try {
+                    List<Extension> extensions = readFromExtensionsYml(extensionsPath.toString()).getExtensions();
+                    for(Extension extension : extensions) {
+                        extensionsSet.add(
+                            new DiscoveryExtension(
+                                extension.getName(),
+                                "id",
+                                extension.getEphemeralId(),
+                                extension.getHostName(),
+                                extension.getHostAddress(),
+                                new TransportAddress(TransportAddress.META_ADDRESS, Integer.parseInt(extension.getPort())),
+                                null,
+                                Version.fromString(extension.getVersion()),
+                                pluginInfo
+                    )
+                );
+                    }
+                } catch (Exception e) {
+                    //ignore
+                }
+                */
+
                 /*
                  * TODO: Read from extensions.yml
                  * https://github.com/opensearch-project/OpenSearch/issues/3084
@@ -262,4 +292,12 @@ public String executor() {
             logger.error(e.toString());
         }
     }
+
+    public static ExtensionsSettings readFromExtensionsYml(String filePath) throws Exception {
+        ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory());
+        InputStream input = ExtensionsOrchestrator.class.getResourceAsStream(filePath);
+        ExtensionsSettings extensionSettings = objectMapper.readValue(input, ExtensionsSettings.class);
+        return extensionSettings;
+    }
+
 }