Merge remote-tracking branch 'origin/main' into fix_tra

Author: atris

CHANGELOG.md

@@ -21,6 +21,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Adding ScriptedAvg class to painless spi to allowlist usage from plugins ([#19006](https://github.com/opensearch-project/OpenSearch/pull/19006))
 - Replace centos:8 with almalinux:8 since centos docker images are deprecated ([#19154](https://github.com/opensearch-project/OpenSearch/pull/19154))
 - Add CompletionStage variants to IndicesAdminClient as an alternative to ActionListener ([#19161](https://github.com/opensearch-project/OpenSearch/pull/19161))
+- Remove cap on Java version used by forbidden APIs ([#19163](https://github.com/opensearch-project/OpenSearch/pull/19163))
 
 ### Fixed
 - Fix unnecessary refreshes on update preparation failures ([#15261](https://github.com/opensearch-project/OpenSearch/issues/15261))
@@ -30,6 +31,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix flaky tests in CloseIndexIT by addressing cluster state synchronization issues ([#18878](https://github.com/opensearch-project/OpenSearch/issues/18878))
 - [Tiered Caching] Handle  query execution exception ([#19000](https://github.com/opensearch-project/OpenSearch/issues/19000))
 - Grant access to testclusters dir for tests ([#19085](https://github.com/opensearch-project/OpenSearch/issues/19085))
+- Fix assertion error when collapsing search results with concurrent segment search enabled ([#19053](https://github.com/opensearch-project/OpenSearch/pull/19053))
 - Fix skip_unavailable setting changing to default during node drop issue ([#18766](https://github.com/opensearch-project/OpenSearch/pull/18766))
 
 ### Dependencies

buildSrc/build.gradle

@@ -114,7 +114,7 @@ dependencies {
   api 'com.gradleup.shadow:shadow-gradle-plugin:8.3.5'
   api 'org.jdom:jdom2:2.0.6.1'
   api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
-  api 'de.thetaphi:forbiddenapis:3.8'
+  api 'de.thetaphi:forbiddenapis:3.9'
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.17.12'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
   api 'org.apache.maven:maven-model:3.9.6'

buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenApisPrecommitPlugin.java

@@ -40,7 +40,6 @@
 import org.opensearch.gradle.ExportOpenSearchBuildResourcesTask;
 import org.opensearch.gradle.info.BuildParams;
 import org.opensearch.gradle.util.GradleUtils;
-import org.gradle.api.JavaVersion;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.plugins.ExtraPropertiesExtension;
@@ -53,6 +52,7 @@
 import java.util.Arrays;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 public class ForbiddenApisPrecommitPlugin extends PrecommitPlugin {
     @Override
@@ -89,10 +89,6 @@ public TaskProvider<? extends Task> createTask(Project project) {
             t.setClasspath(project.files(sourceSet.getRuntimeClasspath()).plus(sourceSet.getCompileClasspath()));
 
             t.setTargetCompatibility(BuildParams.getRuntimeJavaVersion().getMajorVersion());
-            if (BuildParams.getRuntimeJavaVersion().compareTo(JavaVersion.VERSION_14) > 0) {
-                // TODO: forbidden apis does not yet support java 15, rethink using runtime version
-                t.setTargetCompatibility(JavaVersion.VERSION_14.getMajorVersion());
-            }
             t.setBundledSignatures(new HashSet<>(Arrays.asList("jdk-unsafe", "jdk-deprecated", "jdk-non-portable", "jdk-system-out")));
             t.setSignaturesFiles(
                 project.files(
@@ -140,6 +136,18 @@ public Void call(Object... names) {
                     return null;
                 }
             });
+            // Use of the deprecated security manager APIs are pervasive so set them to warn
+            // globally for all projects. Replacements for (most of) these APIs are available
+            // so usages can move to the non-deprecated variants to avoid the warnings.
+            t.setSignaturesWithSeverityWarn(
+                Set.of(
+                    "java.security.AccessController",
+                    "java.security.AccessControlContext",
+                    "java.lang.System#getSecurityManager()",
+                    "java.lang.SecurityManager",
+                    "java.security.Policy"
+                )
+            );
         });
         TaskProvider<Task> forbiddenApis = project.getTasks().named("forbiddenApis");
         forbiddenApis.configure(t -> t.setGroup(""));

distribution/tools/plugin-cli/src/main/java/org/opensearch/tools/cli/plugin/InstallPluginCommand.java

@@ -399,7 +399,7 @@ private String getMavenUrl(Terminal terminal, String[] coordinates, String platf
     @SuppressForbidden(reason = "Make HEAD request using URLConnection.connect()")
     boolean urlExists(Terminal terminal, String urlString) throws IOException {
         terminal.println(VERBOSE, "Checking if url exists: " + urlString);
-        URL url = new URL(urlString);
+        URL url = URI.create(urlString).toURL();
         assert "https".equals(url.getProtocol()) : "Use of https protocol is required";
         HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
         urlConnection.addRequestProperty("User-Agent", "opensearch-plugin-installer");
@@ -427,7 +427,7 @@ private List<String> checkMisspelledPlugin(String pluginId) {
     @SuppressForbidden(reason = "We use getInputStream to download plugins")
     Path downloadZip(Terminal terminal, String urlString, Path tmpDir, boolean isBatch) throws IOException {
         terminal.println(VERBOSE, "Retrieving zip from " + urlString);
-        URL url = new URL(urlString);
+        URL url = URI.create(urlString).toURL();
         Path zip = Files.createTempFile(tmpDir, null, ".zip");
         URLConnection urlConnection = url.openConnection();
         urlConnection.addRequestProperty("User-Agent", "opensearch-plugin-installer");
@@ -684,7 +684,7 @@ InputStream getPublicKey() {
      */
     // pkg private for tests
     URL openUrl(String urlString) throws IOException {
-        URL checksumUrl = new URL(urlString);
+        URL checksumUrl = URI.create(urlString).toURL();
         HttpURLConnection connection = (HttpURLConnection) checksumUrl.openConnection();
         if (connection.getResponseCode() == 404) {
             return null;

distribution/tools/plugin-cli/src/test/java/org/opensearch/tools/cli/plugin/InstallPluginCommandTests.java

@@ -526,7 +526,7 @@ public void testSpaceInUrl() throws Exception {
         Path pluginDir = createPluginDir(temp);
         String pluginZip = createPluginUrl("fake", pluginDir);
         Path pluginZipWithSpaces = createTempFile("foo bar", ".zip");
-        try (InputStream in = FileSystemUtils.openFileURLStream(new URL(pluginZip))) {
+        try (InputStream in = FileSystemUtils.openFileURLStream(URI.create(pluginZip).toURL())) {
             Files.copy(in, pluginZipWithSpaces, StandardCopyOption.REPLACE_EXISTING);
         }
         installPlugin(pluginZipWithSpaces.toUri().toURL().toString(), env.v1());
@@ -536,8 +536,8 @@ public void testSpaceInUrl() throws Exception {
     public void testMalformedUrlNotMaven() throws Exception {
         Tuple<Path, Environment> env = createEnv(fs, temp);
         // has two colons, so it appears similar to maven coordinates
-        MalformedURLException e = expectThrows(MalformedURLException.class, () -> installPlugin("://host:1234", env.v1()));
-        assertTrue(e.getMessage(), e.getMessage().contains("no protocol"));
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> installPlugin("://host:1234", env.v1()));
+        assertThat(e.getMessage(), startsWith("Expected scheme name"));
     }
 
     public void testFileNotMaven() throws Exception {

libs/core/src/test/java/org/opensearch/core/util/FileSystemUtilsTests.java

@@ -40,6 +40,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.ByteBuffer;
@@ -132,21 +133,21 @@ public void testIsHidden() {
     }
 
     public void testOpenFileURLStream() throws IOException {
-        URL urlWithWrongProtocol = new URL("http://www.google.com");
+        URL urlWithWrongProtocol = URI.create("http://www.google.com").toURL();
         try (InputStream is = FileSystemUtils.openFileURLStream(urlWithWrongProtocol)) {
             fail("Should throw IllegalArgumentException due to invalid protocol");
         } catch (IllegalArgumentException e) {
             assertEquals("Invalid protocol [http], must be [file] or [jar]", e.getMessage());
         }
 
-        URL urlWithHost = new URL("file", "localhost", txtFile.toString());
+        URL urlWithHost = URI.create("file://localhost/" + txtFile.toString()).toURL();
         try (InputStream is = FileSystemUtils.openFileURLStream(urlWithHost)) {
             fail("Should throw IllegalArgumentException due to host");
         } catch (IllegalArgumentException e) {
             assertEquals("URL cannot have host. Found: [localhost]", e.getMessage());
         }
 
-        URL urlWithPort = new URL("file", "", 80, txtFile.toString());
+        URL urlWithPort = URI.create("file://:80/" + txtFile.toString()).toURL();
         try (InputStream is = FileSystemUtils.openFileURLStream(urlWithPort)) {
             fail("Should throw IllegalArgumentException due to port");
         } catch (IllegalArgumentException e) {

libs/ssl-config/src/test/java/org/opensearch/common/ssl/DefaultJdkTrustConfigTests.java

@@ -77,12 +77,12 @@ private void assertStandardIssuers(X509ExtendedTrustManager trustManager) {
     private void assertHasTrustedIssuer(X509ExtendedTrustManager trustManager, String name) {
         final String lowerName = name.toLowerCase(Locale.ROOT);
         final Optional<X509Certificate> ca = Stream.of(trustManager.getAcceptedIssuers())
-            .filter(cert -> cert.getSubjectDN().getName().toLowerCase(Locale.ROOT).contains(lowerName))
+            .filter(cert -> cert.getSubjectX500Principal().getName().toLowerCase(Locale.ROOT).contains(lowerName))
             .findAny();
         if (ca.isPresent() == false) {
             logger.info("Failed to find issuer [{}] in trust manager, but did find ...", lowerName);
             for (X509Certificate cert : trustManager.getAcceptedIssuers()) {
-                logger.info(" - {}", cert.getSubjectDN().getName().replaceFirst("^\\w+=([^,]+),.*", "$1"));
+                logger.info(" - {}", cert.getSubjectX500Principal().getName().replaceFirst("^\\w+=([^,]+),.*", "$1"));
             }
             Assert.fail("Cannot find trusted issuer with name [" + name + "].");
         }

libs/ssl-config/src/test/java/org/opensearch/common/ssl/PemKeyConfigTests.java

@@ -154,8 +154,8 @@ private void assertCertificateAndKey(PemKeyConfig keyConfig, String expectedDN)
         assertThat(chain, notNullValue());
         assertThat(chain, arrayWithSize(1));
         final X509Certificate certificate = chain[0];
-        assertThat(certificate.getIssuerDN().getName(), is("CN=Test CA 1"));
-        assertThat(certificate.getSubjectDN().getName(), is(expectedDN));
+        assertThat(certificate.getIssuerX500Principal().getName(), is("CN=Test CA 1"));
+        assertThat(certificate.getSubjectX500Principal().getName(), is(expectedDN));
         assertThat(certificate.getSubjectAlternativeNames(), iterableWithSize(2));
         assertThat(
             certificate.getSubjectAlternativeNames(),

libs/ssl-config/src/test/java/org/opensearch/common/ssl/PemTrustConfigTests.java

@@ -146,7 +146,7 @@ private void assertCertificateChain(PemTrustConfig trustConfig, String... caName
         final X509ExtendedTrustManager trustManager = trustConfig.createTrustManager();
         final X509Certificate[] issuers = trustManager.getAcceptedIssuers();
         final Set<String> issuerNames = Stream.of(issuers)
-            .map(X509Certificate::getSubjectDN)
+            .map(X509Certificate::getSubjectX500Principal)
             .map(Principal::getName)
             .collect(Collectors.toSet());
 

libs/ssl-config/src/test/java/org/opensearch/common/ssl/StoreKeyConfigTests.java

@@ -183,8 +183,8 @@ private void assertKeysLoaded(StoreKeyConfig keyConfig, String... names) throws
             assertThat(chain, notNullValue());
             assertThat(chain, arrayWithSize(1));
             final X509Certificate certificate = chain[0];
-            assertThat(certificate.getIssuerDN().getName(), is("CN=Test CA 1"));
-            assertThat(certificate.getSubjectDN().getName(), is("CN=" + name));
+            assertThat(certificate.getIssuerX500Principal().getName(), is("CN=Test CA 1"));
+            assertThat(certificate.getSubjectX500Principal().getName(), is("CN=" + name));
             assertThat(certificate.getSubjectAlternativeNames(), iterableWithSize(2));
             assertThat(
                 certificate.getSubjectAlternativeNames(),