Merge remote-tracking branch 'upstream/main' into add_shrink_parameter

opensearch-project · Nov 23, 2022 · 8b7a9d6 · 8b7a9d6
2 parents 07a153b + bdb8efe
commit 8b7a9d6
Show file tree

Hide file tree

Showing 84 changed files with 1,884 additions and 1,192 deletions.
diff --git a/.ci/bwcVersions b/.ci/bwcVersions
@@ -1,47 +1,4 @@
 BWC_VERSION:
-  - "7.0.0"
-  - "7.0.1"
-  - "7.1.0"
-  - "7.1.1"
-  - "7.2.0"
-  - "7.2.1"
-  - "7.3.0"
-  - "7.3.1"
-  - "7.3.2"
-  - "7.4.0"
-  - "7.4.1"
-  - "7.4.2"
-  - "7.5.0"
-  - "7.5.1"
-  - "7.5.2"
-  - "7.6.0"
-  - "7.6.1"
-  - "7.6.2"
-  - "7.7.0"
-  - "7.7.1"
-  - "7.8.0"
-  - "7.8.1"
-  - "7.9.0"
-  - "7.9.1"
-  - "7.9.2"
-  - "7.9.3"
-  - "7.10.0"
-  - "7.10.1"
-  - "7.10.2"
-  - "1.0.0"
-  - "1.1.0"
-  - "1.2.0"
-  - "1.2.1"
-  - "1.2.2"
-  - "1.2.3"
-  - "1.2.4"
-  - "1.2.5"
-  - "1.3.0"
-  - "1.3.1"
-  - "1.3.2"
-  - "1.3.3"
-  - "1.3.4"
-  - "1.3.5"
   - "2.0.0"
   - "2.0.1"
   - "2.0.2"
@@ -53,4 +10,5 @@ BWC_VERSION:
   - "2.3.0"
   - "2.3.1"
   - "2.4.0"
+  - "2.4.1"
   - "2.5.0"
diff --git a/.github/workflows/gradle-check.yml b/.github/workflows/gradle-check.yml
@@ -81,9 +81,16 @@ jobs:
         if: ${{ github.event_name == 'pull_request_target' && env.result != 'SUCCESS' }}
         run: |
             TEST_FAILURES=`curl -s "${{ env.workflow_url }}/testReport/api/json?tree=suites\[cases\[status,className,name\]\]" | jq -r '.. | objects | select(.status=="FAILED",.status=="REGRESSION") | (.className + "." +  .name)' | uniq -c | sort -n -r | head -n 10`
-            echo "test_failures<<EOF" >> $GITHUB_ENV
-            echo "$TEST_FAILURES" >> $GITHUB_ENV
-            echo "EOF" >> $GITHUB_ENV
+            if [[ "$TEST_FAILURES" != "" ]]
+            then
+              echo "test_failures<<EOF" >> $GITHUB_ENV
+              echo "" >> $GITHUB_ENV
+              echo "* **TEST FAILURES:**" >> $GITHUB_ENV
+              echo '```' >> $GITHUB_ENV
+              echo "$TEST_FAILURES" >> $GITHUB_ENV
+              echo '```' >> $GITHUB_ENV
+              echo "EOF" >> $GITHUB_ENV
+            fi
 
       - name: Create Comment Flaky
         if: ${{ github.event_name == 'pull_request_target' && success() && env.result != 'SUCCESS' }}
@@ -92,16 +99,10 @@ jobs:
           issue-number: ${{ env.pr_number }}
           body: |
             ### Gradle Check (Jenkins) Run Completed with:
-            * **RESULT:** ${{ env.result }} :grey_exclamation:
-            * **FLAKY TEST FAILURES:**
-            The following tests failed but succeeded upon retry:
-            ```
-            ${{ env.test_failures }}
-            ```
+            * **RESULT:** ${{ env.result }} :grey_exclamation: ${{ env.test_failures }}
             * **URL:** ${{ env.workflow_url }}
             * **CommitID:** ${{ env.pr_from_sha }}
-            Please examine the workflow log, locate, and copy-paste the failure below, then iterate to green.
-            Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?
+            Please review all [flaky tests](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) that succeeded after retry and create an issue if one does not already exist to track the flaky failure.
 
       - name: Create Comment Failure
         if: ${{ github.event_name == 'pull_request_target' && failure() }}
@@ -110,12 +111,8 @@ jobs:
           issue-number: ${{ env.pr_number }}
           body: |
               ### Gradle Check (Jenkins) Run Completed with:
-              * **RESULT:** ${{ env.result }} :x:
-              * **FAILURES:**
-              ```
-              ${{ env.test_failures }}
-              ```
+              * **RESULT:** ${{ env.result }} :x: ${{ env.test_failures }}
               * **URL:** ${{ env.workflow_url }}
               * **CommitID:** ${{ env.pr_from_sha }}
-              Please examine the workflow log, locate, and copy-paste the failure below, then iterate to green.
+              Please examine the workflow log, locate, and copy-paste the failure(s) below, then iterate to green.
               Is the failure [a flaky test](https://github.com/opensearch-project/OpenSearch/blob/main/DEVELOPER_GUIDE.md#flaky-tests) unrelated to your change?
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -35,6 +35,16 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bumps `com.diffplug.spotless` from 6.10.0 to 6.11.0 ([#4547](https://github.com/opensearch-project/OpenSearch/pull/4547))
 - Bumps `reactor-core` from 3.4.18 to 3.4.23 ([#4548](https://github.com/opensearch-project/OpenSearch/pull/4548))
 - Bumps `jempbox` from 1.8.16 to 1.8.17 ([#4550](https://github.com/opensearch-project/OpenSearch/pull/4550))
+- Bumps `commons-compress` from 1.21 to 1.22
+- Bumps `jcodings` from 1.0.57 to 1.0.58 ([#5233](https://github.com/opensearch-project/OpenSearch/pull/5233))
+- Bumps `google-http-client-jackson2` from 1.35.0 to 1.42.3 ([#5234](https://github.com/opensearch-project/OpenSearch/pull/5234))
+- Bumps `maxmind-db` from 2.0.0 to 2.1.0 ([#5236](https://github.com/opensearch-project/OpenSearch/pull/5236))
+- Bumps `azure-core` from 1.33.0 to 1.34.0 ([#5235](https://github.com/opensearch-project/OpenSearch/pull/5235))
+- Bumps `azure-core-http-netty` from 1.12.4 to 1.12.7 ([#5235](https://github.com/opensearch-project/OpenSearch/pull/5235))
+- Bumps `spock-core` from from 2.1-groovy-3.0 to 2.3-groovy-3.0 ([#5315](https://github.com/opensearch-project/OpenSearch/pull/5315))
+- Bumps `json-schema-validator` from 1.0.69 to 1.0.73 ([#5316](https://github.com/opensearch-project/OpenSearch/pull/5316))
+- Bumps `proto-google-common-protos` from 2.8.0 to 2.10.0 ([#5318](https://github.com/opensearch-project/OpenSearch/pull/5318))
+- Bumps `protobuf-java` from 3.21.7 to 3.21.9 ([#5319](https://github.com/opensearch-project/OpenSearch/pull/5319))
 - Update Apache Lucene to 9.5.0-snapshot-a4ef70f ([#4979](https://github.com/opensearch-project/OpenSearch/pull/4979))
 - Update to Gradle 7.6 and JDK-19 ([#4973](https://github.com/opensearch-project/OpenSearch/pull/4973))
 
@@ -43,6 +53,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Relax visibility of the HTTP_CHANNEL_KEY and HTTP_SERVER_CHANNEL_KEY to make it possible for the plugins to access associated Netty4HttpChannel / Netty4HttpServerChannel instance ([#4638](https://github.com/opensearch-project/OpenSearch/pull/4638))
 - Use ReplicationFailedException instead of OpensearchException in ReplicationTarget ([#4725](https://github.com/opensearch-project/OpenSearch/pull/4725))
 - Migrate client transports to Apache HttpClient / Core 5.x ([#4459](https://github.com/opensearch-project/OpenSearch/pull/4459))
+- Support remote translog transfer for request level durability([#4480](https://github.com/opensearch-project/OpenSearch/pull/4480))
 
 ### Deprecated
 
@@ -64,7 +75,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Fixed
 - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))
 - Fixed compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944))
-- Add jvm option to allow security manager ([#5194](https://github.com/opensearch-project/OpenSearch/pull/5194))
+- Reject bulk requests with invalid actions ([#5299](https://github.com/opensearch-project/OpenSearch/issues/5299))
+
 ### Security
 
 ## [Unreleased 2.x]
@@ -76,6 +88,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bumps `commons-compress` from 1.21 to 1.22 ([#5104](https://github.com/opensearch-project/OpenSearch/pull/5104))
 - Bump `opencensus-contrib-http-util` from 0.18.0 to 0.31.1 ([#3633](https://github.com/opensearch-project/OpenSearch/pull/3633))
 - Bump `geoip2` from 3.0.1 to 3.0.2 ([#5103](https://github.com/opensearch-project/OpenSearch/pull/5103))
+- Bump gradle-extra-configurations-plugin from 7.0.0 to 8.0.0 ([#4808](https://github.com/opensearch-project/OpenSearch/pull/4808))
 ### Changed
 ### Deprecated
 ### Removed

diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -105,7 +105,7 @@ dependencies {
   api 'commons-codec:commons-codec:1.15'
   api 'org.apache.commons:commons-compress:1.21'
   api 'org.apache.ant:ant:1.10.12'
-  api 'com.netflix.nebula:gradle-extra-configurations-plugin:7.0.0'
+  api 'com.netflix.nebula:gradle-extra-configurations-plugin:8.0.0'
   api 'com.netflix.nebula:nebula-publishing-plugin:4.6.0'
   api 'com.netflix.nebula:gradle-info-plugin:11.3.3'
   api 'org.apache.rat:apache-rat:0.13'
@@ -118,7 +118,7 @@ dependencies {
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.15.2'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
   api 'org.apache.maven:maven-model:3.6.2'
-  api 'com.networknt:json-schema-validator:1.0.69'
+  api 'com.networknt:json-schema-validator:1.0.73'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"
 
   testFixturesApi "junit:junit:${props.getProperty('junit')}"
@@ -127,7 +127,7 @@ dependencies {
   testFixturesApi gradleTestKit()
   testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.33.2'
   testImplementation "org.mockito:mockito-core:${props.getProperty('mockito')}"
-  integTestImplementation('org.spockframework:spock-core:2.1-groovy-3.0') {
+  integTestImplementation('org.spockframework:spock-core:2.3-groovy-3.0') {
     exclude module: "groovy"
   }
 }

diff --git a/buildSrc/reaper/src/main/java/org/elasticsearch/gradle/reaper/package-info.java b/buildSrc/reaper/src/main/java/org/elasticsearch/gradle/reaper/package-info.java
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/BwcVersions.java b/buildSrc/src/main/java/org/opensearch/gradle/BwcVersions.java
@@ -105,7 +105,7 @@ public class BwcVersions {
     private final Map<Integer, List<Version>> groupByMajor;
     private final Map<Version, UnreleasedVersionInfo> unreleased;
 
-    public class UnreleasedVersionInfo {
+    public static class UnreleasedVersionInfo {
         public final Version version;
         public final String branch;
         public final String gradleProjectPath;
@@ -149,13 +149,7 @@ public BwcVersions(SortedSet<Version> allVersions, Version currentVersionPropert
 
         groupByMajor = allVersions.stream()
             // We only care about the last 2 majors when it comes to BWC.
-            // It might take us time to remove the older ones from versionLines, so we allow them to exist.
-            // Adjust the major number since OpenSearch 1.x is released after predecessor version 7.x
-            .filter(
-                version -> (version.getMajor() == 1 ? 7 : version.getMajor()) > (currentVersion.getMajor() == 1
-                    ? 7
-                    : currentVersion.getMajor()) - 2
-            )
+            .filter(version -> version.getMajor() > currentVersion.getMajor() - 2)
             .collect(Collectors.groupingBy(Version::getMajor, Collectors.toList()));
 
         assertCurrentVersionMatchesParsed(currentVersionProperty);
@@ -174,9 +168,7 @@ public BwcVersions(SortedSet<Version> allVersions, Version currentVersionPropert
 
     private void assertNoOlderThanTwoMajors() {
         Set<Integer> majors = groupByMajor.keySet();
-        // until OpenSearch 3.0 we will need to carry three major support
-        // (1, 7, 6) && (2, 1, 7) since OpenSearch 1.0 === Legacy 7.x
-        int numSupportedMajors = (currentVersion.getMajor() < 3) ? 3 : 2;
+        int numSupportedMajors = 2;
         if (majors.size() != numSupportedMajors && currentVersion.getMinor() != 0 && currentVersion.getRevision() != 0) {
             throw new IllegalStateException("Expected exactly 2 majors in parsed versions but found: " + majors);
         }
@@ -207,7 +199,7 @@ public void forPreviousUnreleased(Consumer<UnreleasedVersionInfo> consumer) {
             .map(version -> new UnreleasedVersionInfo(version, getBranchFor(version), getGradleProjectPathFor(version)))
             .collect(Collectors.toList());
 
-        collect.forEach(uvi -> consumer.accept(uvi));
+        collect.forEach(consumer);
     }
 
     private String getGradleProjectPathFor(Version version) {
@@ -271,18 +263,9 @@ public List<Version> getUnreleased() {
         // The current version is being worked, is always unreleased
         unreleased.add(currentVersion);
 
-        // No unreleased versions for 1.0.0
-        // todo remove this hack
-        if (currentVersion.equals(Version.fromString("1.0.0"))) {
-            return unmodifiableList(unreleased);
-        }
-
         // the tip of the previous major is unreleased for sure, be it a minor or a bugfix
         if (currentVersion.getMajor() != 1) {
-            final Version latestOfPreviousMajor = getLatestVersionByKey(
-                this.groupByMajor,
-                currentVersion.getMajor() == 1 ? 7 : currentVersion.getMajor() - 1
-            );
+            final Version latestOfPreviousMajor = getLatestVersionByKey(this.groupByMajor, currentVersion.getMajor() - 1);
             unreleased.add(latestOfPreviousMajor);
             if (latestOfPreviousMajor.getRevision() == 0) {
                 // if the previous major is a x.y.0 release, then the tip of the minor before that (y-1) is also unreleased
@@ -311,7 +294,7 @@ public List<Version> getUnreleased() {
             }
         }
 
-        return unmodifiableList(unreleased.stream().sorted().distinct().collect(Collectors.toList()));
+        return unreleased.stream().sorted().distinct().collect(Collectors.toUnmodifiableList());
     }
 
     private Version getLatestInMinor(int major, int minor) {
@@ -342,7 +325,7 @@ private Map<Integer, List<Version>> getReleasedMajorGroupedByMinor() {
 
     public void compareToAuthoritative(List<Version> authoritativeReleasedVersions) {
         Set<Version> notReallyReleased = new HashSet<>(getReleased());
-        notReallyReleased.removeAll(authoritativeReleasedVersions);
+        authoritativeReleasedVersions.forEach(notReallyReleased::remove);
         if (notReallyReleased.isEmpty() == false) {
             throw new IllegalStateException(
                 "out-of-date released versions"
@@ -370,32 +353,21 @@ private List<Version> getReleased() {
             .stream()
             .flatMap(Collection::stream)
             .filter(each -> unreleased.contains(each) == false)
-            // this is to make sure we only consider OpenSearch versions
-            // TODO remove this filter once legacy ES versions are no longer supported
-            .filter(v -> v.onOrAfter("1.0.0"))
             .collect(Collectors.toList());
     }
 
     public List<Version> getIndexCompatible() {
         int currentMajor = currentVersion.getMajor();
         int prevMajor = getPreviousMajor(currentMajor);
-        List<Version> result = Stream.concat(groupByMajor.get(prevMajor).stream(), groupByMajor.get(currentMajor).stream())
+        return Stream.concat(groupByMajor.get(prevMajor).stream(), groupByMajor.get(currentMajor).stream())
             .filter(version -> version.equals(currentVersion) == false)
-            .collect(Collectors.toList());
-        if (currentMajor == 1) {
-            // add 6.x compatible for OpenSearch 1.0.0
-            return unmodifiableList(Stream.concat(groupByMajor.get(prevMajor - 1).stream(), result.stream()).collect(Collectors.toList()));
-        } else if (currentMajor == 2) {
-            // add 7.x compatible for OpenSearch 2.0.0
-            return unmodifiableList(Stream.concat(groupByMajor.get(7).stream(), result.stream()).collect(Collectors.toList()));
-        }
-        return unmodifiableList(result);
+            .collect(Collectors.toUnmodifiableList());
     }
 
     public List<Version> getWireCompatible() {
         List<Version> wireCompat = new ArrayList<>();
         int currentMajor = currentVersion.getMajor();
-        int lastMajor = currentMajor == 1 ? 6 : currentMajor == 2 ? 7 : currentMajor - 1;
+        int lastMajor = currentMajor - 1;
         List<Version> lastMajorList = groupByMajor.get(lastMajor);
         if (lastMajorList == null) {
             throw new IllegalStateException("Expected to find a list of versions for version: " + lastMajor);
@@ -405,20 +377,6 @@ public List<Version> getWireCompatible() {
             wireCompat.add(lastMajorList.get(i));
         }
 
-        // if current is OpenSearch 1.0.0 add all of the 7.x line:
-        if (currentMajor == 1) {
-            List<Version> previousMajor = groupByMajor.get(7);
-            for (Version v : previousMajor) {
-                wireCompat.add(v);
-            }
-        } else if (currentMajor == 2) {
-            // add all of the 1.x line:
-            List<Version> previousMajor = groupByMajor.get(1);
-            for (Version v : previousMajor) {
-                wireCompat.add(v);
-            }
-        }
-
         wireCompat.addAll(groupByMajor.get(currentMajor));
         wireCompat.remove(currentVersion);
         wireCompat.sort(Version::compareTo);
@@ -438,7 +396,7 @@ public List<Version> getUnreleasedWireCompatible() {
     }
 
     private int getPreviousMajor(int currentMajor) {
-        return currentMajor == 1 ? 7 : currentMajor - 1;
+        return currentMajor - 1;
     }
 
 }