[DO NOT MERGE] Backport some fixes from master to ozone-1.4

.github/workflows/ci.yml

@@ -94,10 +94,11 @@ jobs:
       - name: Cache for maven dependencies
         uses: actions/cache@v3
         with:
-          path: ~/.m2/repository
-          key: maven-repo-${{ hashFiles('**/pom.xml') }}-${{ matrix.java }}
+          path: |
+            ~/.m2/repository
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
-            maven-repo-${{ hashFiles('**/pom.xml') }}
             maven-repo-
       - name: Setup java
         uses: actions/setup-java@v3
@@ -129,12 +130,6 @@ jobs:
           path: |
             ~/.m2/repository/org/apache/ozone
           retention-days: 1
-      - name: Delete temporary build artifacts before caching
-        run: |
-          #Never cache local artifacts
-          rm -rf ~/.m2/repository/org/apache/ozone/hdds*
-          rm -rf ~/.m2/repository/org/apache/ozone/ozone*
-        if: always()
   compile:
     needs:
       - build-info
@@ -162,12 +157,13 @@ jobs:
           git config user.email 'noreply@github.com'
           git commit --allow-empty -a -m 'workaround for HADOOP-19011'
       - name: Cache for maven dependencies
-        uses: actions/cache@v3
+        uses: actions/cache/restore@v3
         with:
-          path: ~/.m2/repository
-          key: maven-repo-${{ hashFiles('**/pom.xml') }}-${{ matrix.java }}
+          path: |
+            ~/.m2/repository
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
-            maven-repo-${{ hashFiles('**/pom.xml') }}
             maven-repo-
       - name: Setup java
         uses: actions/setup-java@v3
@@ -179,12 +175,6 @@ jobs:
         env:
           OZONE_WITH_COVERAGE: false
           GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
-      - name: Delete temporary build artifacts before caching
-        run: |
-          #Never cache local artifacts
-          rm -rf ~/.m2/repository/org/apache/ozone/hdds*
-          rm -rf ~/.m2/repository/org/apache/ozone/ozone*
-        if: always()
   basic:
     needs:
       - build-info
@@ -205,13 +195,13 @@ jobs:
           fetch-depth: 0
         if: matrix.check == 'bats'
       - name: Cache for maven dependencies
-        uses: actions/cache@v3
+        uses: actions/cache/restore@v3
         with:
-          path: ~/.m2/repository
-          key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ matrix.check }}
+          path: |
+            ~/.m2/repository
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
-            maven-repo-${{ hashFiles('**/pom.xml') }}-8
-            maven-repo-${{ hashFiles('**/pom.xml') }}
             maven-repo-
         if: ${{ !contains('author,bats,docs', matrix.check) }}
       - name: Setup java
@@ -234,12 +224,6 @@ jobs:
           name: ${{ matrix.check }}
           path: target/${{ matrix.check }}
         continue-on-error: true
-      - name: Delete temporary build artifacts before caching
-        run: |
-          #Never cache local artifacts
-          rm -rf ~/.m2/repository/org/apache/ozone/hdds*
-          rm -rf ~/.m2/repository/org/apache/ozone/ozone*
-        if: always()
   unit:
     needs:
       - build-info
@@ -255,13 +239,13 @@ jobs:
       - name: Checkout project
         uses: actions/checkout@v3
       - name: Cache for maven dependencies
-        uses: actions/cache@v3
+        uses: actions/cache/restore@v3
         with:
-          path: ~/.m2/repository
-          key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ matrix.profile }}
+          path: |
+            ~/.m2/repository
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
-            maven-repo-${{ hashFiles('**/pom.xml') }}-8
-            maven-repo-${{ hashFiles('**/pom.xml') }}
             maven-repo-
       - name: Setup java
         uses: actions/setup-java@v3
@@ -283,12 +267,6 @@ jobs:
           name: ${{ matrix.check }}
           path: target/${{ matrix.check }}
         continue-on-error: true
-      - name: Delete temporary build artifacts before caching
-        run: |
-          #Never cache local artifacts
-          rm -rf ~/.m2/repository/org/apache/ozone/hdds*
-          rm -rf ~/.m2/repository/org/apache/ozone/ozone*
-        if: always()
   dependency:
     needs:
       - build-info
@@ -328,6 +306,15 @@ jobs:
     steps:
       - name: Checkout project
         uses: actions/checkout@v3
+      - name: Cache for maven dependencies
+        uses: actions/cache/restore@v3
+        with:
+          path: |
+            ~/.m2/repository
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            maven-repo-
       - name: Download Ozone repo
         id: download-ozone-repo
         uses: actions/download-artifact@v3
@@ -456,13 +443,13 @@ jobs:
       - name: Checkout project
         uses: actions/checkout@v3
       - name: Cache for maven dependencies
-        uses: actions/cache@v3
+        uses: actions/cache/restore@v3
         with:
-          path: ~/.m2/repository
-          key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ matrix.profile }}
+          path: |
+            ~/.m2/repository
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
-            maven-repo-${{ hashFiles('**/pom.xml') }}-8
-            maven-repo-${{ hashFiles('**/pom.xml') }}
             maven-repo-
       - name: Download Ozone repo
         id: download-ozone-repo
@@ -502,12 +489,6 @@ jobs:
           name: it-${{ matrix.profile }}
           path: target/integration
         continue-on-error: true
-      - name: Delete temporary build artifacts before caching
-        run: |
-          #Never cache local artifacts
-          rm -rf ~/.m2/repository/org/apache/ozone/hdds*
-          rm -rf ~/.m2/repository/org/apache/ozone/ozone*
-        if: always()
   coverage:
     runs-on: ubuntu-20.04
     timeout-minutes: 30
@@ -522,13 +503,13 @@ jobs:
         with:
           fetch-depth: 0
       - name: Cache for maven dependencies
-        uses: actions/cache@v3
+        uses: actions/cache/restore@v3
         with:
-          path: ~/.m2/repository
-          key: maven-repo-${{ hashFiles('**/pom.xml') }}-8-${{ github.job }}
+          path: |
+            ~/.m2/repository
+            !~/.m2/repository/org/apache/ozone
+          key: maven-repo-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
-            maven-repo-${{ hashFiles('**/pom.xml') }}-8
-            maven-repo-${{ hashFiles('**/pom.xml') }}
             maven-repo-
       - name: Download artifacts
         uses: actions/download-artifact@v3
@@ -557,9 +538,3 @@ jobs:
           name: coverage
           path: target/coverage
         continue-on-error: true
-      - name: Delete temporary build artifacts before caching
-        run: |
-          #Never cache local artifacts
-          rm -rf ~/.m2/repository/org/apache/ozone/hdds*
-          rm -rf ~/.m2/repository/org/apache/ozone/ozone*
-        if: always()

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java

@@ -17,6 +17,7 @@
  */
 package org.apache.hadoop.hdds.scm;
 
+import com.google.common.annotations.VisibleForTesting;
 import org.apache.hadoop.hdds.conf.Config;
 import org.apache.hadoop.hdds.conf.ConfigGroup;
 import org.apache.hadoop.hdds.conf.ConfigTag;
@@ -254,6 +255,7 @@ public long getStreamBufferFlushSize() {
     return streamBufferFlushSize;
   }
 
+  @VisibleForTesting
   public void setStreamBufferFlushSize(long streamBufferFlushSize) {
     this.streamBufferFlushSize = streamBufferFlushSize;
   }
@@ -262,6 +264,7 @@ public int getStreamBufferSize() {
     return streamBufferSize;
   }
 
+  @VisibleForTesting
   public void setStreamBufferSize(int streamBufferSize) {
     this.streamBufferSize = streamBufferSize;
   }
@@ -270,6 +273,7 @@ public boolean isStreamBufferFlushDelay() {
     return streamBufferFlushDelay;
   }
 
+  @VisibleForTesting
   public void setStreamBufferFlushDelay(boolean streamBufferFlushDelay) {
     this.streamBufferFlushDelay = streamBufferFlushDelay;
   }
@@ -278,6 +282,7 @@ public long getStreamBufferMaxSize() {
     return streamBufferMaxSize;
   }
 
+  @VisibleForTesting
   public void setStreamBufferMaxSize(long streamBufferMaxSize) {
     this.streamBufferMaxSize = streamBufferMaxSize;
   }

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/StreamBufferArgs.java

@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.hadoop.hdds.scm;
+
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.client.ReplicationConfig;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+
+/**
+ * This class encapsulates the arguments that are
+ * required for Ozone client StreamBuffer.
+ */
+public class StreamBufferArgs {
+
+  private int streamBufferSize;
+  private long streamBufferFlushSize;
+  private long streamBufferMaxSize;
+  private boolean streamBufferFlushDelay;
+
+  protected StreamBufferArgs(Builder builder) {
+    this.streamBufferSize = builder.bufferSize;
+    this.streamBufferFlushSize = builder.bufferFlushSize;
+    this.streamBufferMaxSize = builder.bufferMaxSize;
+    this.streamBufferFlushDelay = builder.streamBufferFlushDelay;
+  }
+
+  public int getStreamBufferSize() {
+    return streamBufferSize;
+  }
+
+  public long getStreamBufferFlushSize() {
+    return streamBufferFlushSize;
+  }
+
+  public long getStreamBufferMaxSize() {
+    return streamBufferMaxSize;
+  }
+
+  public boolean isStreamBufferFlushDelay() {
+    return streamBufferFlushDelay;
+  }
+
+  public void setStreamBufferFlushDelay(boolean streamBufferFlushDelay) {
+    this.streamBufferFlushDelay = streamBufferFlushDelay;
+  }
+
+  protected void setStreamBufferSize(int streamBufferSize) {
+    this.streamBufferSize = streamBufferSize;
+  }
+
+  protected void setStreamBufferFlushSize(long streamBufferFlushSize) {
+    this.streamBufferFlushSize = streamBufferFlushSize;
+  }
+
+  protected void setStreamBufferMaxSize(long streamBufferMaxSize) {
+    this.streamBufferMaxSize = streamBufferMaxSize;
+  }
+
+  /**
+   * Builder class for StreamBufferArgs.
+   */
+  public static class Builder {
+    private int bufferSize;
+    private long bufferFlushSize;
+    private long bufferMaxSize;
+    private boolean streamBufferFlushDelay;
+
+    public Builder setBufferSize(int bufferSize) {
+      this.bufferSize = bufferSize;
+      return this;
+    }
+
+    public Builder setBufferFlushSize(long bufferFlushSize) {
+      this.bufferFlushSize = bufferFlushSize;
+      return this;
+    }
+
+    public Builder setBufferMaxSize(long bufferMaxSize) {
+      this.bufferMaxSize = bufferMaxSize;
+      return this;
+    }
+
+    public Builder setStreamBufferFlushDelay(boolean streamBufferFlushDelay) {
+      this.streamBufferFlushDelay = streamBufferFlushDelay;
+      return this;
+    }
+
+    public StreamBufferArgs build() {
+      return new StreamBufferArgs(this);
+    }
+
+    public static Builder getNewBuilder() {
+      return new Builder();
+    }
+  }
+
+  public static StreamBufferArgs getDefaultStreamBufferArgs(
+      ReplicationConfig replicationConfig, OzoneClientConfig clientConfig) {
+    int bufferSize;
+    long flushSize;
+    long bufferMaxSize;
+    boolean streamBufferFlushDelay = clientConfig.isStreamBufferFlushDelay();
+    if (replicationConfig.getReplicationType() == HddsProtos.ReplicationType.EC) {
+      bufferSize = ((ECReplicationConfig) replicationConfig).getEcChunkSize();
+      flushSize = ((ECReplicationConfig) replicationConfig).getEcChunkSize();
+      bufferMaxSize = ((ECReplicationConfig) replicationConfig).getEcChunkSize();
+    } else {
+      bufferSize = clientConfig.getStreamBufferSize();
+      flushSize = clientConfig.getStreamBufferFlushSize();
+      bufferMaxSize = clientConfig.getStreamBufferMaxSize();
+    }
+
+    return Builder.getNewBuilder()
+        .setBufferSize(bufferSize)
+        .setBufferFlushSize(flushSize)
+        .setBufferMaxSize(bufferMaxSize)
+        .setStreamBufferFlushDelay(streamBufferFlushDelay)
+        .build();
+  }
+}