Merge branch 'HDDS-3698-upgrade' into HDDS-4463

* HDDS-3698-upgrade: (47 commits)
  HDDS-4468. Fix Goofys listBucket large than 1000 objects will stuck forever (apache#1595)
  HDDS-4417. Simplify Ozone client code with configuration object -- addendum (apache#1581)
  HDDS-4476. Improve the ZH translation of the HA.md in doc. (apache#1597)
  HDDS-4432. Update Ratis version to latest snapshot. (apache#1586)
  HDDS-4488. Open RocksDB read only when loading containers at Datanode startup (apache#1605)
  HDDS-4478. Large deletedKeyset slows down OM via listStatus. (apache#1598)
  HDDS-4452. findbugs.sh couldn't be executed after a full build (apache#1576)
  HDDS-4427. Avoid ContainerCache in ContainerReader at Datanode startup (apache#1549)
  HDDS-4448. Duplicate refreshPipeline in listStatus (apache#1569)
  HDDS-4450. Cannot run ozone if HADOOP_HOME points to Hadoop install (apache#1572)
  HDDS-4346.Ozone specific Trash Policy (apache#1535)
  HDDS-4426. SCM should create transactions using all blocks received from OM (apache#1561)
  HDDS-4399. Safe mode rule for piplelines should only consider open pipelines. (apache#1526)
  HDDS-4367. Configuration for deletion service intervals should be different for OM, SCM and datanodes (apache#1573)
  HDDS-4462. Add --frozen-lockfile to pnpm install to prevent ozone-recon-web/pnpm-lock.yaml from being updated automatically (apache#1589)
  HDDS-4082. Create ZH translation of HA.md in doc. (apache#1591)
  HDDS-4464. Upgrade httpclient version due to CVE-2020-13956. (apache#1590)
  HDDS-4467. Acceptance test fails due to new Hadoop 3 image (apache#1594)
  HDDS-4466. Update url in .asf.yaml to use TLP project (apache#1592)
  HDDS-4458. Fix Max Transaction ID value in OM. (apache#1585)
  ...

.asf.yaml

@@ -14,7 +14,7 @@
 # limitations under the License.
 github:
   description: "Scalable, redundant, and distributed object store for Apache Hadoop"
-  homepage: https://hadoop.apache.org/ozone/
+  homepage: https://ozone.apache.org
   labels:
     - hadoop
     - object-store

README.md

@@ -13,7 +13,7 @@ Ozone is a scalable, redundant, and distributed object store for Hadoop and Clou
 
 ## Documentation
 
-The latest documentation is generated together with the releases and hosted on the apache side.
+The latest documentation is generated together with the releases and hosted on the apache site.
 
 Please check [the documentation page](https://ozone.apache.org/docs/) for more information.
 

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java

@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdds.scm;
+
+import org.apache.hadoop.hdds.conf.Config;
+import org.apache.hadoop.hdds.conf.ConfigGroup;
+import org.apache.hadoop.hdds.conf.ConfigTag;
+import org.apache.hadoop.hdds.conf.ConfigType;
+import org.apache.hadoop.hdds.conf.PostConstruct;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChecksumType;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+
+import com.google.common.base.Preconditions;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Configuration values for Ozone Client.
+ */
+@ConfigGroup(prefix = "ozone.client")
+public class OzoneClientConfig {
+
+  private static final Logger LOG =
+      LoggerFactory.getLogger(OzoneClientConfig.class);
+
+  @Config(key = "stream.buffer.flush.size",
+      defaultValue = "16MB",
+      type = ConfigType.SIZE,
+      description = "Size which determines at what buffer position a partial "
+          + "flush will be initiated during write. It should be a multiple of"
+          + " ozone.client.stream.buffer.size",
+      tags = ConfigTag.CLIENT)
+  private long streamBufferFlushSize = 16 * 1024 * 1024;
+
+  @Config(key = "stream.buffer.size",
+      defaultValue = "4MB",
+      type = ConfigType.SIZE,
+      description = "The size of chunks the client will send to the server",
+      tags = ConfigTag.CLIENT)
+  private int streamBufferSize = 4 * 1024 * 1024;
+
+  @Config(key = "stream.buffer.flush.delay",
+      defaultValue = "true",
+      description = "Default true, when call flush() and determine whether "
+          + "the data in the current buffer is greater than ozone.client"
+          + ".stream.buffer.size, if greater than then send buffer to the "
+          + "datanode. You can  turn this off by setting this configuration "
+          + "to false.", tags = ConfigTag.CLIENT)
+  private boolean streamBufferFlushDelay = true;
+
+  @Config(key = "stream.buffer.max.size",
+      defaultValue = "32MB",
+      type = ConfigType.SIZE,
+      description = "Size which determines at what buffer position write call"
+          + " be blocked till acknowledgement of the first partial flush "
+          + "happens by all servers.",
+      tags = ConfigTag.CLIENT)
+  private long streamBufferMaxSize = 32 * 1024 * 1024;
+
+  @Config(key = "max.retries",
+      defaultValue = "5",
+      description = "Maximum number of retries by Ozone Client on "
+          + "encountering exception while writing a key",
+      tags = ConfigTag.CLIENT)
+  private int maxRetryCount = 5;
+
+  @Config(key = "retry.interval",
+      defaultValue = "0",
+      description =
+          "Indicates the time duration a client will wait before retrying a "
+              + "write key request on encountering an exception. By default "
+              + "there is no wait",
+      tags = ConfigTag.CLIENT)
+  private int retryInterval = 0;
+
+  @Config(key = "checksum.type",
+      defaultValue = "CRC32",
+      description = "The checksum type [NONE/ CRC32/ CRC32C/ SHA256/ MD5] "
+          + "determines which algorithm would be used to compute checksum for "
+          + "chunk data. Default checksum type is CRC32.",
+      tags = ConfigTag.CLIENT)
+  private String checksumType = ChecksumType.CRC32.name();
+
+  @Config(key = "bytes.per.checksum",
+      defaultValue = "1MB",
+      type = ConfigType.SIZE,
+      description = "Checksum will be computed for every bytes per checksum "
+          + "number of bytes and stored sequentially. The minimum value for "
+          + "this config is 256KB.",
+      tags = ConfigTag.CLIENT)
+  private int bytesPerChecksum = 1024 * 1024;
+
+  @Config(key = "verify.checksum",
+      defaultValue = "true",
+      description = "Ozone client to verify checksum of the checksum "
+          + "blocksize data.",
+      tags = ConfigTag.CLIENT)
+  private boolean checksumVerify = true;
+
+  @PostConstruct
+  private void validate() {
+    Preconditions.checkState(streamBufferSize > 0);
+    Preconditions.checkState(streamBufferFlushSize > 0);
+    Preconditions.checkState(streamBufferMaxSize > 0);
+
+    Preconditions.checkState(streamBufferMaxSize % streamBufferFlushSize == 0,
+        "expected max. buffer size (%s) to be a multiple of flush size (%s)",
+        streamBufferMaxSize, streamBufferFlushSize);
+    Preconditions.checkState(streamBufferFlushSize % streamBufferSize == 0,
+        "expected flush size (%s) to be a multiple of buffer size (%s)",
+        streamBufferFlushSize, streamBufferSize);
+
+    if (bytesPerChecksum <
+        OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM_MIN_SIZE) {
+      LOG.warn("The checksum size ({}) is not allowed to be less than the " +
+              "minimum size ({}), resetting to the minimum size.",
+          bytesPerChecksum,
+          OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM_MIN_SIZE);
+      bytesPerChecksum =
+          OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM_MIN_SIZE;
+    }
+
+  }
+
+  public long getStreamBufferFlushSize() {
+    return streamBufferFlushSize;
+  }
+
+  public void setStreamBufferFlushSize(long streamBufferFlushSize) {
+    this.streamBufferFlushSize = streamBufferFlushSize;
+  }
+
+  public int getStreamBufferSize() {
+    return streamBufferSize;
+  }
+
+  public void setStreamBufferSize(int streamBufferSize) {
+    this.streamBufferSize = streamBufferSize;
+  }
+
+  public boolean isStreamBufferFlushDelay() {
+    return streamBufferFlushDelay;
+  }
+
+  public void setStreamBufferFlushDelay(boolean streamBufferFlushDelay) {
+    this.streamBufferFlushDelay = streamBufferFlushDelay;
+  }
+
+  public long getStreamBufferMaxSize() {
+    return streamBufferMaxSize;
+  }
+
+  public void setStreamBufferMaxSize(long streamBufferMaxSize) {
+    this.streamBufferMaxSize = streamBufferMaxSize;
+  }
+
+  public int getMaxRetryCount() {
+    return maxRetryCount;
+  }
+
+  public void setMaxRetryCount(int maxRetryCount) {
+    this.maxRetryCount = maxRetryCount;
+  }
+
+  public int getRetryInterval() {
+    return retryInterval;
+  }
+
+  public void setRetryInterval(int retryInterval) {
+    this.retryInterval = retryInterval;
+  }
+
+  public ChecksumType getChecksumType() {
+    return ChecksumType.valueOf(checksumType);
+  }
+
+  public void setChecksumType(ChecksumType checksumType) {
+    this.checksumType = checksumType.name();
+  }
+
+  public int getBytesPerChecksum() {
+    return bytesPerChecksum;
+  }
+
+  public void setBytesPerChecksum(int bytesPerChecksum) {
+    this.bytesPerChecksum = bytesPerChecksum;
+  }
+
+  public boolean isChecksumVerify() {
+    return checksumVerify;
+  }
+
+  public void setChecksumVerify(boolean checksumVerify) {
+    this.checksumVerify = checksumVerify;
+  }
+
+}

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java

@@ -217,12 +217,12 @@ private CompletableFuture<RaftClientReply> sendRequestAsync(
             if (LOG.isDebugEnabled()) {
               LOG.debug("sendCommandAsync ReadOnly {}", message);
             }
-            return getClient().sendReadOnlyAsync(message);
+            return getClient().async().sendReadOnly(message);
           } else {
             if (LOG.isDebugEnabled()) {
               LOG.debug("sendCommandAsync {}", message);
             }
-            return getClient().sendAsync(message);
+            return getClient().async().send(message);
           }
 
         }
@@ -258,17 +258,17 @@ public XceiverClientReply watchForCommit(long index)
     }
     RaftClientReply reply;
     try {
-      CompletableFuture<RaftClientReply> replyFuture = getClient()
-          .sendWatchAsync(index, RaftProtos.ReplicationLevel.ALL_COMMITTED);
+      CompletableFuture<RaftClientReply> replyFuture = getClient().async()
+          .watch(index, RaftProtos.ReplicationLevel.ALL_COMMITTED);
       replyFuture.get();
     } catch (Exception e) {
       Throwable t = HddsClientUtils.checkForException(e);
       LOG.warn("3 way commit failed on pipeline {}", pipeline, e);
       if (t instanceof GroupMismatchException) {
         throw e;
       }
-      reply = getClient()
-          .sendWatchAsync(index, RaftProtos.ReplicationLevel.MAJORITY_COMMITTED)
+      reply = getClient().async()
+          .watch(index, RaftProtos.ReplicationLevel.MAJORITY_COMMITTED)
           .get();
       List<RaftProtos.CommitInfoProto> commitInfoProtoList =
           reply.getCommitInfos().stream()