Merge remote-tracking branch 'origin/master' into HDDS-10656-atomic-k…

…ey-overwrite

.github/workflows/ci-with-ratis.yml

@@ -50,3 +50,4 @@ jobs:
     secrets: inherit
     with:
       ratis_args: "-Dratis.version=${{ needs.ratis.outputs.ratis-version }} -Dratis.thirdparty.version=${{ needs.ratis.outputs.thirdparty-version }} -Dio.grpc.version=${{ needs.ratis.outputs.grpc-version }} -Dnetty.version=${{ needs.ratis.outputs.netty-version }} -Dgrpc.protobuf-compile.version=${{ needs.ratis.outputs.protobuf-version }}"
+      ref: ${{ github.event.inputs.ref }}

.github/workflows/ci.yml

@@ -21,10 +21,15 @@ on:
         description: Version overrides from custom Ratis build
         default: ''
         required: false
+      ref:
+        type: string
+        description: Ozone ref (branch, tag or commit SHA)
+        default: ''
+        required: false
 env:
   FAIL_FAST: ${{ github.event_name == 'pull_request' }}
   MAVEN_OPTS: -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3
-  OZONE_WITH_COVERAGE: ${{ github.repository == 'apache/ozone' && github.event_name != 'pull_request' }}
+  OZONE_WITH_COVERAGE: ${{ github.repository == 'apache/ozone' && github.event_name == 'push' }}
 jobs:
   build-info:
     runs-on: ubuntu-20.04
@@ -41,18 +46,35 @@ jobs:
       needs-dependency-check: ${{ steps.selective-checks.outputs.needs-dependency-check }}
       needs-integration-tests: ${{ steps.selective-checks.outputs.needs-integration-tests }}
       needs-kubernetes-tests: ${{ steps.selective-checks.outputs.needs-kubernetes-tests }}
+      sha: ${{ steps.get-sha.outputs.sha }}
     steps:
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+      - name: "Checkout ${{ github.ref }} / ${{ github.sha }} (push)"
         uses: actions/checkout@v4
         with:
           persist-credentials: false
-      - name: Fetch incoming commit ${{ github.sha }} with its parent
+        if: github.event_name  == 'push'
+      - name: "Checkout ${{ github.sha }} with its parent (pull request)"
         uses: actions/checkout@v4
         with:
           ref: ${{ github.sha }}
           fetch-depth: 2
           persist-credentials: false
         if: github.event_name  == 'pull_request'
+      - name: "Checkout ${{ inputs.ref }} given in workflow input (manual dispatch)"
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          persist-credentials: false
+        if: github.event_name  == 'workflow_dispatch'
+      - name: Get SHA of ${{ inputs.ref || github.ref }}
+        id: get-sha
+        run: |
+          if [[ "${GITHUB_EVENT_NAME}" == "workflow_dispatch" ]]; then
+            sha="$(git rev-parse --verify HEAD)"
+          else
+            sha="${GITHUB_SHA}"
+          fi
+          echo "sha=$sha" >> $GITHUB_OUTPUT
       - name: Selective checks
         id: selective-checks
         env:
@@ -87,6 +109,8 @@ jobs:
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.build-info.outputs.sha }}
       - name: Cache for npm dependencies
         uses: actions/cache@v4
         with:
@@ -211,11 +235,14 @@ jobs:
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.build-info.outputs.sha }}
         if: matrix.check != 'bats'
       - name: Checkout project with history
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          ref: ${{ needs.build-info.outputs.sha }}
         if: matrix.check == 'bats'
       - name: Cache for maven dependencies
         uses: actions/cache/restore@v4
@@ -264,6 +291,8 @@ jobs:
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.build-info.outputs.sha }}
       - name: Cache for maven dependencies
         uses: actions/cache/restore@v4
         with:
@@ -310,6 +339,8 @@ jobs:
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.build-info.outputs.sha }}
       - name: Download compiled Ozone binaries
         uses: actions/download-artifact@v4
         with:
@@ -339,6 +370,8 @@ jobs:
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.build-info.outputs.sha }}
       - name: Cache for maven dependencies
         uses: actions/cache/restore@v4
         with:
@@ -384,6 +417,8 @@ jobs:
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.build-info.outputs.sha }}
       - name: Download compiled Ozone binaries
         uses: actions/download-artifact@v4
         with:
@@ -426,6 +461,8 @@ jobs:
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.build-info.outputs.sha }}
       - name: Download compiled Ozone binaries
         uses: actions/download-artifact@v4
         with:
@@ -475,6 +512,8 @@ jobs:
     steps:
       - name: Checkout project
         uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.build-info.outputs.sha }}
       - name: Cache for maven dependencies
         uses: actions/cache/restore@v4
         with:
@@ -520,8 +559,9 @@ jobs:
   coverage:
     runs-on: ubuntu-20.04
     timeout-minutes: 30
-    if: github.repository == 'apache/ozone' && github.event_name != 'pull_request'
+    if: github.repository == 'apache/ozone' && github.event_name == 'push'
     needs:
+      - build-info
       - acceptance
       - integration
       - native
@@ -530,6 +570,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           fetch-depth: 0
+          ref: ${{ needs.build-info.outputs.sha }}
       - name: Cache for maven dependencies
         uses: actions/cache/restore@v4
         with:

.mvn/extensions.xml

@@ -24,7 +24,7 @@
   <extension>
     <groupId>com.gradle</groupId>
     <artifactId>develocity-maven-extension</artifactId>
-    <version>1.21.4</version>
+    <version>1.21.5</version>
   </extension>
   <extension>
     <groupId>com.gradle</groupId>

hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java

@@ -56,8 +56,10 @@
 import org.apache.ratis.client.api.DataStreamApi;
 import org.apache.ratis.grpc.GrpcTlsConfig;
 import org.apache.ratis.proto.RaftProtos;
+import org.apache.ratis.proto.RaftProtos.CommitInfoProto;
 import org.apache.ratis.proto.RaftProtos.ReplicationLevel;
 import org.apache.ratis.protocol.exceptions.GroupMismatchException;
+import org.apache.ratis.protocol.exceptions.NotReplicatedException;
 import org.apache.ratis.protocol.exceptions.RaftException;
 import org.apache.ratis.protocol.RaftClientReply;
 import org.apache.ratis.retry.RetryPolicy;
@@ -315,31 +317,46 @@ public XceiverClientReply watchForCommit(long index)
         throw e;
       }
       if (watchType == ReplicationLevel.ALL_COMMITTED) {
-        final RaftClientReply reply = getClient().async()
-            .watch(index, RaftProtos.ReplicationLevel.MAJORITY_COMMITTED)
-            .get();
-        final XceiverClientReply clientReply = newWatchReply(
-            index, ReplicationLevel.MAJORITY_COMMITTED, index);
-        reply.getCommitInfos().stream()
-            .filter(i -> i.getCommitIndex() < index)
-            .forEach(proto -> {
-              UUID address = RatisHelper.toDatanodeId(proto.getServer());
-              addDatanodetoReply(address, clientReply);
-              // since 3 way commit has failed, the updated map from now on will
-              // only store entries for those datanodes which have had successful
-              // replication.
-              commitInfoMap.remove(address);
-              LOG.info(
-                  "Could not commit index {} on pipeline {} to all the nodes. " +
-                      "Server {} has failed. Committed by majority.",
-                  index, pipeline, address);
-            });
-        return clientReply;
+        Throwable nre =
+            HddsClientUtils.containsException(e, NotReplicatedException.class);
+        Collection<CommitInfoProto> commitInfoProtoList;
+        if (nre instanceof NotReplicatedException) {
+          // If NotReplicatedException is thrown from the Datanode leader
+          // we can save one watch request round trip by using the CommitInfoProto
+          // in the NotReplicatedException
+          commitInfoProtoList = ((NotReplicatedException) nre).getCommitInfos();
+        } else {
+          final RaftClientReply reply = getClient().async()
+              .watch(index, RaftProtos.ReplicationLevel.MAJORITY_COMMITTED)
+              .get();
+          commitInfoProtoList = reply.getCommitInfos();
+        }
+        return handleFailedAllCommit(index, commitInfoProtoList);
       }
       throw e;
     }
   }
 
+  private XceiverClientReply handleFailedAllCommit(long index, Collection<CommitInfoProto> commitInfoProtoList) {
+    final XceiverClientReply clientReply = newWatchReply(
+        index, ReplicationLevel.MAJORITY_COMMITTED, index);
+    commitInfoProtoList.stream()
+        .filter(i -> i.getCommitIndex() < index)
+        .forEach(proto -> {
+          UUID address = RatisHelper.toDatanodeId(proto.getServer());
+          addDatanodetoReply(address, clientReply);
+          // since 3 way commit has failed, the updated map from now on  will
+          // only store entries for those datanodes which have had successful
+          // replication.
+          commitInfoMap.remove(address);
+          LOG.info(
+              "Could not commit index {} on pipeline {} to all the nodes. " +
+                  "Server {} has failed. Committed by majority.",
+              index, pipeline, address);
+        });
+    return clientReply;
+  }
+
   /**
    * Sends a given command to server gets a waitable future back.
    *

hadoop-hdds/common/pom.xml

@@ -40,6 +40,10 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
       <groupId>info.picocli</groupId>
       <artifactId>picocli</artifactId>
     </dependency>
+    <dependency>
+      <groupId>com.github.stephenc.jcip</groupId>
+      <artifactId>jcip-annotations</artifactId>
+    </dependency>
     <dependency>
       <groupId>com.google.protobuf</groupId>
       <artifactId>protobuf-java</artifactId>

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/StringUtils.java

@@ -20,19 +20,9 @@
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Map;
 
 import com.google.common.base.Preconditions;
-import org.apache.commons.lang3.SystemUtils;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.utils.SignalLogger;
-import org.apache.hadoop.hdds.utils.VersionInfo;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.util.ShutdownHookManager;
 import org.apache.ratis.thirdparty.io.netty.buffer.Unpooled;
-import org.slf4j.Logger;
 
 /**
  * Simple utility class to collection string conversion methods.
@@ -44,11 +34,6 @@ private StringUtils() {
 
   private static final Charset UTF8 = StandardCharsets.UTF_8;
 
-  /**
-   * Priority of the StringUtils shutdown hook.
-   */
-  private static final int SHUTDOWN_HOOK_PRIORITY = 0;
-
   /**
    * Decode a specific range of bytes of the given byte array to a string
    * using UTF8.
@@ -107,70 +92,6 @@ public static byte[] string2Bytes(String str) {
     return str.getBytes(UTF8);
   }
 
-  /**
-   * Return a message for logging.
-   * @param prefix prefix keyword for the message
-   * @param msg content of the message
-   * @return a message for logging
-   */
-  public static String toStartupShutdownString(String prefix, String... msg) {
-    StringBuilder b = new StringBuilder(prefix);
-    b.append("\n/************************************************************");
-    for (String s : msg) {
-      b.append("\n").append(prefix).append(s);
-    }
-    b.append("\n************************************************************/");
-    return b.toString();
-  }
-
-  public static void startupShutdownMessage(VersionInfo versionInfo,
-      Class<?> clazz, String[] args, Logger log, OzoneConfiguration conf) {
-    final String hostname = NetUtils.getHostname();
-    final String className = clazz.getSimpleName();
-
-    if (log.isInfoEnabled()) {
-      log.info(createStartupShutdownMessage(versionInfo, className, hostname,
-          args, HddsUtils.processForLogging(conf)));
-    }
-
-    if (SystemUtils.IS_OS_UNIX) {
-      try {
-        SignalLogger.INSTANCE.register(log);
-      } catch (Throwable t) {
-        log.warn("failed to register any UNIX signal loggers: ", t);
-      }
-    }
-    ShutdownHookManager.get().addShutdownHook(
-        () -> log.info(toStartupShutdownString("SHUTDOWN_MSG: ",
-            "Shutting down " + className + " at " + hostname)),
-        SHUTDOWN_HOOK_PRIORITY);
-
-  }
-
-  /**
-   * Generate the text for the startup/shutdown message of processes.
-   * @param className short name of the class
-   * @param hostname hostname
-   * @param args Command arguments
-   * @return a string to log.
-   */
-  public static String createStartupShutdownMessage(VersionInfo versionInfo,
-      String className, String hostname, String[] args,
-      Map<String, String> conf) {
-    return toStartupShutdownString("STARTUP_MSG: ",
-        "Starting " + className,
-        "  host = " + hostname,
-        "  args = " + (args != null ? Arrays.asList(args) : new ArrayList<>()),
-        "  version = " + versionInfo.getVersion(),
-        "  classpath = " + System.getProperty("java.class.path"),
-        "  build = " + versionInfo.getUrl() + "/"
-            + versionInfo.getRevision()
-            + " ; compiled by '" + versionInfo.getUser()
-            + "' on " + versionInfo.getDate(),
-        "  java = " + System.getProperty("java.version"),
-        "  conf = " + conf);
-  }
-
   public static String appendIfNotPresent(String str, char c) {
     Preconditions.checkNotNull(str, "Input string is null");
     return str.isEmpty() || str.charAt(str.length() - 1) != c ? str + c : str;

hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/DecommissionUtils.java

@@ -126,7 +126,9 @@ public static Map<String, Object> getCountsMap(DatanodeDetails datanode, JsonNod
                                                  Map<String, Object> countsMap, String errMsg)
       throws IOException {
     for (int i = 1; i <= numDecomNodes; i++) {
-      if (datanode.getHostName().equals(counts.get("tag.datanode." + i).asText())) {
+      String datanodeHostName =
+          (counts.get("tag.datanode." + i) != null) ? (counts.get("tag.datanode." + i).asText()) : "";
+      if (datanode.getHostName().equals(datanodeHostName)) {
         JsonNode pipelinesDN = counts.get("PipelinesWaitingToCloseDN." + i);
         JsonNode underReplicatedDN = counts.get("UnderReplicatedDN." + i);
         JsonNode unclosedDN = counts.get("UnclosedContainersDN." + i);