From bc46c815f01c7a7177e24e60c4eaf0aecb11c9a4 Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Tue, 2 Jan 2024 17:39:09 +0800 Subject: [PATCH 1/6] HDFS-17317 DebugAdmin metaOut not need multiple close --- .../src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java index 7116c2578ca59..4c31b29684e62 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java @@ -306,7 +306,6 @@ int run(List args) throws IOException { new BufferedOutputStream(Files.newOutputStream(srcMeta.toPath()), smallBufferSize)); BlockMetadataHeader.writeHeader(metaOut, checksum); - metaOut.close(); FsDatasetUtil.computeChecksum( srcMeta, srcMeta, blockFile, smallBufferSize, conf); System.out.println( From 9fe2548f4775be200bb8883523800416e3d7fbee Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Wed, 3 Jan 2024 19:22:09 +0800 Subject: [PATCH 2/6] fix --- .../src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java | 1 + 1 file changed, 1 insertion(+) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java index 4c31b29684e62..54f3bb3417c2c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java @@ -306,6 +306,7 @@ int run(List args) throws IOException { new BufferedOutputStream(Files.newOutputStream(srcMeta.toPath()), smallBufferSize)); BlockMetadataHeader.writeHeader(metaOut, checksum); + metaOut.flush(); FsDatasetUtil.computeChecksum( srcMeta, srcMeta, blockFile, smallBufferSize, conf); System.out.println( From dd3c3e745dee375cc9aca457a76d9c03a0acc4cd Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Wed, 3 Jan 2024 19:38:25 +0800 Subject: [PATCH 3/6] refactor --- .../apache/hadoop/hdfs/tools/DebugAdmin.java | 39 +++++++++---------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java index 54f3bb3417c2c..29706a0f0a96c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java @@ -295,27 +295,24 @@ int run(List args) throws IOException { } DataOutputStream metaOut = null; - try { - final Configuration conf = new Configuration(); - final Options.ChecksumOpt checksumOpt = - DfsClientConf.getChecksumOptFromConf(conf); - final DataChecksum checksum = createChecksum(checksumOpt); - - final int smallBufferSize = DFSUtilClient.getSmallBufferSize(conf); - metaOut = new DataOutputStream( - new BufferedOutputStream(Files.newOutputStream(srcMeta.toPath()), - smallBufferSize)); - BlockMetadataHeader.writeHeader(metaOut, checksum); - metaOut.flush(); - FsDatasetUtil.computeChecksum( - srcMeta, srcMeta, blockFile, smallBufferSize, conf); - System.out.println( - "Checksum calculation succeeded on block file " + name - + " saved metadata to meta file " + outFile); - return 0; - } finally { - IOUtils.cleanupWithLogger(null, metaOut); - } + final Configuration conf = new Configuration(); + final Options.ChecksumOpt checksumOpt = + DfsClientConf.getChecksumOptFromConf(conf); + final DataChecksum checksum = createChecksum(checksumOpt); + + final int smallBufferSize = DFSUtilClient.getSmallBufferSize(conf); + metaOut = new DataOutputStream( + new BufferedOutputStream(Files.newOutputStream(srcMeta.toPath()), + smallBufferSize)); + BlockMetadataHeader.writeHeader(metaOut, checksum); + metaOut.close(); + metaOut = null; + FsDatasetUtil.computeChecksum( + srcMeta, srcMeta, blockFile, smallBufferSize, conf); + System.out.println( + "Checksum calculation succeeded on block file " + name + + " saved metadata to meta file " + outFile); + return 0; } } From 8805de9adfaa9f7d6285fbd7a2ab18fa63ccb89e Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Wed, 3 Jan 2024 19:44:35 +0800 Subject: [PATCH 4/6] refactor --- .../apache/hadoop/hdfs/tools/DebugAdmin.java | 41 +++++++++++-------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java index 29706a0f0a96c..5460a31a79dde 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java @@ -295,24 +295,29 @@ int run(List args) throws IOException { } DataOutputStream metaOut = null; - final Configuration conf = new Configuration(); - final Options.ChecksumOpt checksumOpt = - DfsClientConf.getChecksumOptFromConf(conf); - final DataChecksum checksum = createChecksum(checksumOpt); - - final int smallBufferSize = DFSUtilClient.getSmallBufferSize(conf); - metaOut = new DataOutputStream( - new BufferedOutputStream(Files.newOutputStream(srcMeta.toPath()), - smallBufferSize)); - BlockMetadataHeader.writeHeader(metaOut, checksum); - metaOut.close(); - metaOut = null; - FsDatasetUtil.computeChecksum( - srcMeta, srcMeta, blockFile, smallBufferSize, conf); - System.out.println( - "Checksum calculation succeeded on block file " + name - + " saved metadata to meta file " + outFile); - return 0; + try { + final Configuration conf = new Configuration(); + final Options.ChecksumOpt checksumOpt = + DfsClientConf.getChecksumOptFromConf(conf); + final DataChecksum checksum = createChecksum(checksumOpt); + + final int smallBufferSize = DFSUtilClient.getSmallBufferSize(conf); + metaOut = new DataOutputStream( + new BufferedOutputStream(Files.newOutputStream(srcMeta.toPath()), + smallBufferSize)); + BlockMetadataHeader.writeHeader(metaOut, checksum); + metaOut.close(); + metaOut = null; + FsDatasetUtil.computeChecksum( + srcMeta, srcMeta, blockFile, smallBufferSize, conf); + System.out.println( + "Checksum calculation succeeded on block file " + name + + " saved metadata to meta file " + outFile); + return 0; + } finally { + IOUtils.cleanupWithLogger(null, metaOut); + } + } } From fca6a16e33a6812ecbae05029d1b5f48f4d82d8e Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Wed, 3 Jan 2024 19:46:28 +0800 Subject: [PATCH 5/6] style change --- .../org/apache/hadoop/hdfs/tools/DebugAdmin.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java index 5460a31a79dde..34556bf3f669a 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java @@ -298,21 +298,21 @@ int run(List args) throws IOException { try { final Configuration conf = new Configuration(); final Options.ChecksumOpt checksumOpt = - DfsClientConf.getChecksumOptFromConf(conf); + DfsClientConf.getChecksumOptFromConf(conf); final DataChecksum checksum = createChecksum(checksumOpt); final int smallBufferSize = DFSUtilClient.getSmallBufferSize(conf); metaOut = new DataOutputStream( - new BufferedOutputStream(Files.newOutputStream(srcMeta.toPath()), - smallBufferSize)); + new BufferedOutputStream(Files.newOutputStream(srcMeta.toPath()), + smallBufferSize)); BlockMetadataHeader.writeHeader(metaOut, checksum); metaOut.close(); metaOut = null; FsDatasetUtil.computeChecksum( - srcMeta, srcMeta, blockFile, smallBufferSize, conf); + srcMeta, srcMeta, blockFile, smallBufferSize, conf); System.out.println( - "Checksum calculation succeeded on block file " + name - + " saved metadata to meta file " + outFile); + "Checksum calculation succeeded on block file " + name + + " saved metadata to meta file " + outFile); return 0; } finally { IOUtils.cleanupWithLogger(null, metaOut); From eba716aec26d9623dbaea7f4f07b50830bf03a79 Mon Sep 17 00:00:00 2001 From: xuyu <11161569@vivo.com> Date: Wed, 3 Jan 2024 19:46:58 +0800 Subject: [PATCH 6/6] style new line remove --- .../src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java index 34556bf3f669a..7333fe016468b 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DebugAdmin.java @@ -317,7 +317,6 @@ int run(List args) throws IOException { } finally { IOUtils.cleanupWithLogger(null, metaOut); } - } }