HADOOP-16954. Add -S option in "Count" command to show only Snapshot Counts. Contributed by hemanthboyina.

Author: tasanuma

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ContentSummary.java

@@ -281,6 +281,21 @@ public int hashCode() {
 
   private static final String ALL_HEADER = QUOTA_HEADER + SUMMARY_HEADER;
 
+  /**
+   * Output format:<-------18-------> <----------24---------->
+   * <----------24---------->. <-------------28------------> SNAPSHOT_LENGTH
+   * SNAPSHOT_FILE_COUNT SNAPSHOT_DIR_COUNT SNAPSHOT_SPACE_CONSUMED
+   */
+  private static final String SNAPSHOT_FORMAT = "%18s %24s %24s %28s ";
+
+  private static final String[] SNAPSHOT_HEADER_FIELDS =
+      new String[] {"SNAPSHOT_LENGTH", "SNAPSHOT_FILE_COUNT",
+          "SNAPSHOT_DIR_COUNT", "SNAPSHOT_SPACE_CONSUMED"};
+
+  /** The header string. */
+  private static final String SNAPSHOT_HEADER =
+      String.format(SNAPSHOT_FORMAT, (Object[]) SNAPSHOT_HEADER_FIELDS);
+
 
   /** Return the header of the output.
    * if qOption is false, output directory count, file count, and content size;
@@ -293,7 +308,9 @@ public static String getHeader(boolean qOption) {
     return qOption ? ALL_HEADER : SUMMARY_HEADER;
   }
 
-
+  public static String getSnapshotHeader() {
+    return SNAPSHOT_HEADER;
+  }
 
   /**
    * Returns the names of the fields from the summary header.
@@ -416,7 +433,7 @@ public String toString(boolean qOption, boolean hOption, boolean tOption,
   }
 
   /**
-   * Formats a size to be human readable or in bytes
+   * Formats a size to be human readable or in bytes.
    * @param size value to be formatted
    * @param humanReadable flag indicating human readable or not
    * @return String representation of the size
@@ -426,4 +443,17 @@ private String formatSize(long size, boolean humanReadable) {
       ? StringUtils.TraditionalBinaryPrefix.long2String(size, "", 1)
       : String.valueOf(size);
   }
+
+  /**
+   * Return the string representation of the snapshot counts in the output
+   * format.
+   * @param hOption flag indicating human readable or not
+   * @return String representation of the snapshot counts
+   */
+  public String toSnapshot(boolean hOption) {
+    return String.format(SNAPSHOT_FORMAT, formatSize(snapshotLength, hOption),
+        formatSize(snapshotFileCount, hOption),
+        formatSize(snapshotDirectoryCount, hOption),
+        formatSize(snapshotSpaceConsumed, hOption));
+  }
 }

hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Count.java

@@ -56,13 +56,14 @@ public static void registerCommands(CommandFactory factory) {
   //return the quota, namespace count and disk space usage.
   private static final String OPTION_QUOTA_AND_USAGE = "u";
   private static final String OPTION_ECPOLICY = "e";
+  private static final String OPTION_SNAPSHOT_COUNT = "s";
 
   public static final String NAME = "count";
   public static final String USAGE =
       "[-" + OPTION_QUOTA + "] [-" + OPTION_HUMAN + "] [-" + OPTION_HEADER
           + "] [-" + OPTION_TYPE + " [<storage type>]] [-" +
           OPTION_QUOTA_AND_USAGE + "] [-" + OPTION_EXCLUDE_SNAPSHOT
-          + "] [-" + OPTION_ECPOLICY
+          + "] [-" + OPTION_ECPOLICY + "] [-" + OPTION_SNAPSHOT_COUNT
           + "] <path> ...";
   public static final String DESCRIPTION =
       "Count the number of directories, files and bytes under the paths\n" +
@@ -93,7 +94,8 @@ public static void registerCommands(CommandFactory factory) {
           "the storage types.\n" +
           "The -" + OPTION_QUOTA_AND_USAGE + " option shows the quota and \n" +
           "the usage against the quota without the detailed content summary."+
-          "The -"+ OPTION_ECPOLICY +" option shows the erasure coding policy.";
+          "The -" + OPTION_ECPOLICY + " option shows the erasure coding policy."
+          + "The -" + OPTION_SNAPSHOT_COUNT + " option shows snapshot counts.";
 
   private boolean showQuotas;
   private boolean humanReadable;
@@ -102,6 +104,7 @@ public static void registerCommands(CommandFactory factory) {
   private boolean showQuotasAndUsageOnly;
   private boolean excludeSnapshots;
   private boolean displayECPolicy;
+  private boolean showSnapshot;
 
   /** Constructor */
   public Count() {}
@@ -123,7 +126,7 @@ protected void processOptions(LinkedList<String> args) {
     CommandFormat cf = new CommandFormat(1, Integer.MAX_VALUE,
         OPTION_QUOTA, OPTION_HUMAN, OPTION_HEADER, OPTION_QUOTA_AND_USAGE,
         OPTION_EXCLUDE_SNAPSHOT,
-        OPTION_ECPOLICY);
+        OPTION_ECPOLICY, OPTION_SNAPSHOT_COUNT);
     cf.addOptionWithValue(OPTION_TYPE);
     cf.parse(args);
     if (args.isEmpty()) { // default path is the current working directory
@@ -134,6 +137,7 @@ protected void processOptions(LinkedList<String> args) {
     showQuotasAndUsageOnly = cf.getOpt(OPTION_QUOTA_AND_USAGE);
     excludeSnapshots = cf.getOpt(OPTION_EXCLUDE_SNAPSHOT);
     displayECPolicy = cf.getOpt(OPTION_ECPOLICY);
+    showSnapshot = cf.getOpt(OPTION_SNAPSHOT_COUNT);
 
     if (showQuotas || showQuotasAndUsageOnly) {
       String types = cf.getOptValue(OPTION_TYPE);
@@ -165,6 +169,9 @@ protected void processOptions(LinkedList<String> args) {
       if(displayECPolicy){
         headString.append("ERASURECODING_POLICY ");
       }
+      if (showSnapshot) {
+        headString.append(ContentSummary.getSnapshotHeader());
+      }
       headString.append("PATHNAME");
       out.println(headString.toString());
     }
@@ -205,6 +212,10 @@ protected void processPath(PathData src) throws IOException {
       outputString.append(summary.getErasureCodingPolicy())
           .append(" ");
     }
+    if (showSnapshot) {
+      ContentSummary summary = src.fs.getContentSummary(src.path);
+      outputString.append(summary.toSnapshot(isHumanReadable()));
+    }
     outputString.append(src);
     out.println(outputString.toString());
   }

hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md

@@ -145,7 +145,7 @@ Similar to get command, except that the destination is restricted to a local fil
 count
 -----
 
-Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t [<storage type>]] [-u] [-e] <paths> `
+Usage: `hadoop fs -count [-q] [-h] [-v] [-x] [-t [<storage type>]] [-u] [-e] [-s] <paths> `
 
 Count the number of directories, files and bytes under the paths that match the specified file pattern. Get the quota and the usage. The output columns with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
 
@@ -169,6 +169,8 @@ The output columns with -count -e are: DIR\_COUNT, FILE\_COUNT, CONTENT_SIZE, ER
 
 The ERASURECODING\_POLICY is name of the policy for the file. If a erasure coding policy is setted on that file, it will return name of the policy. If no erasure coding policy is setted, it will return \"Replicated\" which means it use replication storage strategy.
 
+The -s option shows the snapshot counts for each directory.
+
 Example:
 
 * `hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2`
@@ -179,6 +181,7 @@ Example:
 * `hadoop fs -count -u -h hdfs://nn1.example.com/file1`
 * `hadoop fs -count -u -h -v hdfs://nn1.example.com/file1`
 * `hadoop fs -count -e hdfs://nn1.example.com/file1`
+* `hadoop fs -count -s hdfs://nn1.example.com/file1`
 
 Exit Code:
 

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestContentSummary.java

@@ -253,4 +253,40 @@ public void testToStringHumanNoShowQuota() {
     String expected = "      32.6 K      211.9 M              8.0 E ";
     assertEquals(expected, contentSummary.toString(false, true));
   }
+
+  // check the toSnapshot method with human readable.
+  @Test
+  public void testToSnapshotHumanReadable() {
+    long snapshotLength = Long.MAX_VALUE;
+    long snapshotFileCount = 222222222;
+    long snapshotDirectoryCount = 33333;
+    long snapshotSpaceConsumed = 222256578;
+
+    ContentSummary contentSummary = new ContentSummary.Builder()
+        .snapshotLength(snapshotLength).snapshotFileCount(snapshotFileCount)
+        .snapshotDirectoryCount(snapshotDirectoryCount)
+        .snapshotSpaceConsumed(snapshotSpaceConsumed).build();
+    String expected =
+        "             8.0 E                  211.9 M                   32.6 K "
+            + "                     212.0 M ";
+    assertEquals(expected, contentSummary.toSnapshot(true));
+  }
+
+  // check the toSnapshot method with human readable disabled.
+  @Test
+  public void testToSnapshotNotHumanReadable() {
+    long snapshotLength = 1111;
+    long snapshotFileCount = 2222;
+    long snapshotDirectoryCount = 3333;
+    long snapshotSpaceConsumed = 4444;
+
+    ContentSummary contentSummary = new ContentSummary.Builder()
+        .snapshotLength(snapshotLength).snapshotFileCount(snapshotFileCount)
+        .snapshotDirectoryCount(snapshotDirectoryCount)
+        .snapshotSpaceConsumed(snapshotSpaceConsumed).build();
+    String expected =
+        "              1111                     2222                     3333 "
+            + "                        4444 ";
+    assertEquals(expected, contentSummary.toSnapshot(false));
+  }
 }

hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCount.java

@@ -411,6 +411,25 @@ public void processPathWithQuotasByMultipleStorageTypes() throws Exception {
     verifyNoMoreInteractions(out);
   }
 
+  @Test
+  public void processPathWithSnapshotHeader() throws Exception {
+    Path path = new Path("mockfs:/test");
+    when(mockFs.getFileStatus(eq(path))).thenReturn(fileStat);
+    PrintStream out = mock(PrintStream.class);
+    Count count = new Count();
+    count.out = out;
+    LinkedList<String> options = new LinkedList<String>();
+    options.add("-s");
+    options.add("-v");
+    options.add("dummy");
+    count.processOptions(options);
+    String withSnapshotHeader = "   DIR_COUNT   FILE_COUNT       CONTENT_SIZE "
+        + "   SNAPSHOT_LENGTH      SNAPSHOT_FILE_COUNT      "
+        + " SNAPSHOT_DIR_COUNT      SNAPSHOT_SPACE_CONSUMED PATHNAME";
+    verify(out).println(withSnapshotHeader);
+    verifyNoMoreInteractions(out);
+  }
+
   @Test
   public void getCommandName() {
     Count count = new Count();
@@ -448,7 +467,8 @@ public void getUsage() {
     Count count = new Count();
     String actual = count.getUsage();
     String expected =
-        "-count [-q] [-h] [-v] [-t [<storage type>]] [-u] [-x] [-e] <path> ...";
+        "-count [-q] [-h] [-v] [-t [<storage type>]]"
+        + " [-u] [-x] [-e] [-s] <path> ...";
     assertEquals("Count.getUsage", expected, actual);
   }
 
@@ -480,7 +500,8 @@ public void getDescription() {
         + "storage types.\n"
         + "The -u option shows the quota and \n"
         + "the usage against the quota without the detailed content summary."
-        + "The -e option shows the erasure coding policy.";
+        + "The -e option shows the erasure coding policy."
+        + "The -s option shows snapshot counts.";
 
     assertEquals("Count.getDescription", expected, actual);
   }

hadoop-common-project/hadoop-common/src/test/resources/testConf.xml

@@ -282,7 +282,7 @@
       <comparators>
         <comparator>
           <type>RegexpComparator</type>
-          <expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[&lt;storage type&gt;\]\] \[-u\] \[-x\] \[-e\] &lt;path&gt; \.\.\. :( )*</expected-output>
+          <expected-output>^-count \[-q\] \[-h\] \[-v\] \[-t \[&lt;storage type&gt;\]\] \[-u\] \[-x\] \[-e\] \[-s\] &lt;path&gt; \.\.\. :( )*</expected-output>
         </comparator>
         <comparator>
           <type>RegexpComparator</type>