Format reports on Golang CLI

cli/src/alluxio.org/cli/cmd/info/report.go

@@ -12,9 +12,17 @@
 package info
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
+	"io"
+	"math"
+	"os"
+	"strconv"
 	"strings"
+	"time"
 
+	"github.com/iancoleman/orderedmap"
 	"github.com/palantir/stacktrace"
 	"github.com/spf13/cobra"
 
@@ -32,7 +40,7 @@ var Report = &ReportCommand{
 
 type ReportCommand struct {
 	*env.BaseJavaCommand
-	format string
+	raw bool
 }
 
 func (c *ReportCommand) Base() *env.BaseJavaCommand {
@@ -57,8 +65,8 @@ Defaults to summary if no arg is provided
 			return c.Run(args)
 		},
 	})
-	cmd.Flags().StringVar(&c.format, "format", "json",
-		"Set output format, any of [json, yaml]")
+	cmd.Flags().BoolVar(&c.raw, "raw", false,
+		"Output raw JSON data instead of human-readable format for bytes, datetime, and duration.")
 	return cmd
 }
 
@@ -80,9 +88,95 @@ func (c *ReportCommand) Run(args []string) error {
 		}
 		reportArg = args[0]
 	}
-	// TODO: output all in a serializable format and filter/trim as specified by flags
-	if c.format != "json" && c.format != "yaml" {
-		return stacktrace.NewError("Invalid format %v, must be one of [json, yaml]", c.format)
+
+	buf := &bytes.Buffer{}
+	if err := c.RunWithIO([]string{reportArg}, nil, buf, os.Stderr); err != nil {
+		io.Copy(os.Stdout, buf)
+		return err
+	}
+
+	obj := orderedmap.New()
+	if err := obj.UnmarshalJSON(buf.Bytes()); err != nil {
+		return stacktrace.Propagate(err, "error unmarshalling json from java command")
+	}
+
+	if !c.raw {
+		newObj := orderedmap.New()
+		for _, key := range obj.Keys() {
+			if val, ok := obj.Get(key); ok {
+				k, v := processKeyValuePair(key, val)
+				newObj.Set(k, v)
+			}
+		}
+		obj = newObj
+	}
+
+	prettyJson, err := json.MarshalIndent(obj, "", "    ")
+	if err != nil {
+		return stacktrace.Propagate(err, "error marshalling json to pretty format")
+	}
+	os.Stdout.Write(append(prettyJson, '\n'))
+	return nil
+}
+
+func convertMsToDatetime(timeMs float64) string {
+	tm := time.Unix(int64(timeMs)/1000, 0)
+	return tm.Format(time.RFC3339)
+}
+
+func convertMsToDuration(timeMs float64) string {
+	duration := time.Duration(int64(timeMs)) * time.Millisecond
+	days := duration / (24 * time.Hour)
+	duration = duration % (24 * time.Hour)
+	hours := duration / time.Hour
+	duration = duration % time.Hour
+	minutes := duration / time.Minute
+	seconds := duration % time.Minute / time.Second
+	return fmt.Sprintf("%dd %02dh%02dm%02ds", days, hours, minutes, seconds)
+}
+
+func convertBytesToString(bytes float64) string {
+	const unit = 1024
+	suffixes := []string{"B", "KB", "MB", "GB", "TB", "PB"}
+	exp, n := 0, int64(bytes)
+	for n > 5*unit && exp < len(suffixes)-1 {
+		n /= unit
+		exp++
+	}
+	value := bytes / math.Pow(unit, float64(exp))
+	size := strconv.FormatFloat(value, 'f', 2, 64)
+	return size + suffixes[exp]
+}
+
+func processKeyValuePair(key string, data interface{}) (string, interface{}) {
+	switch value := data.(type) {
+	case float64:
+		if strings.HasSuffix(key, "Time") {
+			return strings.TrimSuffix(key, "Time"), convertMsToDatetime(value)
+		} else if strings.HasSuffix(key, "Duration") {
+			return strings.TrimSuffix(key, "Duration"), convertMsToDuration(value)
+		} else if strings.HasSuffix(key, "Bytes") {
+			return strings.TrimSuffix(key, "Bytes"), convertBytesToString(value)
+		} else {
+			return key, value
+		}
+	case []interface{}:
+		array := make([]interface{}, len(value))
+		for i, item := range value {
+			_, processedItem := processKeyValuePair(key, item)
+			array[i] = processedItem
+		}
+		return key, array
+	case orderedmap.OrderedMap:
+		processedMap := orderedmap.New()
+		for _, key := range value.Keys() {
+			if val, ok := value.Get(key); ok {
+				k, v := processKeyValuePair(key, val)
+				processedMap.Set(k, v)
+			}
+		}
+		return key, processedMap
+	default:
+		return key, value
 	}
-	return c.Base().RunAndFormat(c.format, nil, []string{reportArg})
 }

cli/src/alluxio.org/go.mod

@@ -3,6 +3,7 @@ module alluxio.org
 go 1.18
 
 require (
+	github.com/iancoleman/orderedmap v0.3.0
 	github.com/palantir/stacktrace v0.0.0-20161112013806-78658fd2d177
 	github.com/shirou/gopsutil v3.21.11+incompatible
 	github.com/sirupsen/logrus v1.9.0

cli/src/alluxio.org/go.sum

@@ -123,6 +123,8 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJn+Ichc=
+github.com/iancoleman/orderedmap v0.3.0/go.mod h1:XuLcCUkdL5owUCQeF2Ue9uuw1EptkJDkXXS7VoV7XGE=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=

dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java

@@ -23,11 +23,7 @@
 
 import java.io.IOException;
 import java.io.PrintStream;
-import java.time.ZoneId;
-import java.time.format.DateTimeFormatter;
-import java.time.format.FormatStyle;
 import java.util.List;
-import java.util.Locale;
 
 /**
  * Prints job service metric information.
@@ -53,10 +49,6 @@ public JobServiceMetricsCommand(JobMasterClient JobMasterClient, PrintStream pri
     mDateFormatPattern = dateFormatPattern;
   }
 
-  public static final DateTimeFormatter DATETIME_FORMAT =
-      DateTimeFormatter.ofLocalizedDateTime(FormatStyle.SHORT).ofPattern("yyyyMMdd-HHmmss")
-          .withLocale(Locale.getDefault()).withZone(ZoneId.systemDefault());
-
   /**
    * Runs a job services report metrics command.
    *
@@ -68,8 +60,8 @@ public int run() throws IOException {
     List<JobWorkerHealth> allWorkerHealth = mJobMasterClient.getAllWorkerHealth();
     JobServiceSummary jobServiceSummary = mJobMasterClient.getJobServiceSummary();
 
-    JobServiceOutput jobServiceInfo = new JobServiceOutput(allMasterStatus, allWorkerHealth,
-        jobServiceSummary, mDateFormatPattern);
+    JobServiceOutput jobServiceInfo = new JobServiceOutput(
+        allMasterStatus, allWorkerHealth, jobServiceSummary);
     try {
       String json = objectMapper.writeValueAsString(jobServiceInfo);
       mPrintStream.println(json);

dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceOutput.java

@@ -16,7 +16,6 @@
 import alluxio.job.wire.JobServiceSummary;
 import alluxio.job.wire.JobWorkerHealth;
 import alluxio.job.wire.StatusSummary;
-import alluxio.util.CommonUtils;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -36,15 +35,15 @@ private static class SerializableJobMasterStatus {
     private String mHost;
     private int mPort;
     private String mState;
-    private String mStartTime;
+    private long mStartTime;
     private String mVersion;
     private String mRevision;
 
-    public SerializableJobMasterStatus(JobMasterStatus jobMasterStatus, String dateFormat) {
+    public SerializableJobMasterStatus(JobMasterStatus jobMasterStatus) {
       mHost = jobMasterStatus.getMasterAddress().getHost();
       mPort = jobMasterStatus.getMasterAddress().getRpcPort();
       mState = jobMasterStatus.getState();
-      mStartTime = CommonUtils.convertMsToDate(jobMasterStatus.getStartTime(), dateFormat);
+      mStartTime = jobMasterStatus.getStartTime();
       mVersion = jobMasterStatus.getVersion().getVersion();
       mRevision = jobMasterStatus.getVersion().getRevision();
     }
@@ -73,11 +72,11 @@ public void setState(String state) {
       mState = state;
     }
 
-    public String getStartTime() {
+    public long getStartTime() {
       return mStartTime;
     }
 
-    public void setStartTime(String startTime) {
+    public void setStartTime(long startTime) {
       mStartTime = startTime;
     }
 
@@ -201,24 +200,24 @@ public void setCount(long count) {
   }
 
   private static class SerializableJobInfo {
-    private String mTimestamp;
+    private long mLastUpdatedTime;
     private long mId;
     private String mName;
     private String mStatus;
 
-    public SerializableJobInfo(JobInfo jobInfo, String dateFormat) {
-      mTimestamp = CommonUtils.convertMsToDate(jobInfo.getLastUpdated(), dateFormat);
+    public SerializableJobInfo(JobInfo jobInfo) {
+      mLastUpdatedTime = jobInfo.getLastUpdated();
       mId = jobInfo.getId();
       mName = jobInfo.getName();
       mStatus = jobInfo.getStatus().toString();
     }
 
-    public String getTimestamp() {
-      return mTimestamp;
+    public long getLastUpdatedTime() {
+      return mLastUpdatedTime;
     }
 
-    public void setTimestamp(String timestamp) {
-      mTimestamp = timestamp;
+    public void setLastUpdatedTime(long lastUpdatedTime) {
+      mLastUpdatedTime = lastUpdatedTime;
     }
 
     public long getId() {
@@ -252,15 +251,13 @@ public void setStatus(String status) {
    * @param allMasterStatus status of all masters
    * @param allWorkerHealth health info of all workers
    * @param jobServiceSummary summary of job service
-   * @param dateFormat specify the pattern of dates
    */
   public JobServiceOutput(List<JobMasterStatus> allMasterStatus,
                           List<JobWorkerHealth> allWorkerHealth,
-                          JobServiceSummary jobServiceSummary,
-                          String dateFormat) {
+                          JobServiceSummary jobServiceSummary) {
     mMasterStatus = new ArrayList<>();
     for (JobMasterStatus masterStatus : allMasterStatus) {
-      mMasterStatus.add(new SerializableJobMasterStatus(masterStatus, dateFormat));
+      mMasterStatus.add(new SerializableJobMasterStatus(masterStatus));
     }
 
     mWorkerHealth = new ArrayList<>();
@@ -276,13 +273,13 @@ public JobServiceOutput(List<JobMasterStatus> allMasterStatus,
     mRecentFailedJobs = new ArrayList<>();
     mLongestRunningJobs = new ArrayList<>();
     for (JobInfo jobInfo : jobServiceSummary.getRecentActivities()) {
-      mRecentModifiedJobs.add(new SerializableJobInfo(jobInfo, dateFormat));
+      mRecentModifiedJobs.add(new SerializableJobInfo(jobInfo));
     }
     for (JobInfo jobInfo : jobServiceSummary.getRecentFailures()) {
-      mRecentFailedJobs.add(new SerializableJobInfo(jobInfo, dateFormat));
+      mRecentFailedJobs.add(new SerializableJobInfo(jobInfo));
     }
     for (JobInfo jobInfo : jobServiceSummary.getLongestRunning()) {
-      mLongestRunningJobs.add(new SerializableJobInfo(jobInfo, dateFormat));
+      mLongestRunningJobs.add(new SerializableJobInfo(jobInfo));
     }
   }
 

dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java

@@ -78,7 +78,7 @@ public int run() throws IOException {
     BlockMasterInfo blockMasterInfo = mBlockMasterClient.getBlockMasterInfo(blockMasterInfoFilter);
 
     ObjectMapper objectMapper = new ObjectMapper();
-    SummaryOutput summaryInfo = new SummaryOutput(masterInfo, blockMasterInfo, mDateFormatPattern);
+    SummaryOutput summaryInfo = new SummaryOutput(masterInfo, blockMasterInfo);
     try {
       String json = objectMapper.writeValueAsString(summaryInfo);
       mPrintStream.println(json);