From 9d7a9233dda59737603c12655d8411b5f691f5c7 Mon Sep 17 00:00:00 2001 From: twalluxio Date: Tue, 22 Aug 2023 09:56:46 +0800 Subject: [PATCH 01/21] json-style capacity report --- .../cli/fsadmin/report/CapacityCommand.java | 189 ++++++++---------- 1 file changed, 83 insertions(+), 106 deletions(-) diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/CapacityCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/CapacityCommand.java index 908e705406e9..4e529f990156 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/CapacityCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/CapacityCommand.java @@ -28,7 +28,10 @@ import alluxio.util.FormatUtils; import alluxio.wire.WorkerInfo; -import com.google.common.base.Strings; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.cli.CommandLine; import java.io.IOException; @@ -51,11 +54,8 @@ * Prints Alluxio capacity information. */ public class CapacityCommand { - private static final int INDENT_SIZE = 4; - private BlockMasterClient mBlockMasterClient; private PrintStream mPrintStream; - private int mIndentationLevel = 0; private long mSumCapacityBytes; private long mSumUsedBytes; private Map mSumCapacityBytesOnTierMap; @@ -63,6 +63,8 @@ public class CapacityCommand { private TreeMap> mCapacityTierInfoMap; private Map> mUsedTierInfoMap; + private ObjectMapper mMapper; + private ObjectNode mCapacityInfo; private static final String LIVE_WORKER_STATE = "In Service"; private static final String LOST_WORKER_STATE = "Out of Service"; @@ -118,17 +120,20 @@ public void generateCapacityReport(GetWorkerReportOptions options, AlluxioConfig workerInfoList = mBlockMasterClient.getWorkerReport(options); } if (workerInfoList.size() == 0) { - print("No workers found."); + mPrintStream.println("No workers found."); return; } Collections.sort(workerInfoList, new WorkerInfo.LastContactSecComparator()); collectWorkerInfo(workerInfoList); - printAggregatedInfo(options); - printWorkerInfo(workerInfoList); + generateAggregatedInfo(options); + generateWorkerInfo(workerInfoList); if (workerRegisterToAllMasters) { printWorkerAllMasterConnectionInfo(allMastersWorkerInfo); } + mPrintStream.printf("Capacity information for %s workers:%n", + options.getWorkerRange().toString().toLowerCase()); + mPrintStream.println(mMapper.writerWithDefaultPrettyPrinter().writeValueAsString(mCapacityInfo)); } /** @@ -177,36 +182,44 @@ private void collectWorkerInfo(List workerInfoList) { * * @param options GetWorkerReportOptions to check if input is invalid */ - private void printAggregatedInfo(GetWorkerReportOptions options) { - mIndentationLevel = 0; - print(String.format("Capacity information for %s workers: ", - options.getWorkerRange().toString().toLowerCase())); - - mIndentationLevel++; - print("Total Capacity: " + FormatUtils.getSizeFromBytes(mSumCapacityBytes)); - mIndentationLevel++; + private void generateAggregatedInfo(GetWorkerReportOptions options) throws JsonProcessingException { + ObjectNode capacityMetrics = mMapper.createObjectNode(); + ObjectNode totalCapacity = mMapper.createObjectNode(); + ObjectNode usedCapacity = mMapper.createObjectNode(); + + ArrayNode totalCapacityTiers = mMapper.createArrayNode(); for (Map.Entry totalBytesTier : mSumCapacityBytesOnTierMap.entrySet()) { long value = totalBytesTier.getValue(); - print("Tier: " + totalBytesTier.getKey() - + " Size: " + FormatUtils.getSizeFromBytes(value)); + ObjectNode totalCapacityPerTier = mMapper.createObjectNode(); + totalCapacityPerTier.put("Tier", totalBytesTier.getKey()); + totalCapacityPerTier.put("Size", FormatUtils.getSizeFromBytes(value)); + totalCapacityTiers.add(totalCapacityPerTier); } - mIndentationLevel--; - print("Used Capacity: " - + FormatUtils.getSizeFromBytes(mSumUsedBytes)); - mIndentationLevel++; + ArrayNode usedCapacityTiers = mMapper.createArrayNode(); for (Map.Entry usedBytesTier : mSumUsedBytesOnTierMap.entrySet()) { long value = usedBytesTier.getValue(); - print("Tier: " + usedBytesTier.getKey() - + " Size: " + FormatUtils.getSizeFromBytes(value)); + ObjectNode usedCapacityPerTier = mMapper.createObjectNode(); + usedCapacityPerTier.put("Tier", usedBytesTier.getKey()); + usedCapacityPerTier.put("Size", FormatUtils.getSizeFromBytes(value)); + usedCapacityTiers.add(usedCapacityPerTier); } - mIndentationLevel--; + + totalCapacity.put("All", FormatUtils.getSizeFromBytes(mSumCapacityBytes)); + totalCapacity.set("Tiers", totalCapacityTiers); + usedCapacity.put("All", FormatUtils.getSizeFromBytes(mSumUsedBytes)); + usedCapacity.set("Tiers", usedCapacityTiers); + + capacityMetrics.set("Total Capacity", totalCapacity); + capacityMetrics.set("Used Capacity", usedCapacity); if (mSumCapacityBytes != 0) { int usedPercentage = (int) (100L * mSumUsedBytes / mSumCapacityBytes); - print(String.format("Used Percentage: " + "%s%%", usedPercentage)); - print(String.format("Free Percentage: " + "%s%%", 100 - usedPercentage)); + capacityMetrics.put("Used Percentage", String.format("%s%%", usedPercentage)); + capacityMetrics.put("Free Percentage", String.format("%s%%", 100 - usedPercentage)); } + + mCapacityInfo.set("Capacity Metrics", capacityMetrics); } private String getMasterAddressesString(Set addresses) { @@ -296,96 +309,58 @@ private void printWorkerAllMasterConnectionInfo( * * @param workerInfoList the worker info list to get info from */ - private void printWorkerInfo(List workerInfoList) { - mIndentationLevel = 0; + private void generateWorkerInfo(List workerInfoList) { + boolean isShort = false; if (mCapacityTierInfoMap.size() == 0) { return; } else if (mCapacityTierInfoMap.size() == 1) { - // TODO(jiacheng): test BOTH long and short output - // Do not print Total value when only one tier exists - printShortWorkerInfo(workerInfoList); - return; + isShort = true; } + + ArrayNode workerInfo = mMapper.createArrayNode(); Set tiers = mCapacityTierInfoMap.keySet(); - String tiersInfo = String.format(Strings.repeat("%-14s", tiers.size()), tiers.toArray()); - String longInfoFormat = getInfoFormat(workerInfoList, false); - print(String.format("%n" + longInfoFormat, - "Worker Name", "State", "Last Heartbeat", "Storage", "Total", tiersInfo, - "Version", "Revision")); for (WorkerInfo info : workerInfoList) { - String workerName = info.getAddress().getHost(); - - long usedBytes = info.getUsedBytes(); - long capacityBytes = info.getCapacityBytes(); - - String usedPercentageInfo = ""; - if (capacityBytes != 0) { - int usedPercentage = (int) (100L * usedBytes / capacityBytes); - usedPercentageInfo = String.format(" (%s%%)", usedPercentage); + ObjectNode infoPerWorker = mMapper.createObjectNode(); + + ArrayNode tiersInfo = mMapper.createArrayNode(); + + if (isShort) { + // TODO(jiacheng): test BOTH long and short output + // Do not print Total value when only one tier exists + ObjectNode tierInfo = mMapper.createObjectNode(); + tierInfo.put("Tier", mCapacityTierInfoMap.firstKey()); + tierInfo.put("Capacity", FormatUtils.getSizeFromBytes(info.getCapacityBytes())); + tierInfo.put("Used", FormatUtils.getSizeFromBytes(info.getUsedBytes())); + if (info.getCapacityBytes() != 0) { + tierInfo.put("Used percentage", String.format(" %s%%", (int) (100L * info.getUsedBytes() / info.getCapacityBytes()))); + } + tiersInfo.add(tierInfo); + } else { + for (String tier : tiers){ + ObjectNode tierInfo = mMapper.createObjectNode(); + tierInfo.put("Tier", tier); + tierInfo.put("Capacity", getWorkerFormattedTierValues(mCapacityTierInfoMap, info.getAddress().getHost())); + tierInfo.put("Used", getWorkerFormattedTierValues(mUsedTierInfoMap, info.getAddress().getHost())); + if (info.getCapacityBytes() != 0) { + tierInfo.put("Used percentage", + String.format("%s%%", (int) (100L * info.getUsedBytes() / info.getCapacityBytes()))); + } + tiersInfo.add(tierInfo); + } } - String capacityTierInfo = getWorkerFormattedTierValues(mCapacityTierInfoMap, workerName); - String usedTierInfo = getWorkerFormattedTierValues(mUsedTierInfoMap, workerName); - - print(String.format(longInfoFormat, workerName, info.getState(), - info.getLastContactSec(), "capacity", - FormatUtils.getSizeFromBytes(capacityBytes), capacityTierInfo, - info.getVersion(), info.getRevision())); - print(String.format(longInfoFormat, "", "", "", "used", - FormatUtils.getSizeFromBytes(usedBytes) + usedPercentageInfo, usedTierInfo, - "", "")); - } - } - - /** - * Prints worker information when only one tier exists. - * - * @param workerInfoList the worker info list to get info from - */ - private void printShortWorkerInfo(List workerInfoList) { - String tier = String.format("%-16s", mCapacityTierInfoMap.firstKey()); - String shortInfoFormat = getInfoFormat(workerInfoList, true); - print(String.format("%n" + shortInfoFormat, - "Worker Name", "State", "Last Heartbeat", "Storage", tier, "Version", "Revision")); - - for (WorkerInfo info : workerInfoList) { - long capacityBytes = info.getCapacityBytes(); - long usedBytes = info.getUsedBytes(); + infoPerWorker.put("Worker Name", info.getAddress().getHost()); + infoPerWorker.put("State", info.getState()); + infoPerWorker.put("Last Heartbeat", info.getLastContactSec()); + infoPerWorker.set("tiers", tiersInfo); + infoPerWorker.put("Version", info.getVersion()); + infoPerWorker.put("Revision", info.getRevision()); - String usedPercentageInfo = ""; - if (capacityBytes != 0) { - int usedPercentage = (int) (100L * usedBytes / capacityBytes); - usedPercentageInfo = String.format(" (%s%%)", usedPercentage); - } - print(String.format(shortInfoFormat, info.getAddress().getHost(), info.getState(), - info.getLastContactSec(), "capacity", - String.format("%-16s", FormatUtils.getSizeFromBytes(capacityBytes)), - info.getVersion(), info.getRevision())); - print(String.format(shortInfoFormat, "", "", "", "used", - String.format("%-16s", FormatUtils.getSizeFromBytes(usedBytes) + usedPercentageInfo), - "", "")); + workerInfo.add(infoPerWorker); } - } - /** - * Gets the info format according to the longest worker name. - * @param workerInfoList the worker info list to get info from - * @param isShort whether exists only one tier - * @return the info format for printing long/short worker info - */ - private String getInfoFormat(List workerInfoList, boolean isShort) { - int maxWorkerNameLength = workerInfoList.stream().map(w -> w.getAddress().getHost().length()) - .max(Comparator.comparing(Integer::intValue)).get(); - int firstIndent = 16; - if (firstIndent <= maxWorkerNameLength) { - // extend first indent according to the longest worker name by default 5 - firstIndent = maxWorkerNameLength + 5; - } - if (isShort) { - return "%-" + firstIndent + "s %-15s %-16s %-13s %s %-16s %-40s"; - } - return "%-" + firstIndent + "s %-15s %-16s %-13s %-16s %s %-16s %-40s"; + mCapacityInfo.set("Worker Information", workerInfo); } /** @@ -450,6 +425,9 @@ private void initVariables() { // TierInfoMap is of form Map> mCapacityTierInfoMap = new TreeMap<>(FileSystemAdminShellUtils::compareTierNames); mUsedTierInfoMap = new TreeMap<>(FileSystemAdminShellUtils::compareTierNames); + + mMapper = new ObjectMapper(); + mCapacityInfo = mMapper.createObjectNode(); } /** @@ -458,8 +436,7 @@ private void initVariables() { * @param text information to print */ private void print(String text) { - String indent = Strings.repeat(" ", mIndentationLevel * INDENT_SIZE); - mPrintStream.println(indent + text); + mPrintStream.println(text); } /** From e566134dc5fca9f559d31b7223dab10b256c28b2 Mon Sep 17 00:00:00 2001 From: twalluxio Date: Tue, 22 Aug 2023 15:11:30 +0800 Subject: [PATCH 02/21] json-style reports --- .../report/JobServiceMetricsCommand.java | 131 ++++++-------- .../cli/fsadmin/report/MetricsCommand.java | 15 +- .../cli/fsadmin/report/SummaryCommand.java | 167 ++++++++---------- .../cli/fsadmin/report/UfsCommand.java | 64 +++---- 4 files changed, 174 insertions(+), 203 deletions(-) diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java index 030f63203405..b9149cfe51ca 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java @@ -20,6 +20,9 @@ import alluxio.job.wire.StatusSummary; import alluxio.util.CommonUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.commons.lang3.StringUtils; import java.io.IOException; @@ -66,101 +69,75 @@ public JobServiceMetricsCommand(JobMasterClient JobMasterClient, PrintStream pri * @return 0 on success, 1 otherwise */ public int run() throws IOException { + ObjectMapper mapper = new ObjectMapper(); + ObjectNode jobServiceInfo = mapper.createObjectNode(); + + ArrayNode masterInfo = mapper.createArrayNode(); List allMasterStatus = mJobMasterClient.getAllMasterStatus(); - String masterFormat = getMasterInfoFormat(allMasterStatus); - mPrintStream.printf(masterFormat, "Master Address", "State", "Start Time", - "Version", "Revision"); for (JobMasterStatus masterStatus : allMasterStatus) { - NetAddress address = masterStatus.getMasterAddress(); - mPrintStream.printf(masterFormat, - address.getHost() + ":" + address.getRpcPort(), - masterStatus.getState(), - DATETIME_FORMAT.format(Instant.ofEpochMilli(masterStatus.getStartTime())), - masterStatus.getVersion().getVersion(), - masterStatus.getVersion().getRevision()); + ObjectNode master = mapper.createObjectNode(); + master.put("Host", masterStatus.getMasterAddress().getHost()); + master.put("Port", masterStatus.getMasterAddress().getRpcPort()); + master.put("State", masterStatus.getState()); + master.put("Start Time", DATETIME_FORMAT.format(Instant.ofEpochMilli(masterStatus.getStartTime()))); + master.put("Version", masterStatus.getVersion().getVersion()); + master.put("Revision", masterStatus.getVersion().getRevision()); + masterInfo.add(master); } - mPrintStream.println(); + jobServiceInfo.set("Masters", masterInfo); + ArrayNode workerInfo = mapper.createArrayNode(); List allWorkerHealth = mJobMasterClient.getAllWorkerHealth(); - String workerFormat = getWorkerInfoFormat(allWorkerHealth); - mPrintStream.printf(workerFormat, "Job Worker", "Version", "Revision", "Task Pool Size", - "Unfinished Tasks", "Active Tasks", "Load Avg"); - for (JobWorkerHealth workerHealth : allWorkerHealth) { - mPrintStream.printf(workerFormat, - workerHealth.getHostname(), workerHealth.getVersion().getVersion(), - workerHealth.getVersion().getRevision(), - workerHealth.getTaskPoolSize(), workerHealth.getUnfinishedTasks(), - workerHealth.getNumActiveTasks(), - StringUtils.join(workerHealth.getLoadAverage(), ", ")); + ObjectNode worker = mapper.createObjectNode(); + worker.put("Host", workerHealth.getHostname()); + worker.put("Version", workerHealth.getVersion().getVersion()); + worker.put("Revision", workerHealth.getVersion().getRevision()); + worker.put("Task Pool Size", workerHealth.getTaskPoolSize()); + worker.put("Unfinished Tasks", workerHealth.getUnfinishedTasks()); + worker.put("Active Tasks", workerHealth.getNumActiveTasks()); + ObjectNode lAverage = mapper.createObjectNode(); + List loadAverage = workerHealth.getLoadAverage(); + lAverage.put("1 minute", loadAverage.get(0)); + lAverage.put("5 minutes", loadAverage.get(1)); + lAverage.put("15 minutes", loadAverage.get(2)); + worker.set("Load Average", lAverage); + workerInfo.add(worker); } - mPrintStream.println(); + jobServiceInfo.set("Workers", workerInfo); + ArrayNode jobStatusInfo = mapper.createArrayNode(); JobServiceSummary jobServiceSummary = mJobMasterClient.getJobServiceSummary(); - Collection jobStatusSummaries = jobServiceSummary.getSummaryPerStatus(); - for (StatusSummary statusSummary : jobStatusSummaries) { - mPrintStream.print(String.format("Status: %-10s", statusSummary.getStatus())); - mPrintStream.println(String.format("Count: %s", statusSummary.getCount())); + ObjectNode status = mapper.createObjectNode(); + status.put("Status", statusSummary.getStatus().toString()); + status.put("Count", statusSummary.getCount()); + jobStatusInfo.add(status); } + jobServiceInfo.set("Job Status", jobStatusInfo); - mPrintStream.println(); - mPrintStream.println(String.format("%s Most Recently Modified Jobs:", - JobServiceSummary.RECENT_LENGTH)); - - List lastActivities = jobServiceSummary.getRecentActivities(); - printJobInfos(lastActivities); - - mPrintStream.println(String.format("%s Most Recently Failed Jobs:", - JobServiceSummary.RECENT_LENGTH)); - - List lastFailures = jobServiceSummary.getRecentFailures(); - printJobInfos(lastFailures); - - mPrintStream.println(String.format("%s Longest Running Jobs:", - JobServiceSummary.RECENT_LENGTH)); - - List longestRunning = jobServiceSummary.getLongestRunning(); - printJobInfos(longestRunning); + ObjectNode recentJobInfo = mapper.createObjectNode(); + recentJobInfo.set("Recent Modified", getJobInfos(jobServiceSummary.getRecentActivities())); + recentJobInfo.set("Recent Failed", getJobInfos(jobServiceSummary.getRecentFailures())); + recentJobInfo.set("Longest running", getJobInfos(jobServiceSummary.getLongestRunning())); + jobServiceInfo.set(String.format("Top %s jobs", JobServiceSummary.RECENT_LENGTH), recentJobInfo); + mPrintStream.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jobServiceInfo)); return 0; } - private String getMasterInfoFormat(List masters) { - int maxNameLength = 16; - if (masters.size() > 0) { - maxNameLength = masters.stream().map(m -> m.getMasterAddress().getHost().length() + 6) - .max(Comparator.comparing(Integer::intValue)).get(); - } - // hostname:port + state + startTime + version + revision - return "%-" + maxNameLength + "s %-8s %-16s %-32s %-8s%n"; - } - - private String getWorkerInfoFormat(List workers) { - int maxNameLength = 16; - if (workers.size() > 0) { - maxNameLength = workers.stream().map(w -> w.getHostname().length()) - .max(Comparator.comparing(Integer::intValue)).get(); - } - int firstIndent = 16; - if (firstIndent <= maxNameLength) { - // extend first indent according to the longest worker name - firstIndent = maxNameLength + 6; - } - - // hostname + version + revision + poolSize + unfinishedTasks + activeTasks + loadAvg - return "%-" + firstIndent + "s %-32s %-8s %-14s %-16s %-12s %s%n"; - } - - private void printJobInfos(List jobInfos) { + private ArrayNode getJobInfos(List jobInfos) { + ObjectMapper mapper = new ObjectMapper(); + ArrayNode result = mapper.createArrayNode(); for (JobInfo jobInfo : jobInfos) { - mPrintStream.print(String.format("Timestamp: %-30s", - CommonUtils.convertMsToDate(jobInfo.getLastUpdated(), mDateFormatPattern))); - mPrintStream.print(String.format("Id: %-20s", jobInfo.getId())); - mPrintStream.print(String.format("Name: %-20s", jobInfo.getName())); - mPrintStream.println(String.format("Status: %s", jobInfo.getStatus())); + ObjectNode jInfo = mapper.createObjectNode(); + jInfo.put("Timestamp", CommonUtils.convertMsToDate(jobInfo.getLastUpdated(), mDateFormatPattern)); + jInfo.put("Id", jobInfo.getId()); + jInfo.put("Name", jobInfo.getName()); + jInfo.put("Status", jobInfo.getStatus().toString()); + result.add(jInfo); } - mPrintStream.println(); + return result; } } diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java index de0b93b9d1d7..efef5cd188e0 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java @@ -16,6 +16,9 @@ import alluxio.metrics.MetricsSystem; import alluxio.util.FormatUtils; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.math.DoubleMath; import java.io.IOException; @@ -35,7 +38,6 @@ public class MetricsCommand { private static final String THROUGHPUT_METRIC_IDENTIFIER = "Throughput"; private static final DecimalFormat DECIMAL_FORMAT = new DecimalFormat("###,###.#####", new DecimalFormatSymbols(Locale.US)); - private static final String INFO_FORMAT = "%s (Type: %s, Value: %s)%n"; private final MetricsMasterClient mMetricsMasterClient; private final PrintStream mPrintStream; @@ -58,6 +60,9 @@ public MetricsCommand(MetricsMasterClient metricsMasterClient, PrintStream print * @return 0 on success, 1 otherwise */ public int run() throws IOException { + ObjectMapper mapper = new ObjectMapper(); + ArrayNode metricsInfo = mapper.createArrayNode(); + mMetricsMap = mMetricsMasterClient.getMetrics(); SortedSet names = new TreeSet<>(mMetricsMap.keySet()); for (String name : names) { @@ -82,8 +87,14 @@ public int run() throws IOException { strValue = String.valueOf(doubleValue); } } - mPrintStream.printf(INFO_FORMAT, name, metricValue.getMetricType(), strValue); + ObjectNode metric = mapper.createObjectNode(); + metric.put("Name", name); + metric.put("Type", metricValue.getMetricType().toString()); + metric.put("Value", strValue); + metricsInfo.add(metric); } + + mPrintStream.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(metricsInfo)); return 0; } diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java index 7cf44993e8a7..d96501661e2a 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java @@ -23,6 +23,9 @@ import alluxio.wire.BlockMasterInfo; import alluxio.wire.BlockMasterInfo.BlockMasterInfoField; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.base.Strings; import java.io.IOException; @@ -38,12 +41,9 @@ * Prints Alluxio cluster summarized information. */ public class SummaryCommand { - private static final int INDENT_SIZE = 4; - - private int mIndentationLevel = 0; - private MetaMasterClient mMetaMasterClient; - private BlockMasterClient mBlockMasterClient; - private PrintStream mPrintStream; + private final MetaMasterClient mMetaMasterClient; + private final BlockMasterClient mBlockMasterClient; + private final PrintStream mPrintStream; private final String mDateFormatPattern; /** @@ -68,121 +68,110 @@ public SummaryCommand(MetaMasterClient metaMasterClient, * @return 0 on success, 1 otherwise */ public int run() throws IOException { - print("Alluxio cluster summary: "); - printMetaMasterInfo(); - printBlockMasterInfo(); - return 0; - } + ObjectMapper mapper = new ObjectMapper(); + ObjectNode summeryInfo = mapper.createObjectNode(); - /** - * Prints Alluxio meta master information. - */ - private void printMetaMasterInfo() throws IOException { - mIndentationLevel++; Set masterInfoFilter = new HashSet<>(Arrays - .asList(MasterInfoField.LEADER_MASTER_ADDRESS, MasterInfoField.WEB_PORT, - MasterInfoField.RPC_PORT, MasterInfoField.START_TIME_MS, - MasterInfoField.UP_TIME_MS, MasterInfoField.VERSION, - MasterInfoField.SAFE_MODE, MasterInfoField.ZOOKEEPER_ADDRESSES, - MasterInfoField.RAFT_JOURNAL, MasterInfoField.RAFT_ADDRESSES, - MasterInfoField.MASTER_VERSION)); + .asList(MasterInfoField.LEADER_MASTER_ADDRESS, MasterInfoField.WEB_PORT, + MasterInfoField.RPC_PORT, MasterInfoField.START_TIME_MS, + MasterInfoField.UP_TIME_MS, MasterInfoField.VERSION, + MasterInfoField.SAFE_MODE, MasterInfoField.ZOOKEEPER_ADDRESSES, + MasterInfoField.RAFT_JOURNAL, MasterInfoField.RAFT_ADDRESSES, + MasterInfoField.MASTER_VERSION)); MasterInfo masterInfo = mMetaMasterClient.getMasterInfo(masterInfoFilter); - print("Master Address: " + masterInfo.getLeaderMasterAddress()); - print("Web Port: " + masterInfo.getWebPort()); - print("Rpc Port: " + masterInfo.getRpcPort()); - print("Started: " + CommonUtils.convertMsToDate(masterInfo.getStartTimeMs(), - mDateFormatPattern)); - print("Uptime: " + CommonUtils.convertMsToClockTime(masterInfo.getUpTimeMs())); - print("Version: " + masterInfo.getVersion()); - print("Safe Mode: " + masterInfo.getSafeMode()); + summeryInfo.put("Master Address", masterInfo.getLeaderMasterAddress()); + summeryInfo.put("Web Port", masterInfo.getWebPort()); + summeryInfo.put("Rpc Port", masterInfo.getRpcPort()); + summeryInfo.put("Started", CommonUtils.convertMsToDate(masterInfo.getStartTimeMs(), mDateFormatPattern)); + summeryInfo.put("Uptime", CommonUtils.convertMsToClockTime(masterInfo.getUpTimeMs())); + summeryInfo.put("Version", masterInfo.getVersion()); + summeryInfo.put("Safe Mode", masterInfo.getSafeMode()); + ObjectNode zooKeeper = mapper.createObjectNode(); List zookeeperAddresses = masterInfo.getZookeeperAddressesList(); + ArrayNode zkAddresses = mapper.createArrayNode(); if (zookeeperAddresses == null || zookeeperAddresses.isEmpty()) { - print("Zookeeper Enabled: false"); + zooKeeper.put("Enabled", "false"); } else { - print("Zookeeper Enabled: true"); - print("Zookeeper Addresses: "); - mIndentationLevel++; - for (String zkAddress : zookeeperAddresses) { - print(zkAddress); + zooKeeper.put("Enabled", "true"); + for (String zookeeperAddress : zookeeperAddresses) { + zkAddresses.add(zookeeperAddress); } - mIndentationLevel--; } + zooKeeper.set("Addresses", zkAddresses); + summeryInfo.set("Zookeeper", zooKeeper); - if (masterInfo.getRaftJournal()) { - print("Raft-based Journal: true"); - print("Raft Journal Addresses: "); - mIndentationLevel++; + ObjectNode raftJournal = mapper.createObjectNode(); + ArrayNode raftAddresses = mapper.createArrayNode(); + if (!masterInfo.getRaftJournal()) { + raftJournal.put("Enabled", "false"); + } else { + raftJournal.put("Enabled", "true"); for (String raftAddress : masterInfo.getRaftAddressList()) { - print(raftAddress); + raftAddresses.add(raftAddress); } - mIndentationLevel--; - } else { - print("Raft-based Journal: false"); } - String formatString = "%-32s %-8s %-32s"; - print(String.format(formatString, "Master Address", "State", "Version")); + raftJournal.set("Addresses", raftAddresses); + summeryInfo.set("Raft Journal", raftJournal); + + ArrayNode mVersions = mapper.createArrayNode(); for (MasterVersion masterVersion: masterInfo.getMasterVersionsList()) { NetAddress address = masterVersion.getAddresses(); - print(String.format(formatString, - address.getHost() + ":" + address.getRpcPort(), - masterVersion.getState(), - masterVersion.getVersion())); + ObjectNode mVersion = mapper.createObjectNode(); + mVersion.put("Host", masterVersion.getAddresses().getHost()); + mVersion.put("Port", masterVersion.getAddresses().getRpcPort()); + mVersion.put("State", masterVersion.getState()); + mVersion.put("Version", masterVersion.getVersion()); + mVersions.add(mVersion); } - } + summeryInfo.set("Master status", mVersions); - /** - * Prints Alluxio block master information. - */ - private void printBlockMasterInfo() throws IOException { Set blockMasterInfoFilter = new HashSet<>(Arrays - .asList(BlockMasterInfoField.LIVE_WORKER_NUM, BlockMasterInfoField.LOST_WORKER_NUM, - BlockMasterInfoField.CAPACITY_BYTES, BlockMasterInfoField.USED_BYTES, - BlockMasterInfoField.FREE_BYTES, BlockMasterInfoField.CAPACITY_BYTES_ON_TIERS, - BlockMasterInfoField.USED_BYTES_ON_TIERS)); + .asList(BlockMasterInfoField.LIVE_WORKER_NUM, BlockMasterInfoField.LOST_WORKER_NUM, + BlockMasterInfoField.CAPACITY_BYTES, BlockMasterInfoField.USED_BYTES, + BlockMasterInfoField.FREE_BYTES, BlockMasterInfoField.CAPACITY_BYTES_ON_TIERS, + BlockMasterInfoField.USED_BYTES_ON_TIERS)); BlockMasterInfo blockMasterInfo = mBlockMasterClient.getBlockMasterInfo(blockMasterInfoFilter); - print("Live Workers: " + blockMasterInfo.getLiveWorkerNum()); - print("Lost Workers: " + blockMasterInfo.getLostWorkerNum()); - - print("Total Capacity: " - + FormatUtils.getSizeFromBytes(blockMasterInfo.getCapacityBytes())); + summeryInfo.put("Live Workers", blockMasterInfo.getLiveWorkerNum()); + summeryInfo.put("Lost Workers", blockMasterInfo.getLostWorkerNum()); - mIndentationLevel++; + ArrayNode totalCapacity = mapper.createArrayNode(); + ObjectNode allTotalCapacity = mapper.createObjectNode(); + allTotalCapacity.put("Tier", "ALL"); + allTotalCapacity.put("Size", FormatUtils.getSizeFromBytes(blockMasterInfo.getCapacityBytes())); + totalCapacity.add(allTotalCapacity); Map totalCapacityOnTiers = new TreeMap<>((a, b) - -> (FileSystemAdminShellUtils.compareTierNames(a, b))); + -> (FileSystemAdminShellUtils.compareTierNames(a, b))); totalCapacityOnTiers.putAll(blockMasterInfo.getCapacityBytesOnTiers()); for (Map.Entry capacityBytesTier : totalCapacityOnTiers.entrySet()) { - print("Tier: " + capacityBytesTier.getKey() - + " Size: " + FormatUtils.getSizeFromBytes(capacityBytesTier.getValue())); + ObjectNode tierTotalCapacity = mapper.createObjectNode(); + tierTotalCapacity.put("Tier", capacityBytesTier.getKey()); + tierTotalCapacity.put("Size", FormatUtils.getSizeFromBytes(capacityBytesTier.getValue())); + totalCapacity.add(tierTotalCapacity); } + summeryInfo.set("Total Capacity", totalCapacity); - mIndentationLevel--; - print("Used Capacity: " - + FormatUtils.getSizeFromBytes(blockMasterInfo.getUsedBytes())); - - mIndentationLevel++; + ArrayNode usedCapacity = mapper.createArrayNode(); + ObjectNode allUsedCapacity = mapper.createObjectNode(); + allUsedCapacity.put("Tier", "ALL"); + allUsedCapacity.put("Size", FormatUtils.getSizeFromBytes(blockMasterInfo.getUsedBytes())); + usedCapacity.add(allUsedCapacity); Map usedCapacityOnTiers = new TreeMap<>((a, b) - -> (FileSystemAdminShellUtils.compareTierNames(a, b))); + -> (FileSystemAdminShellUtils.compareTierNames(a, b))); usedCapacityOnTiers.putAll(blockMasterInfo.getUsedBytesOnTiers()); for (Map.Entry usedBytesTier: usedCapacityOnTiers.entrySet()) { - print("Tier: " + usedBytesTier.getKey() - + " Size: " + FormatUtils.getSizeFromBytes(usedBytesTier.getValue())); + ObjectNode tierUsedCapacity = mapper.createObjectNode(); + tierUsedCapacity.put("Tier", usedBytesTier.getKey()); + tierUsedCapacity.put("Size", FormatUtils.getSizeFromBytes(usedBytesTier.getValue())); + usedCapacity.add(tierUsedCapacity); } + summeryInfo.set("Used Capacity", usedCapacity); - mIndentationLevel--; - print("Free Capacity: " - + FormatUtils.getSizeFromBytes(blockMasterInfo.getFreeBytes())); - } + summeryInfo.put("Free Capacity", FormatUtils.getSizeFromBytes(blockMasterInfo.getFreeBytes())); - /** - * Prints indented information. - * - * @param text information to print - */ - private void print(String text) { - String indent = Strings.repeat(" ", mIndentationLevel * INDENT_SIZE); - mPrintStream.println(indent + text); + mPrintStream.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(summeryInfo)); + return 0; } } diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java index 2454a4ac0ac2..a03922cb36a7 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java @@ -14,8 +14,12 @@ import alluxio.client.file.FileSystemMasterClient; import alluxio.util.FormatUtils; import alluxio.wire.MountPointInfo; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; +import java.io.PrintStream; import java.util.Comparator; import java.util.Map; @@ -40,55 +44,45 @@ public UfsCommand(FileSystemMasterClient fileSystemMasterClient) { * @return 0 on success, 1 otherwise */ public int run() throws IOException { - Map mountTable = mFileSystemMasterClient.getMountTable(); - System.out.println("Alluxio under storage system information:"); - printMountInfo(mountTable); - return 0; - } + ObjectMapper mapper = new ObjectMapper(); + ArrayNode ufsInfo = mapper.createArrayNode(); - /** - * Prints mount information for a mount table. - * - * @param mountTable the mount table to get information from - */ - public static void printMountInfo(Map mountTable) { + Map mountTable = mFileSystemMasterClient.getMountTable(); for (Map.Entry entry : mountTable.entrySet()) { + ObjectNode mountInfo = mapper.createObjectNode(); + String mountPoint = entry.getKey(); MountPointInfo mountPointInfo = entry.getValue(); - long capacityBytes = mountPointInfo.getUfsCapacityBytes(); long usedBytes = mountPointInfo.getUfsUsedBytes(); - String usedPercentageInfo = ""; if (capacityBytes > 0) { int usedPercentage = (int) (100.0 * usedBytes / capacityBytes); usedPercentageInfo = String.format("(%s%%)", usedPercentage); } - String leftAlignFormat = getAlignFormat(mountTable); + mountInfo.put("URI", mountPointInfo.getUfsUri()); + mountInfo.put("Mount Point", mountPoint); + mountInfo.put("UFS Type", mountPointInfo.getUfsType()); + mountInfo.put("Total Capacity", FormatUtils.getSizeFromBytes(capacityBytes)); + mountInfo.put("Used Capacity", FormatUtils.getSizeFromBytes(usedBytes) + usedPercentageInfo); + mountInfo.put("Readonly", mountPointInfo.getReadOnly()); + mountInfo.put("Shared", mountPointInfo.getShared()); - System.out.format(leftAlignFormat, mountPointInfo.getUfsUri(), mountPoint, - mountPointInfo.getUfsType(), FormatUtils.getSizeFromBytes(capacityBytes), - FormatUtils.getSizeFromBytes(usedBytes) + usedPercentageInfo, - mountPointInfo.getReadOnly() ? "" : "not ", - mountPointInfo.getShared() ? "" : "not "); - System.out.println("properties=" + mountPointInfo.getProperties() + ")"); - } - } + ArrayNode props = mapper.createArrayNode(); + Map properties = mountPointInfo.getProperties(); + for (Map.Entry property : properties.entrySet()) { + ObjectNode prop = mapper.createObjectNode(); + prop.put("key", property.getKey()); + prop.put("value", property.getValue()); + props.add(prop); + } + mountInfo.set("Properties", props); - /** - * Gets the align format according to the longest mount point/under storage path. - * @param mountTable the mount table to get information from - * @return the align format for printing mounted info - */ - private static String getAlignFormat(Map mountTable) { - int mountPointLength = mountTable.entrySet().stream().map(w -> w.getKey().length()) - .max(Comparator.comparing(Integer::intValue)).get(); - int usfLength = mountTable.entrySet().stream().map(w -> w.getValue().getUfsUri().length()) - .max(Comparator.comparing(Integer::intValue)).get(); + ufsInfo.add(mountInfo); + } - String leftAlignFormat = "%-" + usfLength + "s on %-" + mountPointLength - + "s (%s, capacity=%s, used=%s, %sread-only, %sshared, "; - return leftAlignFormat; + System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(ufsInfo)); + return 0; } } From 8490c83c280c08b135df62db4ae23bb67cf07ffc Mon Sep 17 00:00:00 2001 From: twalluxio Date: Wed, 23 Aug 2023 14:46:43 +0800 Subject: [PATCH 03/21] new json-style ufs report --- .../java/alluxio/wire/FileSizeSerializer.java | 18 ++++++++ .../java/alluxio/wire/MountPointInfo.java | 13 +++++- .../cli/fsadmin/report/UfsCommand.java | 44 +++---------------- 3 files changed, 37 insertions(+), 38 deletions(-) create mode 100644 dora/core/common/src/main/java/alluxio/wire/FileSizeSerializer.java diff --git a/dora/core/common/src/main/java/alluxio/wire/FileSizeSerializer.java b/dora/core/common/src/main/java/alluxio/wire/FileSizeSerializer.java new file mode 100644 index 000000000000..b780ea691899 --- /dev/null +++ b/dora/core/common/src/main/java/alluxio/wire/FileSizeSerializer.java @@ -0,0 +1,18 @@ +package alluxio.wire; + +import alluxio.util.FormatUtils; +import com.amazonaws.util.StringUtils; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.SerializerProvider; + +import java.io.IOException; + +public class FileSizeSerializer extends JsonSerializer { + @Override + public void serialize(Long value, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) + throws IOException { + String formattedSize = FormatUtils.getSizeFromBytes(value); + jsonGenerator.writeString(StringUtils.upperCase(formattedSize)); + } +} diff --git a/dora/core/common/src/main/java/alluxio/wire/MountPointInfo.java b/dora/core/common/src/main/java/alluxio/wire/MountPointInfo.java index 0109fac1aeba..bab26fb22acd 100644 --- a/dora/core/common/src/main/java/alluxio/wire/MountPointInfo.java +++ b/dora/core/common/src/main/java/alluxio/wire/MountPointInfo.java @@ -13,6 +13,9 @@ import alluxio.annotation.PublicApi; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.google.common.base.MoreObjects; import com.google.common.base.Objects; @@ -27,19 +30,27 @@ @NotThreadSafe @PublicApi public class MountPointInfo implements Serializable { + @JsonIgnore private static final long serialVersionUID = -2912330427506888886L; + @JsonIgnore private static final long UNKNOWN_CAPACITY_BYTES = -1; + @JsonIgnore private static final long UNKNOWN_USED_BYTES = -1; private String mUfsUri = ""; private String mUfsType = ""; + @JsonIgnore private long mMountId = 0; + @JsonSerialize(using = FileSizeSerializer.class) + @JsonProperty("ufsCapacity") private long mUfsCapacityBytes = UNKNOWN_CAPACITY_BYTES; + @JsonSerialize(using = FileSizeSerializer.class) + @JsonProperty("ufsUsed") private long mUfsUsedBytes = UNKNOWN_USED_BYTES; private boolean mReadOnly; - private HashMap mProperties = new HashMap<>(); private boolean mShared; + private HashMap mProperties = new HashMap<>(); /** * Creates a new instance of {@link MountPointInfo}. diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java index a03922cb36a7..eb6f0289231c 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java @@ -14,6 +14,7 @@ import alluxio.client.file.FileSystemMasterClient; import alluxio.util.FormatUtils; import alluxio.wire.MountPointInfo; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -44,45 +45,14 @@ public UfsCommand(FileSystemMasterClient fileSystemMasterClient) { * @return 0 on success, 1 otherwise */ public int run() throws IOException { - ObjectMapper mapper = new ObjectMapper(); - ArrayNode ufsInfo = mapper.createArrayNode(); - Map mountTable = mFileSystemMasterClient.getMountTable(); - for (Map.Entry entry : mountTable.entrySet()) { - ObjectNode mountInfo = mapper.createObjectNode(); - - String mountPoint = entry.getKey(); - MountPointInfo mountPointInfo = entry.getValue(); - long capacityBytes = mountPointInfo.getUfsCapacityBytes(); - long usedBytes = mountPointInfo.getUfsUsedBytes(); - String usedPercentageInfo = ""; - if (capacityBytes > 0) { - int usedPercentage = (int) (100.0 * usedBytes / capacityBytes); - usedPercentageInfo = String.format("(%s%%)", usedPercentage); - } - - mountInfo.put("URI", mountPointInfo.getUfsUri()); - mountInfo.put("Mount Point", mountPoint); - mountInfo.put("UFS Type", mountPointInfo.getUfsType()); - mountInfo.put("Total Capacity", FormatUtils.getSizeFromBytes(capacityBytes)); - mountInfo.put("Used Capacity", FormatUtils.getSizeFromBytes(usedBytes) + usedPercentageInfo); - mountInfo.put("Readonly", mountPointInfo.getReadOnly()); - mountInfo.put("Shared", mountPointInfo.getShared()); - - ArrayNode props = mapper.createArrayNode(); - Map properties = mountPointInfo.getProperties(); - for (Map.Entry property : properties.entrySet()) { - ObjectNode prop = mapper.createObjectNode(); - prop.put("key", property.getKey()); - prop.put("value", property.getValue()); - props.add(prop); - } - mountInfo.set("Properties", props); - - ufsInfo.add(mountInfo); + ObjectMapper objectMapper = new ObjectMapper(); + try { + String json = objectMapper.writeValueAsString(mountTable); + System.out.println(json); + } catch (JsonProcessingException e) { + e.printStackTrace(); } - - System.out.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(ufsInfo)); return 0; } } From e4c41410492e1f7819971c9c0134206735d7bedd Mon Sep 17 00:00:00 2001 From: twalluxio Date: Thu, 24 Aug 2023 11:00:46 +0800 Subject: [PATCH 04/21] fix MountPointInfo.java --- .../common/src/main/java/alluxio/wire/MountPointInfo.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/dora/core/common/src/main/java/alluxio/wire/MountPointInfo.java b/dora/core/common/src/main/java/alluxio/wire/MountPointInfo.java index bab26fb22acd..dcd1fb41bf62 100644 --- a/dora/core/common/src/main/java/alluxio/wire/MountPointInfo.java +++ b/dora/core/common/src/main/java/alluxio/wire/MountPointInfo.java @@ -30,12 +30,9 @@ @NotThreadSafe @PublicApi public class MountPointInfo implements Serializable { - @JsonIgnore private static final long serialVersionUID = -2912330427506888886L; - @JsonIgnore private static final long UNKNOWN_CAPACITY_BYTES = -1; - @JsonIgnore private static final long UNKNOWN_USED_BYTES = -1; private String mUfsUri = ""; @@ -43,10 +40,10 @@ public class MountPointInfo implements Serializable { @JsonIgnore private long mMountId = 0; @JsonSerialize(using = FileSizeSerializer.class) - @JsonProperty("ufsCapacity") + @JsonProperty("ufsCapacityBytes") private long mUfsCapacityBytes = UNKNOWN_CAPACITY_BYTES; @JsonSerialize(using = FileSizeSerializer.class) - @JsonProperty("ufsUsed") + @JsonProperty("ufsUsedBytes") private long mUfsUsedBytes = UNKNOWN_USED_BYTES; private boolean mReadOnly; private boolean mShared; From d9cd0bdb9df5549aad0d3f653f1695cf450d6fd2 Mon Sep 17 00:00:00 2001 From: twalluxio Date: Thu, 24 Aug 2023 11:00:59 +0800 Subject: [PATCH 05/21] new json-style summary report --- dora/shell/pom.xml | 6 +- .../cli/fsadmin/report/SummaryCommand.java | 105 +------ .../cli/fsadmin/report/SummaryOutput.java | 263 ++++++++++++++++++ 3 files changed, 276 insertions(+), 98 deletions(-) create mode 100644 dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java diff --git a/dora/shell/pom.xml b/dora/shell/pom.xml index fb8abb67a365..ce5144fa7824 100644 --- a/dora/shell/pom.xml +++ b/dora/shell/pom.xml @@ -39,6 +39,11 @@ com.google.guava guava + + com.google.protobuf + protobuf-java-util + 3.18.1 + commons-cli commons-cli @@ -82,7 +87,6 @@ org.jline jline - org.alluxio alluxio-core-common diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java index d96501661e2a..48e4b29bf976 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java @@ -11,31 +11,21 @@ package alluxio.cli.fsadmin.report; -import alluxio.cli.fsadmin.FileSystemAdminShellUtils; import alluxio.client.block.BlockMasterClient; import alluxio.client.meta.MetaMasterClient; import alluxio.grpc.MasterInfo; import alluxio.grpc.MasterInfoField; -import alluxio.grpc.MasterVersion; -import alluxio.grpc.NetAddress; -import alluxio.util.CommonUtils; -import alluxio.util.FormatUtils; import alluxio.wire.BlockMasterInfo; import alluxio.wire.BlockMasterInfo.BlockMasterInfoField; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.base.Strings; import java.io.IOException; import java.io.PrintStream; import java.util.Arrays; import java.util.HashSet; -import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.TreeMap; /** * Prints Alluxio cluster summarized information. @@ -68,9 +58,6 @@ public SummaryCommand(MetaMasterClient metaMasterClient, * @return 0 on success, 1 otherwise */ public int run() throws IOException { - ObjectMapper mapper = new ObjectMapper(); - ObjectNode summeryInfo = mapper.createObjectNode(); - Set masterInfoFilter = new HashSet<>(Arrays .asList(MasterInfoField.LEADER_MASTER_ADDRESS, MasterInfoField.WEB_PORT, MasterInfoField.RPC_PORT, MasterInfoField.START_TIME_MS, @@ -80,53 +67,6 @@ public int run() throws IOException { MasterInfoField.MASTER_VERSION)); MasterInfo masterInfo = mMetaMasterClient.getMasterInfo(masterInfoFilter); - summeryInfo.put("Master Address", masterInfo.getLeaderMasterAddress()); - summeryInfo.put("Web Port", masterInfo.getWebPort()); - summeryInfo.put("Rpc Port", masterInfo.getRpcPort()); - summeryInfo.put("Started", CommonUtils.convertMsToDate(masterInfo.getStartTimeMs(), mDateFormatPattern)); - summeryInfo.put("Uptime", CommonUtils.convertMsToClockTime(masterInfo.getUpTimeMs())); - summeryInfo.put("Version", masterInfo.getVersion()); - summeryInfo.put("Safe Mode", masterInfo.getSafeMode()); - - ObjectNode zooKeeper = mapper.createObjectNode(); - List zookeeperAddresses = masterInfo.getZookeeperAddressesList(); - ArrayNode zkAddresses = mapper.createArrayNode(); - if (zookeeperAddresses == null || zookeeperAddresses.isEmpty()) { - zooKeeper.put("Enabled", "false"); - } else { - zooKeeper.put("Enabled", "true"); - for (String zookeeperAddress : zookeeperAddresses) { - zkAddresses.add(zookeeperAddress); - } - } - zooKeeper.set("Addresses", zkAddresses); - summeryInfo.set("Zookeeper", zooKeeper); - - ObjectNode raftJournal = mapper.createObjectNode(); - ArrayNode raftAddresses = mapper.createArrayNode(); - if (!masterInfo.getRaftJournal()) { - raftJournal.put("Enabled", "false"); - } else { - raftJournal.put("Enabled", "true"); - for (String raftAddress : masterInfo.getRaftAddressList()) { - raftAddresses.add(raftAddress); - } - } - raftJournal.set("Addresses", raftAddresses); - summeryInfo.set("Raft Journal", raftJournal); - - ArrayNode mVersions = mapper.createArrayNode(); - for (MasterVersion masterVersion: masterInfo.getMasterVersionsList()) { - NetAddress address = masterVersion.getAddresses(); - ObjectNode mVersion = mapper.createObjectNode(); - mVersion.put("Host", masterVersion.getAddresses().getHost()); - mVersion.put("Port", masterVersion.getAddresses().getRpcPort()); - mVersion.put("State", masterVersion.getState()); - mVersion.put("Version", masterVersion.getVersion()); - mVersions.add(mVersion); - } - summeryInfo.set("Master status", mVersions); - Set blockMasterInfoFilter = new HashSet<>(Arrays .asList(BlockMasterInfoField.LIVE_WORKER_NUM, BlockMasterInfoField.LOST_WORKER_NUM, BlockMasterInfoField.CAPACITY_BYTES, BlockMasterInfoField.USED_BYTES, @@ -134,44 +74,15 @@ public int run() throws IOException { BlockMasterInfoField.USED_BYTES_ON_TIERS)); BlockMasterInfo blockMasterInfo = mBlockMasterClient.getBlockMasterInfo(blockMasterInfoFilter); - summeryInfo.put("Live Workers", blockMasterInfo.getLiveWorkerNum()); - summeryInfo.put("Lost Workers", blockMasterInfo.getLostWorkerNum()); - - ArrayNode totalCapacity = mapper.createArrayNode(); - ObjectNode allTotalCapacity = mapper.createObjectNode(); - allTotalCapacity.put("Tier", "ALL"); - allTotalCapacity.put("Size", FormatUtils.getSizeFromBytes(blockMasterInfo.getCapacityBytes())); - totalCapacity.add(allTotalCapacity); - Map totalCapacityOnTiers = new TreeMap<>((a, b) - -> (FileSystemAdminShellUtils.compareTierNames(a, b))); - totalCapacityOnTiers.putAll(blockMasterInfo.getCapacityBytesOnTiers()); - for (Map.Entry capacityBytesTier : totalCapacityOnTiers.entrySet()) { - ObjectNode tierTotalCapacity = mapper.createObjectNode(); - tierTotalCapacity.put("Tier", capacityBytesTier.getKey()); - tierTotalCapacity.put("Size", FormatUtils.getSizeFromBytes(capacityBytesTier.getValue())); - totalCapacity.add(tierTotalCapacity); + ObjectMapper objectMapper = new ObjectMapper(); + SummaryOutput summaryInfo = new SummaryOutput(masterInfo, blockMasterInfo, mDateFormatPattern); + try { + String json = objectMapper.writeValueAsString(summaryInfo); + mPrintStream.println(json); + } catch (JsonProcessingException e) { + e.printStackTrace(); } - summeryInfo.set("Total Capacity", totalCapacity); - - ArrayNode usedCapacity = mapper.createArrayNode(); - ObjectNode allUsedCapacity = mapper.createObjectNode(); - allUsedCapacity.put("Tier", "ALL"); - allUsedCapacity.put("Size", FormatUtils.getSizeFromBytes(blockMasterInfo.getUsedBytes())); - usedCapacity.add(allUsedCapacity); - Map usedCapacityOnTiers = new TreeMap<>((a, b) - -> (FileSystemAdminShellUtils.compareTierNames(a, b))); - usedCapacityOnTiers.putAll(blockMasterInfo.getUsedBytesOnTiers()); - for (Map.Entry usedBytesTier: usedCapacityOnTiers.entrySet()) { - ObjectNode tierUsedCapacity = mapper.createObjectNode(); - tierUsedCapacity.put("Tier", usedBytesTier.getKey()); - tierUsedCapacity.put("Size", FormatUtils.getSizeFromBytes(usedBytesTier.getValue())); - usedCapacity.add(tierUsedCapacity); - } - summeryInfo.set("Used Capacity", usedCapacity); - - summeryInfo.put("Free Capacity", FormatUtils.getSizeFromBytes(blockMasterInfo.getFreeBytes())); - mPrintStream.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(summeryInfo)); return 0; } } diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java new file mode 100644 index 000000000000..73fd8bbc03ec --- /dev/null +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java @@ -0,0 +1,263 @@ +package alluxio.cli.fsadmin.report; + +import alluxio.cli.fsadmin.FileSystemAdminShellUtils; +import alluxio.grpc.MasterInfo; +import alluxio.grpc.MasterVersion; +import alluxio.wire.BlockMasterInfo; +import alluxio.util.CommonUtils; +import alluxio.util.FormatUtils; +import com.fasterxml.jackson.annotation.JsonProperty; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; + +public class SummaryOutput { + private String mMasterAddress; + private int mWebPort; + private int mRpcPort; + private String mStarted; + private String mUptime; + private String mVersion; + private boolean mSafeMode; + + private List mZookeeperAddress; + private boolean mZookeeper; + private List mRaftJournalAddress; + private boolean mRaftJournal; + private List mMasterVersions; + + private int mLiveWorkers; + private int mLostWorkers; + private Map mTotalCapacityOnTiers; + private Map mUsedCapacityOnTiers; + private String mFreeCapacity; + + private static class SerializableMasterVersion { + @JsonProperty("Host") + private String mHost; + @JsonProperty("Port") + private int mPort; + @JsonProperty("State") + private String mState; + @JsonProperty("Version") + private String mVersion; + + public SerializableMasterVersion(MasterVersion masterVersion) { + mHost = masterVersion.getAddresses().getHost(); + mPort = masterVersion.getAddresses().getRpcPort(); + mState = masterVersion.getState(); + mVersion = masterVersion.getVersion(); + } + + public String getHost() { + return mHost; + } + + public void setHost(String host) { + this.mHost = host; + } + + public int getPort() { + return mPort; + } + + public void setPort(int port) { + this.mPort = port; + } + + public String getState() { + return mState; + } + + public void setState(String state) { + this.mState = state; + } + + public String getVersion() { + return mVersion; + } + + public void setVersion(String version) { + this.mVersion = version; + } + } + + public SummaryOutput(MasterInfo masterInfo, BlockMasterInfo blockMasterInfo, String dateFormatPattern) { + // give values to internal properties + mMasterAddress = masterInfo.getLeaderMasterAddress(); + mWebPort = masterInfo.getWebPort(); + mRpcPort = masterInfo.getRpcPort(); + mStarted = CommonUtils.convertMsToDate(masterInfo.getStartTimeMs(), dateFormatPattern); + mUptime = CommonUtils.convertMsToClockTime(masterInfo.getUpTimeMs()); + mVersion = masterInfo.getVersion(); + mSafeMode = masterInfo.getSafeMode(); + + mZookeeperAddress = masterInfo.getZookeeperAddressesList(); + mZookeeper = !mZookeeperAddress.isEmpty(); + mRaftJournal = masterInfo.getRaftJournal(); + if (mRaftJournal) { + mRaftJournalAddress = masterInfo.getRaftAddressList(); + } else { + mRaftJournalAddress = new ArrayList<>(); + } + mMasterVersions = new ArrayList<>(); + for (MasterVersion masterVersion : masterInfo.getMasterVersionsList()) { + mMasterVersions.add(new SerializableMasterVersion(masterVersion)); + } + + mLiveWorkers = blockMasterInfo.getLiveWorkerNum(); + mLostWorkers = blockMasterInfo.getLostWorkerNum(); + + mTotalCapacityOnTiers = new TreeMap<>(); + mUsedCapacityOnTiers = new TreeMap<>(); + Map totalBytesOnTiers = new TreeMap<>(FileSystemAdminShellUtils::compareTierNames); + totalBytesOnTiers.putAll(blockMasterInfo.getCapacityBytesOnTiers()); + for (Map.Entry totalBytesOnTier : totalBytesOnTiers.entrySet()) { + mTotalCapacityOnTiers.put(totalBytesOnTier.getKey(), FormatUtils.getSizeFromBytes(totalBytesOnTier.getValue())); + } + Map usedBytesOnTiers = new TreeMap<>(FileSystemAdminShellUtils::compareTierNames); + usedBytesOnTiers.putAll(blockMasterInfo.getUsedBytesOnTiers()); + for (Map.Entry usedBytesOnTier : usedBytesOnTiers.entrySet()) { + mUsedCapacityOnTiers.put(usedBytesOnTier.getKey(), FormatUtils.getSizeFromBytes(usedBytesOnTier.getValue())); + } + mFreeCapacity = FormatUtils.getSizeFromBytes(blockMasterInfo.getFreeBytes()); + } + + public String getMasterAddress() { + return mMasterAddress; + } + + public void setMasterAddress(String masterAddress) { + this.mMasterAddress = masterAddress; + } + + public int getWebPort() { + return mWebPort; + } + + public void setWebPort(int webPort) { + this.mWebPort = webPort; + } + + public int getRpcPort() { + return mRpcPort; + } + + public void setRpcPort(int rpcPort) { + this.mRpcPort = rpcPort; + } + + public String getStarted() { + return mStarted; + } + + public void setStarted(String started) { + this.mStarted = started; + } + + public String getUptime() { + return mUptime; + } + + public void setUptime(String uptime) { + this.mUptime = uptime; + } + + public String getVersion() { + return mVersion; + } + + public void setVersion(String version) { + this.mVersion = version; + } + + public boolean ismSafeMode() { + return mSafeMode; + } + + public void setSafeMode(boolean safeMode) { + this.mSafeMode = safeMode; + } + + public List getZookeeperAddress() { + return mZookeeperAddress; + } + + public void setZookeeperAddress(List zookeeperAddress) { + this.mZookeeperAddress = zookeeperAddress; + } + + public boolean ismZookeeper() { + return mZookeeper; + } + + public void setZookeeper(boolean mZookeeper) { + this.mZookeeper = mZookeeper; + } + + public List getRaftJournalAddress() { + return mRaftJournalAddress; + } + + public void setRaftJournalAddress(List raftJournalAddress) { + this.mRaftJournalAddress = raftJournalAddress; + } + + public boolean ismRaftJournal() { + return mRaftJournal; + } + + public void setRaftJournal(boolean raftJournal) { + this.mRaftJournal = raftJournal; + } + + public List getMasterVersions() { + return mMasterVersions; + } + + public void setMasterVersions(List masterVersions) { + this.mMasterVersions = masterVersions; + } + + public int getLiveWorkers() { + return mLiveWorkers; + } + + public void setLiveWorkers(int liveWorkers) { + this.mLiveWorkers = liveWorkers; + } + + public int getLostWorkers() { + return mLostWorkers; + } + + public void setLostWorkers(int lostWorkers) { + this.mLostWorkers = lostWorkers; + } + + public String getFreeCapacity() { + return mFreeCapacity; + } + + public void setFreeCapacity(String freeCapacity) { + this.mFreeCapacity = freeCapacity; + } + + public Map getTotalCapacityOnTiers() { + return mTotalCapacityOnTiers; + } + + public void setTotalCapacityOnTiers(Map totalCapacityOnTiers) { + this.mTotalCapacityOnTiers = totalCapacityOnTiers; + } + + public Map getUsedCapacityOnTiers() { + return mUsedCapacityOnTiers; + } + + public void setUsedCapacityOnTiers(Map usedCapacityOnTiers) { + this.mUsedCapacityOnTiers = usedCapacityOnTiers; + } +} \ No newline at end of file From 32e094c5cd6fa1bce91e5eae9f9702ef85e3cff2 Mon Sep 17 00:00:00 2001 From: twalluxio Date: Thu, 24 Aug 2023 13:45:56 +0800 Subject: [PATCH 06/21] new json-style job service report --- .../report/JobServiceMetricsCommand.java | 90 +---- .../cli/fsadmin/report/JobServiceOutput.java | 314 ++++++++++++++++++ .../cli/fsadmin/report/SummaryOutput.java | 47 ++- 3 files changed, 343 insertions(+), 108 deletions(-) create mode 100644 dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceOutput.java diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java index b9149cfe51ca..33edf268b110 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java @@ -12,28 +12,15 @@ package alluxio.cli.fsadmin.report; import alluxio.client.job.JobMasterClient; -import alluxio.grpc.JobMasterStatus; -import alluxio.grpc.NetAddress; -import alluxio.job.wire.JobInfo; -import alluxio.job.wire.JobServiceSummary; -import alluxio.job.wire.JobWorkerHealth; -import alluxio.job.wire.StatusSummary; -import alluxio.util.CommonUtils; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import org.apache.commons.lang3.StringUtils; import java.io.IOException; import java.io.PrintStream; -import java.time.Instant; import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.time.format.FormatStyle; -import java.util.Collection; -import java.util.Comparator; -import java.util.List; import java.util.Locale; /** @@ -69,75 +56,14 @@ public JobServiceMetricsCommand(JobMasterClient JobMasterClient, PrintStream pri * @return 0 on success, 1 otherwise */ public int run() throws IOException { - ObjectMapper mapper = new ObjectMapper(); - ObjectNode jobServiceInfo = mapper.createObjectNode(); - - ArrayNode masterInfo = mapper.createArrayNode(); - List allMasterStatus = mJobMasterClient.getAllMasterStatus(); - for (JobMasterStatus masterStatus : allMasterStatus) { - ObjectNode master = mapper.createObjectNode(); - master.put("Host", masterStatus.getMasterAddress().getHost()); - master.put("Port", masterStatus.getMasterAddress().getRpcPort()); - master.put("State", masterStatus.getState()); - master.put("Start Time", DATETIME_FORMAT.format(Instant.ofEpochMilli(masterStatus.getStartTime()))); - master.put("Version", masterStatus.getVersion().getVersion()); - master.put("Revision", masterStatus.getVersion().getRevision()); - masterInfo.add(master); - } - jobServiceInfo.set("Masters", masterInfo); - - ArrayNode workerInfo = mapper.createArrayNode(); - List allWorkerHealth = mJobMasterClient.getAllWorkerHealth(); - for (JobWorkerHealth workerHealth : allWorkerHealth) { - ObjectNode worker = mapper.createObjectNode(); - worker.put("Host", workerHealth.getHostname()); - worker.put("Version", workerHealth.getVersion().getVersion()); - worker.put("Revision", workerHealth.getVersion().getRevision()); - worker.put("Task Pool Size", workerHealth.getTaskPoolSize()); - worker.put("Unfinished Tasks", workerHealth.getUnfinishedTasks()); - worker.put("Active Tasks", workerHealth.getNumActiveTasks()); - ObjectNode lAverage = mapper.createObjectNode(); - List loadAverage = workerHealth.getLoadAverage(); - lAverage.put("1 minute", loadAverage.get(0)); - lAverage.put("5 minutes", loadAverage.get(1)); - lAverage.put("15 minutes", loadAverage.get(2)); - worker.set("Load Average", lAverage); - workerInfo.add(worker); - } - jobServiceInfo.set("Workers", workerInfo); - - ArrayNode jobStatusInfo = mapper.createArrayNode(); - JobServiceSummary jobServiceSummary = mJobMasterClient.getJobServiceSummary(); - Collection jobStatusSummaries = jobServiceSummary.getSummaryPerStatus(); - for (StatusSummary statusSummary : jobStatusSummaries) { - ObjectNode status = mapper.createObjectNode(); - status.put("Status", statusSummary.getStatus().toString()); - status.put("Count", statusSummary.getCount()); - jobStatusInfo.add(status); + ObjectMapper objectMapper = new ObjectMapper(); + JobServiceOutput jobServiceInfo = new JobServiceOutput(mJobMasterClient, mDateFormatPattern); + try { + String json = objectMapper.writeValueAsString(jobServiceInfo); + mPrintStream.println(json); + } catch (JsonProcessingException e) { + e.printStackTrace(); } - jobServiceInfo.set("Job Status", jobStatusInfo); - - ObjectNode recentJobInfo = mapper.createObjectNode(); - recentJobInfo.set("Recent Modified", getJobInfos(jobServiceSummary.getRecentActivities())); - recentJobInfo.set("Recent Failed", getJobInfos(jobServiceSummary.getRecentFailures())); - recentJobInfo.set("Longest running", getJobInfos(jobServiceSummary.getLongestRunning())); - jobServiceInfo.set(String.format("Top %s jobs", JobServiceSummary.RECENT_LENGTH), recentJobInfo); - - mPrintStream.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(jobServiceInfo)); return 0; } - - private ArrayNode getJobInfos(List jobInfos) { - ObjectMapper mapper = new ObjectMapper(); - ArrayNode result = mapper.createArrayNode(); - for (JobInfo jobInfo : jobInfos) { - ObjectNode jInfo = mapper.createObjectNode(); - jInfo.put("Timestamp", CommonUtils.convertMsToDate(jobInfo.getLastUpdated(), mDateFormatPattern)); - jInfo.put("Id", jobInfo.getId()); - jInfo.put("Name", jobInfo.getName()); - jInfo.put("Status", jobInfo.getStatus().toString()); - result.add(jInfo); - } - return result; - } } diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceOutput.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceOutput.java new file mode 100644 index 000000000000..d7f552b28567 --- /dev/null +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceOutput.java @@ -0,0 +1,314 @@ +package alluxio.cli.fsadmin.report; + +import alluxio.client.job.JobMasterClient; +import alluxio.grpc.JobMasterStatus; +import alluxio.job.wire.JobInfo; +import alluxio.job.wire.JobServiceSummary; +import alluxio.job.wire.JobWorkerHealth; +import alluxio.job.wire.StatusSummary; +import alluxio.util.CommonUtils; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class JobServiceOutput { + private List mMasterStatus; + private List mWorkerHealth; + private List mStatusSummary; + private List mRecentModifiedJobs; + private List mRecentFailedJobs; + private List mLongestRunningJobs; + + private static class SerializableJobMasterStatus { + private String mHost; + private int mPort; + private String mState; + private String mStartTime; + private String mVersion; + private String mRevision; + + public SerializableJobMasterStatus(JobMasterStatus jobMasterStatus, String dateFormat) { + mHost = jobMasterStatus.getMasterAddress().getHost(); + mPort = jobMasterStatus.getMasterAddress().getRpcPort(); + mState = jobMasterStatus.getState(); + mStartTime = CommonUtils.convertMsToDate(jobMasterStatus.getStartTime(), dateFormat); + mVersion = jobMasterStatus.getVersion().getVersion(); + mRevision = jobMasterStatus.getVersion().getRevision(); + } + + public String getHost() { + return mHost; + } + + public void setHost(String host) { + mHost = host; + } + + public int getPort() { + return mPort; + } + + public void setPort(int port) { + mPort = port; + } + + public String getState() { + return mState; + } + + public void setState(String state) { + mState = state; + } + + public String getStartTime() { + return mStartTime; + } + + public void setStartTime(String startTime) { + mStartTime = startTime; + } + + public String getVersion() { + return mVersion; + } + + public void setVersion(String version) { + mVersion = version; + } + + public String getRevision() { + return mRevision; + } + + public void setRevision(String revision) { + mRevision = revision; + } + } + + private static class SerializableWorkerHealth { + private String mHost; + private String mVersion; + private String mRevision; + private int mTaskPoolSize; + private int mUnfinishedTasks; + private int mActiveTasks; + private List mLoadAverage; + + public SerializableWorkerHealth(JobWorkerHealth workerHealth) { + mHost = workerHealth.getHostname(); + mVersion = workerHealth.getVersion().getVersion(); + mRevision = workerHealth.getVersion().getRevision(); + mTaskPoolSize = workerHealth.getTaskPoolSize(); + mUnfinishedTasks = workerHealth.getUnfinishedTasks(); + mActiveTasks = workerHealth.getNumActiveTasks(); + mLoadAverage = workerHealth.getLoadAverage(); + } + public String getHost() { + return mHost; + } + + public void setHost(String host) { + mHost = host; + } + + public String getVersion() { + return mVersion; + } + + public void setVersion(String version) { + mVersion = version; + } + + public String getRevision() { + return mRevision; + } + + public void setRevision(String revision) { + mRevision = revision; + } + + public int getTaskPoolSize() { + return mTaskPoolSize; + } + + public void setTaskPoolSize(int taskPoolSize) { + mTaskPoolSize = taskPoolSize; + } + + public int getUnfinishedTasks() { + return mUnfinishedTasks; + } + + public void setUnfinishedTasks(int unfinishedTasks) { + mUnfinishedTasks = unfinishedTasks; + } + + public int getActiveTasks() { + return mActiveTasks; + } + + public void setActiveTasks(int activeTasks) { + mActiveTasks = activeTasks; + } + + public List getLoadAverage() { + return mLoadAverage; + } + + public void setLoadAverage(List loadAverage) { + mLoadAverage = loadAverage; + } + + } + + private static class SerializableStatusSummary { + private String mStatus; + private long mCount; + + public SerializableStatusSummary(StatusSummary statusSummary){ + mStatus = statusSummary.getStatus().toString(); + mCount = statusSummary.getCount(); + } + + public String getStatus() { + return mStatus; + } + + public void setStatus(String status) { + mStatus = status; + } + + public long getCount(){ + return mCount; + } + + public void setCount(long count){ + mCount = count; + } + } + + private static class SerializableJobInfo { + private String mTimestamp; + private long mId; + private String mName; + private String mStatus; + public SerializableJobInfo(JobInfo jobInfo, String dateFormat) { + mTimestamp = CommonUtils.convertMsToDate(jobInfo.getLastUpdated(), dateFormat); + mId = jobInfo.getId(); + mName = jobInfo.getName(); + mStatus = jobInfo.getStatus().toString(); + } + + public String getTimestamp() { + return mTimestamp; + } + + public void setTimestamp(String timestamp) { + mTimestamp = timestamp; + } + + public long getId() { + return mId; + } + + public void setId(long id) { + mId = id; + } + + public String getName() { + return mName; + } + + public void setName(String name) { + mName = name; + } + + public String getStatus() { + return mStatus; + } + + public void setStatus(String status) { + mStatus = status; + } + } + + public JobServiceOutput(JobMasterClient jobMasterClient, String dateFormat) throws IOException { + mMasterStatus = new ArrayList<>(); + for (JobMasterStatus masterStatus : jobMasterClient.getAllMasterStatus()) { + mMasterStatus.add(new SerializableJobMasterStatus(masterStatus, dateFormat)); + } + + mWorkerHealth = new ArrayList<>(); + for (JobWorkerHealth workerHealth : jobMasterClient.getAllWorkerHealth()) { + mWorkerHealth.add(new SerializableWorkerHealth(workerHealth)); + } + + JobServiceSummary jobServiceSummary = jobMasterClient.getJobServiceSummary(); + mStatusSummary = new ArrayList<>(); + for (StatusSummary statusSummary : jobServiceSummary.getSummaryPerStatus()) { + mStatusSummary.add(new SerializableStatusSummary(statusSummary)); + } + + mRecentModifiedJobs = new ArrayList<>(); + mRecentFailedJobs = new ArrayList<>(); + mLongestRunningJobs = new ArrayList<>(); + for (JobInfo jobInfo : jobServiceSummary.getRecentActivities()){ + mRecentModifiedJobs.add(new SerializableJobInfo(jobInfo, dateFormat)); + } + for (JobInfo jobInfo : jobServiceSummary.getRecentFailures()){ + mRecentFailedJobs.add(new SerializableJobInfo(jobInfo, dateFormat)); + } + for (JobInfo jobInfo : jobServiceSummary.getLongestRunning()) { + mLongestRunningJobs.add(new SerializableJobInfo(jobInfo, dateFormat)); + } + } + + public List getMasterStatus() { + return mMasterStatus; + } + + public void setMasterStatus(List masterStatus) { + mMasterStatus = masterStatus; + } + + public List getWorkerHealth() { + return mWorkerHealth; + } + + public void setWorkerHealth(List workerHealth) { + mWorkerHealth = workerHealth; + } + + public List getStatusSummary() { + return mStatusSummary; + } + + public void setStatusSummary(List statusSummary) { + mStatusSummary = statusSummary; + } + + public List getRecentModifiedJobs() { + return mRecentModifiedJobs; + } + + public void setRecentModifiedJobs(List recentModifiedJobs) { + mRecentModifiedJobs = recentModifiedJobs; + } + + public List getRecentFailedJobs() { + return mRecentFailedJobs; + } + + public void setRecentFailedJobs(List recentFailedJobs) { + mRecentFailedJobs = recentFailedJobs; + } + + public List getLongestRunningJobs() { + return mLongestRunningJobs; + } + + public void setLongestRunningJobs(List longestRunningJobs) { + mLongestRunningJobs = longestRunningJobs; + } +} diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java index 73fd8bbc03ec..bf47cf941af0 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java @@ -6,7 +6,6 @@ import alluxio.wire.BlockMasterInfo; import alluxio.util.CommonUtils; import alluxio.util.FormatUtils; -import com.fasterxml.jackson.annotation.JsonProperty; import java.util.ArrayList; import java.util.List; @@ -35,13 +34,9 @@ public class SummaryOutput { private String mFreeCapacity; private static class SerializableMasterVersion { - @JsonProperty("Host") private String mHost; - @JsonProperty("Port") private int mPort; - @JsonProperty("State") private String mState; - @JsonProperty("Version") private String mVersion; public SerializableMasterVersion(MasterVersion masterVersion) { @@ -56,7 +51,7 @@ public String getHost() { } public void setHost(String host) { - this.mHost = host; + mHost = host; } public int getPort() { @@ -64,7 +59,7 @@ public int getPort() { } public void setPort(int port) { - this.mPort = port; + mPort = port; } public String getState() { @@ -72,7 +67,7 @@ public String getState() { } public void setState(String state) { - this.mState = state; + mState = state; } public String getVersion() { @@ -80,7 +75,7 @@ public String getVersion() { } public void setVersion(String version) { - this.mVersion = version; + mVersion = version; } } @@ -130,7 +125,7 @@ public String getMasterAddress() { } public void setMasterAddress(String masterAddress) { - this.mMasterAddress = masterAddress; + mMasterAddress = masterAddress; } public int getWebPort() { @@ -138,7 +133,7 @@ public int getWebPort() { } public void setWebPort(int webPort) { - this.mWebPort = webPort; + mWebPort = webPort; } public int getRpcPort() { @@ -146,7 +141,7 @@ public int getRpcPort() { } public void setRpcPort(int rpcPort) { - this.mRpcPort = rpcPort; + mRpcPort = rpcPort; } public String getStarted() { @@ -154,7 +149,7 @@ public String getStarted() { } public void setStarted(String started) { - this.mStarted = started; + mStarted = started; } public String getUptime() { @@ -162,7 +157,7 @@ public String getUptime() { } public void setUptime(String uptime) { - this.mUptime = uptime; + mUptime = uptime; } public String getVersion() { @@ -170,7 +165,7 @@ public String getVersion() { } public void setVersion(String version) { - this.mVersion = version; + mVersion = version; } public boolean ismSafeMode() { @@ -178,7 +173,7 @@ public boolean ismSafeMode() { } public void setSafeMode(boolean safeMode) { - this.mSafeMode = safeMode; + mSafeMode = safeMode; } public List getZookeeperAddress() { @@ -186,7 +181,7 @@ public List getZookeeperAddress() { } public void setZookeeperAddress(List zookeeperAddress) { - this.mZookeeperAddress = zookeeperAddress; + mZookeeperAddress = zookeeperAddress; } public boolean ismZookeeper() { @@ -194,7 +189,7 @@ public boolean ismZookeeper() { } public void setZookeeper(boolean mZookeeper) { - this.mZookeeper = mZookeeper; + mZookeeper = mZookeeper; } public List getRaftJournalAddress() { @@ -202,7 +197,7 @@ public List getRaftJournalAddress() { } public void setRaftJournalAddress(List raftJournalAddress) { - this.mRaftJournalAddress = raftJournalAddress; + mRaftJournalAddress = raftJournalAddress; } public boolean ismRaftJournal() { @@ -210,7 +205,7 @@ public boolean ismRaftJournal() { } public void setRaftJournal(boolean raftJournal) { - this.mRaftJournal = raftJournal; + mRaftJournal = raftJournal; } public List getMasterVersions() { @@ -218,7 +213,7 @@ public List getMasterVersions() { } public void setMasterVersions(List masterVersions) { - this.mMasterVersions = masterVersions; + mMasterVersions = masterVersions; } public int getLiveWorkers() { @@ -226,7 +221,7 @@ public int getLiveWorkers() { } public void setLiveWorkers(int liveWorkers) { - this.mLiveWorkers = liveWorkers; + mLiveWorkers = liveWorkers; } public int getLostWorkers() { @@ -234,7 +229,7 @@ public int getLostWorkers() { } public void setLostWorkers(int lostWorkers) { - this.mLostWorkers = lostWorkers; + mLostWorkers = lostWorkers; } public String getFreeCapacity() { @@ -242,7 +237,7 @@ public String getFreeCapacity() { } public void setFreeCapacity(String freeCapacity) { - this.mFreeCapacity = freeCapacity; + mFreeCapacity = freeCapacity; } public Map getTotalCapacityOnTiers() { @@ -250,7 +245,7 @@ public Map getTotalCapacityOnTiers() { } public void setTotalCapacityOnTiers(Map totalCapacityOnTiers) { - this.mTotalCapacityOnTiers = totalCapacityOnTiers; + mTotalCapacityOnTiers = totalCapacityOnTiers; } public Map getUsedCapacityOnTiers() { @@ -258,6 +253,6 @@ public Map getUsedCapacityOnTiers() { } public void setUsedCapacityOnTiers(Map usedCapacityOnTiers) { - this.mUsedCapacityOnTiers = usedCapacityOnTiers; + mUsedCapacityOnTiers = usedCapacityOnTiers; } } \ No newline at end of file From fe050c4117b1c34073caa11b12d4ec6752e4f615 Mon Sep 17 00:00:00 2001 From: twalluxio Date: Thu, 24 Aug 2023 14:45:17 +0800 Subject: [PATCH 07/21] new json-style metrics report --- .../cli/fsadmin/report/MetricsCommand.java | 63 ++-------- .../cli/fsadmin/report/MetricsOutput.java | 109 ++++++++++++++++++ 2 files changed, 119 insertions(+), 53 deletions(-) create mode 100644 dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java index efef5cd188e0..41db4752b2b5 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java @@ -16,6 +16,7 @@ import alluxio.metrics.MetricsSystem; import alluxio.util.FormatUtils; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -23,12 +24,10 @@ import java.io.IOException; import java.io.PrintStream; +import java.io.Serializable; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; -import java.util.Locale; -import java.util.Map; -import java.util.SortedSet; -import java.util.TreeSet; +import java.util.*; /** * Prints Alluxio metrics information. @@ -60,56 +59,14 @@ public MetricsCommand(MetricsMasterClient metricsMasterClient, PrintStream print * @return 0 on success, 1 otherwise */ public int run() throws IOException { - ObjectMapper mapper = new ObjectMapper(); - ArrayNode metricsInfo = mapper.createArrayNode(); - - mMetricsMap = mMetricsMasterClient.getMetrics(); - SortedSet names = new TreeSet<>(mMetricsMap.keySet()); - for (String name : names) { - if (!isAlluxioMetric(name)) { - continue; - } - MetricValue metricValue = mMetricsMap.get(name); - String strValue; - if (metricValue.hasStringValue()) { - strValue = metricValue.getStringValue(); - } else { - double doubleValue = metricValue.getDoubleValue(); - if (name.contains(BYTES_METRIC_IDENTIFIER)) { - // Bytes long can be transformed to human-readable format - strValue = FormatUtils.getSizeFromBytes((long) doubleValue); - if (name.contains(THROUGHPUT_METRIC_IDENTIFIER)) { - strValue = strValue + "/MIN"; - } - } else if (DoubleMath.isMathematicalInteger(doubleValue)) { - strValue = DECIMAL_FORMAT.format((long) doubleValue); - } else { - strValue = String.valueOf(doubleValue); - } - } - ObjectNode metric = mapper.createObjectNode(); - metric.put("Name", name); - metric.put("Type", metricValue.getMetricType().toString()); - metric.put("Value", strValue); - metricsInfo.add(metric); + ObjectMapper objectMapper = new ObjectMapper(); + MetricsOutput metricsInfo = new MetricsOutput(mMetricsMasterClient.getMetrics()); + try { + String json = objectMapper.writeValueAsString(metricsInfo); + mPrintStream.println(json); + } catch (JsonProcessingException e) { + e.printStackTrace(); } - - mPrintStream.println(mapper.writerWithDefaultPrettyPrinter().writeValueAsString(metricsInfo)); return 0; } - - /** - * Checks if a metric is Alluxio metric. - * - * @param name name of the metrics to check - * @return true if a metric is an Alluxio metric, false otherwise - */ - private boolean isAlluxioMetric(String name) { - for (MetricsSystem.InstanceType instance : MetricsSystem.InstanceType.values()) { - if (name.startsWith(instance.toString())) { - return true; - } - } - return false; - } } diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java new file mode 100644 index 000000000000..f45772a12414 --- /dev/null +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java @@ -0,0 +1,109 @@ +package alluxio.cli.fsadmin.report; + +import alluxio.grpc.MetricValue; +import alluxio.metrics.MetricsSystem; +import alluxio.util.FormatUtils; +import com.fasterxml.jackson.core.SerializableString; +import com.google.common.math.DoubleMath; + +import java.io.Serializable; +import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +public class MetricsOutput { + private static final String BYTES_METRIC_IDENTIFIER = "Bytes"; + private static final String THROUGHPUT_METRIC_IDENTIFIER = "Throughput"; + private static final DecimalFormat DECIMAL_FORMAT + = new DecimalFormat("###,###.#####", new DecimalFormatSymbols(Locale.US)); + private List mMetricsInfo; + + private class SerializableMetricInfo { + private String mKey; + private String mType; + private String mValue; + + public SerializableMetricInfo(Map.Entry entry) { + mKey = entry.getKey(); + mType = entry.getValue().getMetricType().toString(); + if (entry.getValue().hasStringValue()) { + mValue = entry.getValue().getStringValue(); + } else { + double doubleValue = entry.getValue().getDoubleValue(); + if (mKey.contains(BYTES_METRIC_IDENTIFIER)) { + // Bytes long can be transformed to human-readable format + mValue = FormatUtils.getSizeFromBytes((long) doubleValue); + if (mKey.contains(THROUGHPUT_METRIC_IDENTIFIER)) { + mValue = mValue + "/MIN"; + } + } else if (DoubleMath.isMathematicalInteger(doubleValue)) { + mValue = DECIMAL_FORMAT.format((long) doubleValue); + } else { + mValue = String.valueOf(doubleValue); + } + } + } + + public String getKey() { + return mKey; + } + + public void setKey(String key) { + mKey = key; + } + + public String getType() { + return mType; + } + + public void setType(String type) { + mType = type; + } + + public String getValue() { + return mValue; + } + + public void setValue(String value) { + mValue = value; + } + } + + public MetricsOutput(Map metrics){ + mMetricsInfo = new ArrayList<>(); + for (Map.Entry entry : metrics.entrySet()){ + String key = entry.getKey(); + if (!isAlluxioMetric(key)) { + continue; + } + mMetricsInfo.add(new SerializableMetricInfo(entry)); + } + } + + /** + * Checks if a metric is Alluxio metric. + * + * @param name name of the metrics to check + * @return true if a metric is an Alluxio metric, false otherwise + */ + private boolean isAlluxioMetric(String name) { + for (MetricsSystem.InstanceType instance : MetricsSystem.InstanceType.values()) { + if (name.startsWith(instance.toString())) { + return true; + } + } + return false; + } + + public List getMetricsInfo() { + return mMetricsInfo; + } + + public void setMetricsInfo(List metricsInfo) { + mMetricsInfo = metricsInfo; + } + +} From b61b18748305bde0bb7252b84d40110ce460a234 Mon Sep 17 00:00:00 2001 From: twalluxio Date: Mon, 28 Aug 2023 10:25:16 +0800 Subject: [PATCH 08/21] revert CapacityCommand.java --- .../cli/fsadmin/report/CapacityCommand.java | 190 ++++++++++-------- 1 file changed, 106 insertions(+), 84 deletions(-) diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/CapacityCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/CapacityCommand.java index 4e529f990156..edde152ddc14 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/CapacityCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/CapacityCommand.java @@ -28,10 +28,7 @@ import alluxio.util.FormatUtils; import alluxio.wire.WorkerInfo; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.base.Strings; import org.apache.commons.cli.CommandLine; import java.io.IOException; @@ -54,8 +51,11 @@ * Prints Alluxio capacity information. */ public class CapacityCommand { + private static final int INDENT_SIZE = 4; + private BlockMasterClient mBlockMasterClient; private PrintStream mPrintStream; + private int mIndentationLevel = 0; private long mSumCapacityBytes; private long mSumUsedBytes; private Map mSumCapacityBytesOnTierMap; @@ -63,8 +63,6 @@ public class CapacityCommand { private TreeMap> mCapacityTierInfoMap; private Map> mUsedTierInfoMap; - private ObjectMapper mMapper; - private ObjectNode mCapacityInfo; private static final String LIVE_WORKER_STATE = "In Service"; private static final String LOST_WORKER_STATE = "Out of Service"; @@ -120,20 +118,17 @@ public void generateCapacityReport(GetWorkerReportOptions options, AlluxioConfig workerInfoList = mBlockMasterClient.getWorkerReport(options); } if (workerInfoList.size() == 0) { - mPrintStream.println("No workers found."); + print("No workers found."); return; } Collections.sort(workerInfoList, new WorkerInfo.LastContactSecComparator()); collectWorkerInfo(workerInfoList); - generateAggregatedInfo(options); - generateWorkerInfo(workerInfoList); + printAggregatedInfo(options); + printWorkerInfo(workerInfoList); if (workerRegisterToAllMasters) { printWorkerAllMasterConnectionInfo(allMastersWorkerInfo); } - mPrintStream.printf("Capacity information for %s workers:%n", - options.getWorkerRange().toString().toLowerCase()); - mPrintStream.println(mMapper.writerWithDefaultPrettyPrinter().writeValueAsString(mCapacityInfo)); } /** @@ -182,44 +177,36 @@ private void collectWorkerInfo(List workerInfoList) { * * @param options GetWorkerReportOptions to check if input is invalid */ - private void generateAggregatedInfo(GetWorkerReportOptions options) throws JsonProcessingException { - ObjectNode capacityMetrics = mMapper.createObjectNode(); - ObjectNode totalCapacity = mMapper.createObjectNode(); - ObjectNode usedCapacity = mMapper.createObjectNode(); - - ArrayNode totalCapacityTiers = mMapper.createArrayNode(); + private void printAggregatedInfo(GetWorkerReportOptions options) { + mIndentationLevel = 0; + print(String.format("Capacity information for %s workers: ", + options.getWorkerRange().toString().toLowerCase())); + + mIndentationLevel++; + print("Total Capacity: " + FormatUtils.getSizeFromBytes(mSumCapacityBytes)); + mIndentationLevel++; for (Map.Entry totalBytesTier : mSumCapacityBytesOnTierMap.entrySet()) { long value = totalBytesTier.getValue(); - ObjectNode totalCapacityPerTier = mMapper.createObjectNode(); - totalCapacityPerTier.put("Tier", totalBytesTier.getKey()); - totalCapacityPerTier.put("Size", FormatUtils.getSizeFromBytes(value)); - totalCapacityTiers.add(totalCapacityPerTier); + print("Tier: " + totalBytesTier.getKey() + + " Size: " + FormatUtils.getSizeFromBytes(value)); } + mIndentationLevel--; - ArrayNode usedCapacityTiers = mMapper.createArrayNode(); + print("Used Capacity: " + + FormatUtils.getSizeFromBytes(mSumUsedBytes)); + mIndentationLevel++; for (Map.Entry usedBytesTier : mSumUsedBytesOnTierMap.entrySet()) { long value = usedBytesTier.getValue(); - ObjectNode usedCapacityPerTier = mMapper.createObjectNode(); - usedCapacityPerTier.put("Tier", usedBytesTier.getKey()); - usedCapacityPerTier.put("Size", FormatUtils.getSizeFromBytes(value)); - usedCapacityTiers.add(usedCapacityPerTier); + print("Tier: " + usedBytesTier.getKey() + + " Size: " + FormatUtils.getSizeFromBytes(value)); } - - totalCapacity.put("All", FormatUtils.getSizeFromBytes(mSumCapacityBytes)); - totalCapacity.set("Tiers", totalCapacityTiers); - usedCapacity.put("All", FormatUtils.getSizeFromBytes(mSumUsedBytes)); - usedCapacity.set("Tiers", usedCapacityTiers); - - capacityMetrics.set("Total Capacity", totalCapacity); - capacityMetrics.set("Used Capacity", usedCapacity); + mIndentationLevel--; if (mSumCapacityBytes != 0) { int usedPercentage = (int) (100L * mSumUsedBytes / mSumCapacityBytes); - capacityMetrics.put("Used Percentage", String.format("%s%%", usedPercentage)); - capacityMetrics.put("Free Percentage", String.format("%s%%", 100 - usedPercentage)); + print(String.format("Used Percentage: " + "%s%%", usedPercentage)); + print(String.format("Free Percentage: " + "%s%%", 100 - usedPercentage)); } - - mCapacityInfo.set("Capacity Metrics", capacityMetrics); } private String getMasterAddressesString(Set addresses) { @@ -309,58 +296,95 @@ private void printWorkerAllMasterConnectionInfo( * * @param workerInfoList the worker info list to get info from */ - private void generateWorkerInfo(List workerInfoList) { - boolean isShort = false; + private void printWorkerInfo(List workerInfoList) { + mIndentationLevel = 0; if (mCapacityTierInfoMap.size() == 0) { return; } else if (mCapacityTierInfoMap.size() == 1) { - isShort = true; + // Do not print Total value when only one tier exists + printShortWorkerInfo(workerInfoList); + return; } - - ArrayNode workerInfo = mMapper.createArrayNode(); Set tiers = mCapacityTierInfoMap.keySet(); + String tiersInfo = String.format(Strings.repeat("%-14s", tiers.size()), tiers.toArray()); + String longInfoFormat = getInfoFormat(workerInfoList, false); + print(String.format("%n" + longInfoFormat, + "Worker Name", "State", "Last Heartbeat", "Storage", "Total", tiersInfo, + "Version", "Revision")); for (WorkerInfo info : workerInfoList) { - ObjectNode infoPerWorker = mMapper.createObjectNode(); - - ArrayNode tiersInfo = mMapper.createArrayNode(); - - if (isShort) { - // TODO(jiacheng): test BOTH long and short output - // Do not print Total value when only one tier exists - ObjectNode tierInfo = mMapper.createObjectNode(); - tierInfo.put("Tier", mCapacityTierInfoMap.firstKey()); - tierInfo.put("Capacity", FormatUtils.getSizeFromBytes(info.getCapacityBytes())); - tierInfo.put("Used", FormatUtils.getSizeFromBytes(info.getUsedBytes())); - if (info.getCapacityBytes() != 0) { - tierInfo.put("Used percentage", String.format(" %s%%", (int) (100L * info.getUsedBytes() / info.getCapacityBytes()))); - } - tiersInfo.add(tierInfo); - } else { - for (String tier : tiers){ - ObjectNode tierInfo = mMapper.createObjectNode(); - tierInfo.put("Tier", tier); - tierInfo.put("Capacity", getWorkerFormattedTierValues(mCapacityTierInfoMap, info.getAddress().getHost())); - tierInfo.put("Used", getWorkerFormattedTierValues(mUsedTierInfoMap, info.getAddress().getHost())); - if (info.getCapacityBytes() != 0) { - tierInfo.put("Used percentage", - String.format("%s%%", (int) (100L * info.getUsedBytes() / info.getCapacityBytes()))); - } - tiersInfo.add(tierInfo); - } + String workerName = info.getAddress().getHost(); + + long usedBytes = info.getUsedBytes(); + long capacityBytes = info.getCapacityBytes(); + + String usedPercentageInfo = ""; + if (capacityBytes != 0) { + int usedPercentage = (int) (100L * usedBytes / capacityBytes); + usedPercentageInfo = String.format(" (%s%%)", usedPercentage); } - infoPerWorker.put("Worker Name", info.getAddress().getHost()); - infoPerWorker.put("State", info.getState()); - infoPerWorker.put("Last Heartbeat", info.getLastContactSec()); - infoPerWorker.set("tiers", tiersInfo); - infoPerWorker.put("Version", info.getVersion()); - infoPerWorker.put("Revision", info.getRevision()); + String capacityTierInfo = getWorkerFormattedTierValues(mCapacityTierInfoMap, workerName); + String usedTierInfo = getWorkerFormattedTierValues(mUsedTierInfoMap, workerName); + + print(String.format(longInfoFormat, workerName, info.getState(), + info.getLastContactSec(), "capacity", + FormatUtils.getSizeFromBytes(capacityBytes), capacityTierInfo, + info.getVersion(), info.getRevision())); + print(String.format(longInfoFormat, "", "", "", "used", + FormatUtils.getSizeFromBytes(usedBytes) + usedPercentageInfo, usedTierInfo, + "", "")); + } + } + + /** + * Prints worker information when only one tier exists. + * + * @param workerInfoList the worker info list to get info from + */ + private void printShortWorkerInfo(List workerInfoList) { + String tier = String.format("%-16s", mCapacityTierInfoMap.firstKey()); + String shortInfoFormat = getInfoFormat(workerInfoList, true); + print(String.format("%n" + shortInfoFormat, + "Worker Name", "State", "Last Heartbeat", "Storage", tier, "Version", "Revision")); + + for (WorkerInfo info : workerInfoList) { + long capacityBytes = info.getCapacityBytes(); + long usedBytes = info.getUsedBytes(); - workerInfo.add(infoPerWorker); + String usedPercentageInfo = ""; + if (capacityBytes != 0) { + int usedPercentage = (int) (100L * usedBytes / capacityBytes); + usedPercentageInfo = String.format(" (%s%%)", usedPercentage); + } + print(String.format(shortInfoFormat, info.getAddress().getHost(), info.getState(), + info.getLastContactSec(), "capacity", + String.format("%-16s", FormatUtils.getSizeFromBytes(capacityBytes)), + info.getVersion(), info.getRevision())); + print(String.format(shortInfoFormat, "", "", "", "used", + String.format("%-16s", FormatUtils.getSizeFromBytes(usedBytes) + usedPercentageInfo), + "", "")); } + } - mCapacityInfo.set("Worker Information", workerInfo); + /** + * Gets the info format according to the longest worker name. + * @param workerInfoList the worker info list to get info from + * @param isShort whether exists only one tier + * @return the info format for printing long/short worker info + */ + private String getInfoFormat(List workerInfoList, boolean isShort) { + int maxWorkerNameLength = workerInfoList.stream().map(w -> w.getAddress().getHost().length()) + .max(Comparator.comparing(Integer::intValue)).get(); + int firstIndent = 16; + if (firstIndent <= maxWorkerNameLength) { + // extend first indent according to the longest worker name by default 5 + firstIndent = maxWorkerNameLength + 5; + } + if (isShort) { + return "%-" + firstIndent + "s %-15s %-16s %-13s %s %-16s %-40s"; + } + return "%-" + firstIndent + "s %-15s %-16s %-13s %-16s %s %-16s %-40s"; } /** @@ -425,9 +449,6 @@ private void initVariables() { // TierInfoMap is of form Map> mCapacityTierInfoMap = new TreeMap<>(FileSystemAdminShellUtils::compareTierNames); mUsedTierInfoMap = new TreeMap<>(FileSystemAdminShellUtils::compareTierNames); - - mMapper = new ObjectMapper(); - mCapacityInfo = mMapper.createObjectNode(); } /** @@ -436,7 +457,8 @@ private void initVariables() { * @param text information to print */ private void print(String text) { - mPrintStream.println(text); + String indent = Strings.repeat(" ", mIndentationLevel * INDENT_SIZE); + mPrintStream.println(indent + text); } /** @@ -453,4 +475,4 @@ public static String getUsage() { + " -workers Specified workers, " + "host names or ip addresses separated by \",\"\n"; } -} +} \ No newline at end of file From 3950f4a383968e73eec53d5f4e752d9d71fe0954 Mon Sep 17 00:00:00 2001 From: twalluxio Date: Mon, 28 Aug 2023 10:27:11 +0800 Subject: [PATCH 09/21] hygiene on metrics report --- .../cli/fsadmin/report/MetricsCommand.java | 17 +++++------------ .../cli/fsadmin/report/MetricsOutput.java | 2 -- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java index 41db4752b2b5..0ae8c4dd92e3 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java @@ -13,18 +13,13 @@ import alluxio.client.metrics.MetricsMasterClient; import alluxio.grpc.MetricValue; -import alluxio.metrics.MetricsSystem; -import alluxio.util.FormatUtils; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.math.DoubleMath; +import org.jline.utils.Log; import java.io.IOException; import java.io.PrintStream; -import java.io.Serializable; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; import java.util.*; @@ -33,14 +28,8 @@ * Prints Alluxio metrics information. */ public class MetricsCommand { - private static final String BYTES_METRIC_IDENTIFIER = "Bytes"; - private static final String THROUGHPUT_METRIC_IDENTIFIER = "Throughput"; - private static final DecimalFormat DECIMAL_FORMAT - = new DecimalFormat("###,###.#####", new DecimalFormatSymbols(Locale.US)); - private final MetricsMasterClient mMetricsMasterClient; private final PrintStream mPrintStream; - private Map mMetricsMap; /** * Creates a new instance of {@link MetricsCommand}. @@ -65,7 +54,11 @@ public int run() throws IOException { String json = objectMapper.writeValueAsString(metricsInfo); mPrintStream.println(json); } catch (JsonProcessingException e) { + mPrintStream.println("Failed to convert metricsInfo output to JSON. " + + "Check the command line log for the detailed error message."); + Log.error("Failed to output JSON object {}", metricsInfo); e.printStackTrace(); + return -1; } return 0; } diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java index f45772a12414..2c410c64885c 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java @@ -3,10 +3,8 @@ import alluxio.grpc.MetricValue; import alluxio.metrics.MetricsSystem; import alluxio.util.FormatUtils; -import com.fasterxml.jackson.core.SerializableString; import com.google.common.math.DoubleMath; -import java.io.Serializable; import java.text.DecimalFormat; import java.text.DecimalFormatSymbols; import java.util.ArrayList; From c64f18620c1b46d6acc8754cce5d47dd37ad5525 Mon Sep 17 00:00:00 2001 From: twalluxio Date: Mon, 28 Aug 2023 10:27:38 +0800 Subject: [PATCH 10/21] improve property readability --- .../cli/fsadmin/report/SummaryOutput.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java index bf47cf941af0..5344c9b37ea5 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java @@ -22,9 +22,9 @@ public class SummaryOutput { private boolean mSafeMode; private List mZookeeperAddress; - private boolean mZookeeper; + private boolean mUseZookeeper; private List mRaftJournalAddress; - private boolean mRaftJournal; + private boolean mUseRaftJournal; private List mMasterVersions; private int mLiveWorkers; @@ -90,9 +90,9 @@ public SummaryOutput(MasterInfo masterInfo, BlockMasterInfo blockMasterInfo, Str mSafeMode = masterInfo.getSafeMode(); mZookeeperAddress = masterInfo.getZookeeperAddressesList(); - mZookeeper = !mZookeeperAddress.isEmpty(); - mRaftJournal = masterInfo.getRaftJournal(); - if (mRaftJournal) { + mUseZookeeper = !mZookeeperAddress.isEmpty(); + mUseRaftJournal = masterInfo.getRaftJournal(); + if (mUseRaftJournal) { mRaftJournalAddress = masterInfo.getRaftAddressList(); } else { mRaftJournalAddress = new ArrayList<>(); @@ -184,12 +184,12 @@ public void setZookeeperAddress(List zookeeperAddress) { mZookeeperAddress = zookeeperAddress; } - public boolean ismZookeeper() { - return mZookeeper; + public boolean isUseZookeeper() { + return mUseZookeeper; } - public void setZookeeper(boolean mZookeeper) { - mZookeeper = mZookeeper; + public void setUseZookeeper(boolean useZookeeper) { + mUseZookeeper = useZookeeper; } public List getRaftJournalAddress() { @@ -200,12 +200,12 @@ public void setRaftJournalAddress(List raftJournalAddress) { mRaftJournalAddress = raftJournalAddress; } - public boolean ismRaftJournal() { - return mRaftJournal; + public boolean isUseRaftJournal() { + return mUseRaftJournal; } - public void setRaftJournal(boolean raftJournal) { - mRaftJournal = raftJournal; + public void setUseRaftJournal(boolean useRaftJournal) { + mUseRaftJournal = useRaftJournal; } public List getMasterVersions() { From 326ede000cb8efa3a19d40464cc56e1ea8fa3854 Mon Sep 17 00:00:00 2001 From: twalluxio Date: Mon, 28 Aug 2023 10:28:32 +0800 Subject: [PATCH 11/21] improve exception catcher and indention --- .../report/JobServiceMetricsCommand.java | 5 ++++ .../cli/fsadmin/report/SummaryCommand.java | 25 +++++++++++-------- .../cli/fsadmin/report/UfsCommand.java | 10 ++++---- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java index 33edf268b110..313cf75635f4 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java @@ -15,6 +15,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import org.jline.utils.Log; import java.io.IOException; import java.io.PrintStream; @@ -62,7 +63,11 @@ public int run() throws IOException { String json = objectMapper.writeValueAsString(jobServiceInfo); mPrintStream.println(json); } catch (JsonProcessingException e) { + mPrintStream.println("Failed to convert jobServiceInfo output to JSON. " + + "Check the command line log for the detailed error message."); + Log.error("Failed to output JSON object {}", jobServiceInfo); e.printStackTrace(); + return -1; } return 0; } diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java index 48e4b29bf976..88c7248381a1 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import org.jline.utils.Log; import java.io.IOException; import java.io.PrintStream; @@ -59,19 +60,19 @@ public SummaryCommand(MetaMasterClient metaMasterClient, */ public int run() throws IOException { Set masterInfoFilter = new HashSet<>(Arrays - .asList(MasterInfoField.LEADER_MASTER_ADDRESS, MasterInfoField.WEB_PORT, - MasterInfoField.RPC_PORT, MasterInfoField.START_TIME_MS, - MasterInfoField.UP_TIME_MS, MasterInfoField.VERSION, - MasterInfoField.SAFE_MODE, MasterInfoField.ZOOKEEPER_ADDRESSES, - MasterInfoField.RAFT_JOURNAL, MasterInfoField.RAFT_ADDRESSES, - MasterInfoField.MASTER_VERSION)); + .asList(MasterInfoField.LEADER_MASTER_ADDRESS, MasterInfoField.WEB_PORT, + MasterInfoField.RPC_PORT, MasterInfoField.START_TIME_MS, + MasterInfoField.UP_TIME_MS, MasterInfoField.VERSION, + MasterInfoField.SAFE_MODE, MasterInfoField.ZOOKEEPER_ADDRESSES, + MasterInfoField.RAFT_JOURNAL, MasterInfoField.RAFT_ADDRESSES, + MasterInfoField.MASTER_VERSION)); MasterInfo masterInfo = mMetaMasterClient.getMasterInfo(masterInfoFilter); Set blockMasterInfoFilter = new HashSet<>(Arrays - .asList(BlockMasterInfoField.LIVE_WORKER_NUM, BlockMasterInfoField.LOST_WORKER_NUM, - BlockMasterInfoField.CAPACITY_BYTES, BlockMasterInfoField.USED_BYTES, - BlockMasterInfoField.FREE_BYTES, BlockMasterInfoField.CAPACITY_BYTES_ON_TIERS, - BlockMasterInfoField.USED_BYTES_ON_TIERS)); + .asList(BlockMasterInfoField.LIVE_WORKER_NUM, BlockMasterInfoField.LOST_WORKER_NUM, + BlockMasterInfoField.CAPACITY_BYTES, BlockMasterInfoField.USED_BYTES, + BlockMasterInfoField.FREE_BYTES, BlockMasterInfoField.CAPACITY_BYTES_ON_TIERS, + BlockMasterInfoField.USED_BYTES_ON_TIERS)); BlockMasterInfo blockMasterInfo = mBlockMasterClient.getBlockMasterInfo(blockMasterInfoFilter); ObjectMapper objectMapper = new ObjectMapper(); @@ -80,7 +81,11 @@ public int run() throws IOException { String json = objectMapper.writeValueAsString(summaryInfo); mPrintStream.println(json); } catch (JsonProcessingException e) { + mPrintStream.println("Failed to convert summaryInfo output to JSON. " + + "Check the command line log for the detailed error message."); + Log.error("Failed to output JSON object {}", summaryInfo); e.printStackTrace(); + return -1; } return 0; diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java index eb6f0289231c..25821a2b7b30 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java @@ -12,16 +12,12 @@ package alluxio.cli.fsadmin.report; import alluxio.client.file.FileSystemMasterClient; -import alluxio.util.FormatUtils; import alluxio.wire.MountPointInfo; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; +import org.jline.utils.Log; import java.io.IOException; -import java.io.PrintStream; -import java.util.Comparator; import java.util.Map; /** @@ -51,7 +47,11 @@ public int run() throws IOException { String json = objectMapper.writeValueAsString(mountTable); System.out.println(json); } catch (JsonProcessingException e) { + System.out.println("Failed to convert mountTable output to JSON. " + + "Check the command line log for the detailed error message."); + Log.error("Failed to output JSON object {}", mountTable); e.printStackTrace(); + return -1; } return 0; } From 9d696ae4752ed7f85173f82970cb0ad2d3a16ded Mon Sep 17 00:00:00 2001 From: twalluxio Date: Tue, 29 Aug 2023 15:31:38 +0800 Subject: [PATCH 12/21] remove capacity command in alluxio fsadmin shell --- .../cli/fsadmin/command/ReportCommand.java | 19 - .../cli/fsadmin/report/CapacityCommand.java | 478 ------------------ .../fsadmin/report/CapacityCommandTest.java | 307 ----------- .../CapacityCommandIntegrationTest.java | 92 ---- 4 files changed, 896 deletions(-) delete mode 100644 dora/shell/src/main/java/alluxio/cli/fsadmin/report/CapacityCommand.java delete mode 100644 dora/shell/src/test/java/alluxio/cli/fsadmin/report/CapacityCommandTest.java delete mode 100644 dora/tests/integration/src/test/java/alluxio/client/cli/fsadmin/command/CapacityCommandIntegrationTest.java diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/command/ReportCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/command/ReportCommand.java index 7c87387823b4..a2cafd6f0916 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/command/ReportCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/command/ReportCommand.java @@ -14,7 +14,6 @@ import alluxio.annotation.PublicApi; import alluxio.cli.CommandUtils; import alluxio.cli.fsadmin.FileSystemAdminShellUtils; -import alluxio.cli.fsadmin.report.CapacityCommand; import alluxio.cli.fsadmin.report.JobServiceMetricsCommand; import alluxio.cli.fsadmin.report.MetricsCommand; import alluxio.cli.fsadmin.report.ProxyCommand; @@ -78,7 +77,6 @@ public final class ReportCommand extends AbstractFsAdminCommand { .build(); enum Command { - CAPACITY, // Report worker capacity information METRICS, // Report metrics information SUMMARY, // Report cluster summary UFS, // Report under filesystem information @@ -120,9 +118,6 @@ public int run(CommandLine cl) throws IOException { Command command = Command.SUMMARY; if (args.length == 1) { switch (args[0]) { - case "capacity": - command = Command.CAPACITY; - break; case "metrics": command = Command.METRICS; break; @@ -145,20 +140,7 @@ public int run(CommandLine cl) throws IOException { } } - // Only capacity category has [category args] - if (!command.equals(Command.CAPACITY)) { - if (cl.getOptions().length > 0) { - throw new InvalidArgumentException( - String.format("report %s does not support arguments: %s", - command.toString().toLowerCase(), cl.getOptions()[0].getOpt())); - } - } - switch (command) { - case CAPACITY: - CapacityCommand capacityCommand = new CapacityCommand(mBlockClient, mPrintStream); - capacityCommand.run(cl); - break; case METRICS: MetricsCommand metricsCommand = new MetricsCommand(mMetricsClient, mPrintStream); metricsCommand.run(); @@ -225,7 +207,6 @@ public static String description() { + "Where [category] is an optional argument. If no arguments are passed in, " + "summary information will be printed out.\n" + "[category] can be one of the following:\n" - + " capacity worker capacity information\n" + " metrics metrics information\n" + " summary cluster summary\n" + " ufs under storage system information\n" diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/CapacityCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/CapacityCommand.java deleted file mode 100644 index edde152ddc14..000000000000 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/CapacityCommand.java +++ /dev/null @@ -1,478 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.cli.fsadmin.report; - -import alluxio.cli.fsadmin.FileSystemAdminShellUtils; -import alluxio.cli.fsadmin.command.ReportCommand; -import alluxio.client.block.AllMastersWorkerInfo; -import alluxio.client.block.BlockMasterClient; -import alluxio.client.block.options.GetWorkerReportOptions; -import alluxio.client.block.options.GetWorkerReportOptions.WorkerInfoField; -import alluxio.client.block.options.GetWorkerReportOptions.WorkerRange; -import alluxio.client.block.util.WorkerInfoUtil; -import alluxio.collections.Pair; -import alluxio.conf.AlluxioConfiguration; -import alluxio.conf.Configuration; -import alluxio.conf.PropertyKey; -import alluxio.exception.status.InvalidArgumentException; -import alluxio.grpc.Scope; -import alluxio.util.FormatUtils; -import alluxio.wire.WorkerInfo; - -import com.google.common.base.Strings; -import org.apache.commons.cli.CommandLine; - -import java.io.IOException; -import java.io.PrintStream; -import java.net.InetSocketAddress; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.stream.Collectors; - -/** - * Prints Alluxio capacity information. - */ -public class CapacityCommand { - private static final int INDENT_SIZE = 4; - - private BlockMasterClient mBlockMasterClient; - private PrintStream mPrintStream; - private int mIndentationLevel = 0; - private long mSumCapacityBytes; - private long mSumUsedBytes; - private Map mSumCapacityBytesOnTierMap; - private Map mSumUsedBytesOnTierMap; - private TreeMap> mCapacityTierInfoMap; - private Map> mUsedTierInfoMap; - - private static final String LIVE_WORKER_STATE = "In Service"; - private static final String LOST_WORKER_STATE = "Out of Service"; - - /** - * Creates a new instance of {@link CapacityCommand}. - * - * @param blockMasterClient client to connect to block master - * @param printStream stream to print summary information to - */ - public CapacityCommand(BlockMasterClient blockMasterClient, PrintStream printStream) { - mBlockMasterClient = blockMasterClient; - mPrintStream = printStream; - } - - /** - * Runs report capacity command. - * - * @param cl CommandLine to get client options - * @return 0 on success, 1 otherwise - */ - public int run(CommandLine cl) throws IOException { - if (cl.hasOption(ReportCommand.HELP_OPTION_NAME)) { - System.out.println(getUsage()); - return 0; - } - - GetWorkerReportOptions options = getOptions(cl); - Configuration.loadClusterDefaults(mBlockMasterClient.getConfAddress(), Scope.CLIENT); - generateCapacityReport(options, Configuration.global()); - return 0; - } - - /** - * Generates capacity report. - * - * @param options GetWorkerReportOptions to get worker report - * @param conf the cluster configuration - */ - public void generateCapacityReport(GetWorkerReportOptions options, AlluxioConfiguration conf) - throws IOException { - boolean workerRegisterToAllMasters = - conf.getBoolean(PropertyKey.WORKER_REGISTER_TO_ALL_MASTERS); - - final List workerInfoList; - final AllMastersWorkerInfo allMastersWorkerInfo; - if (workerRegisterToAllMasters) { - allMastersWorkerInfo = - WorkerInfoUtil.getWorkerReportsFromAllMasters( - conf, mBlockMasterClient, options); - workerInfoList = allMastersWorkerInfo.getPrimaryMasterWorkerInfo(); - } else { - allMastersWorkerInfo = null; - workerInfoList = mBlockMasterClient.getWorkerReport(options); - } - if (workerInfoList.size() == 0) { - print("No workers found."); - return; - } - Collections.sort(workerInfoList, new WorkerInfo.LastContactSecComparator()); - - collectWorkerInfo(workerInfoList); - printAggregatedInfo(options); - printWorkerInfo(workerInfoList); - if (workerRegisterToAllMasters) { - printWorkerAllMasterConnectionInfo(allMastersWorkerInfo); - } - } - - /** - * Collects worker capacity information. - * - * @param workerInfoList the worker info list to collect info from - */ - private void collectWorkerInfo(List workerInfoList) { - initVariables(); - for (WorkerInfo workerInfo : workerInfoList) { - long usedBytes = workerInfo.getUsedBytes(); - long capacityBytes = workerInfo.getCapacityBytes(); - mSumCapacityBytes += capacityBytes; - mSumUsedBytes += usedBytes; - - String workerName = workerInfo.getAddress().getHost(); - - Map totalBytesOnTiers = workerInfo.getCapacityBytesOnTiers(); - for (Map.Entry totalBytesTier : totalBytesOnTiers.entrySet()) { - String tier = totalBytesTier.getKey(); - long value = totalBytesTier.getValue(); - mSumCapacityBytesOnTierMap.put(tier, - value + mSumCapacityBytesOnTierMap.getOrDefault(tier, 0L)); - - Map map = mCapacityTierInfoMap.getOrDefault(tier, new HashMap<>()); - map.put(workerName, FormatUtils.getSizeFromBytes(value)); - mCapacityTierInfoMap.put(tier, map); - } - - Map usedBytesOnTiers = workerInfo.getUsedBytesOnTiers(); - for (Map.Entry usedBytesTier: usedBytesOnTiers.entrySet()) { - String tier = usedBytesTier.getKey(); - long value = usedBytesTier.getValue(); - mSumUsedBytesOnTierMap.put(tier, - value + mSumUsedBytesOnTierMap.getOrDefault(tier, 0L)); - - Map map = mUsedTierInfoMap.getOrDefault(tier, new HashMap<>()); - map.put(workerName, FormatUtils.getSizeFromBytes(value)); - mUsedTierInfoMap.put(tier, map); - } - } - } - - /** - * Prints aggregated worker capacity information. - * - * @param options GetWorkerReportOptions to check if input is invalid - */ - private void printAggregatedInfo(GetWorkerReportOptions options) { - mIndentationLevel = 0; - print(String.format("Capacity information for %s workers: ", - options.getWorkerRange().toString().toLowerCase())); - - mIndentationLevel++; - print("Total Capacity: " + FormatUtils.getSizeFromBytes(mSumCapacityBytes)); - mIndentationLevel++; - for (Map.Entry totalBytesTier : mSumCapacityBytesOnTierMap.entrySet()) { - long value = totalBytesTier.getValue(); - print("Tier: " + totalBytesTier.getKey() - + " Size: " + FormatUtils.getSizeFromBytes(value)); - } - mIndentationLevel--; - - print("Used Capacity: " - + FormatUtils.getSizeFromBytes(mSumUsedBytes)); - mIndentationLevel++; - for (Map.Entry usedBytesTier : mSumUsedBytesOnTierMap.entrySet()) { - long value = usedBytesTier.getValue(); - print("Tier: " + usedBytesTier.getKey() - + " Size: " + FormatUtils.getSizeFromBytes(value)); - } - mIndentationLevel--; - - if (mSumCapacityBytes != 0) { - int usedPercentage = (int) (100L * mSumUsedBytes / mSumCapacityBytes); - print(String.format("Used Percentage: " + "%s%%", usedPercentage)); - print(String.format("Free Percentage: " + "%s%%", 100 - usedPercentage)); - } - } - - private String getMasterAddressesString(Set addresses) { - StringBuilder sb = new StringBuilder(); - sb.append("["); - List addressStrings = - addresses.stream().map(it -> masterAddressToString(it, addresses)).sorted().collect( - Collectors.toList()); - for (int i = 0; i < addressStrings.size(); ++i) { - sb.append(addressStrings.get(i)); - if (i != addressStrings.size() - 1) { - sb.append(","); - } - } - sb.append("]"); - return sb.toString(); - } - - private String masterAddressToString( - InetSocketAddress inetSocketAddress, - Collection masterAddresses) { - // If multiple masters share the same host name, we will display the host name + port - // otherwise just the host name. - if (inetSocketAddress.getHostName().equals("localhost") || masterAddresses.stream() - .filter(it -> it.getHostName().equals(inetSocketAddress.getHostName())).count() > 1) { - return inetSocketAddress.toString(); - } - return inetSocketAddress.getHostName(); - } - - private void printWorkerAllMasterConnectionInfo( - AllMastersWorkerInfo allMastersWorkerInfo) { - List masterAddresses = allMastersWorkerInfo.getMasterAddresses(); - int maxWorkerNameLength = - allMastersWorkerInfo.getWorkerIdAddressMap().values().stream() - .map(w -> w.getHostName().length()) - .max(Comparator.comparing(Integer::intValue)).orElse(0); - - int workerNameIndent = 16; - if (workerNameIndent <= maxWorkerNameLength) { - // extend first indent according to the longest worker name by default 5 - workerNameIndent = maxWorkerNameLength + 5; - } - - // Create indentation to tolerate 2 unregistered masters - int maxMasterNameLength = - allMastersWorkerInfo.getWorkerIdAddressMap().values().stream() - .map(w -> masterAddressToString(w, masterAddresses).length()) - .max(Comparator.comparing(Integer::intValue)).orElse(0); - int unregisteredMasterNameIndent = Math.max(24, maxMasterNameLength * 2 + 10); - String format = "%-" + workerNameIndent - + "s %-" + unregisteredMasterNameIndent + "s %-" + unregisteredMasterNameIndent + "s %s"; - print(""); - print(String.format(format, "Worker Name", "Not Registered With", "Lost", "In Service")); - for (Map.Entry>> workerInfoEntry : - allMastersWorkerInfo.getWorkerIdInfoMap().entrySet()) { - if (workerInfoEntry.getValue().stream() - .noneMatch(it -> it.getSecond().getState().equals("In Service"))) { - // Don't display the worker if it has been removed from all masters. - continue; - } - long workerId = workerInfoEntry.getKey(); - InetSocketAddress workerAddress = allMastersWorkerInfo.getWorkerIdAddressMap() - .get(workerId); - String workerName = workerAddress != null ? workerAddress.getHostName() - : "(UNKNOWN, id = " + workerId + ")"; - Set inServiceMasters = - workerInfoEntry.getValue().stream() - .filter(it -> it.getSecond().getState().equals(LIVE_WORKER_STATE)) - .map(alluxio.collections.Pair::getFirst).collect(Collectors.toSet()); - Set lostMasters = - workerInfoEntry.getValue().stream() - .filter(it -> it.getSecond().getState().equals(LOST_WORKER_STATE)) - .map(alluxio.collections.Pair::getFirst).collect(Collectors.toSet()); - Set allMasterAddresses = - new HashSet<>(allMastersWorkerInfo.getMasterAddresses()); - Set notRegisteredMaster = - com.google.common.collect.Sets.difference(allMasterAddresses, - com.google.common.collect.Sets.union(inServiceMasters, lostMasters)); - print(String.format(format, workerName, getMasterAddressesString(notRegisteredMaster), - getMasterAddressesString(lostMasters), getMasterAddressesString(inServiceMasters))); - } - } - - /** - * Prints worker capacity information. - * - * @param workerInfoList the worker info list to get info from - */ - private void printWorkerInfo(List workerInfoList) { - mIndentationLevel = 0; - if (mCapacityTierInfoMap.size() == 0) { - return; - } else if (mCapacityTierInfoMap.size() == 1) { - // Do not print Total value when only one tier exists - printShortWorkerInfo(workerInfoList); - return; - } - Set tiers = mCapacityTierInfoMap.keySet(); - String tiersInfo = String.format(Strings.repeat("%-14s", tiers.size()), tiers.toArray()); - String longInfoFormat = getInfoFormat(workerInfoList, false); - print(String.format("%n" + longInfoFormat, - "Worker Name", "State", "Last Heartbeat", "Storage", "Total", tiersInfo, - "Version", "Revision")); - - for (WorkerInfo info : workerInfoList) { - String workerName = info.getAddress().getHost(); - - long usedBytes = info.getUsedBytes(); - long capacityBytes = info.getCapacityBytes(); - - String usedPercentageInfo = ""; - if (capacityBytes != 0) { - int usedPercentage = (int) (100L * usedBytes / capacityBytes); - usedPercentageInfo = String.format(" (%s%%)", usedPercentage); - } - - String capacityTierInfo = getWorkerFormattedTierValues(mCapacityTierInfoMap, workerName); - String usedTierInfo = getWorkerFormattedTierValues(mUsedTierInfoMap, workerName); - - print(String.format(longInfoFormat, workerName, info.getState(), - info.getLastContactSec(), "capacity", - FormatUtils.getSizeFromBytes(capacityBytes), capacityTierInfo, - info.getVersion(), info.getRevision())); - print(String.format(longInfoFormat, "", "", "", "used", - FormatUtils.getSizeFromBytes(usedBytes) + usedPercentageInfo, usedTierInfo, - "", "")); - } - } - - /** - * Prints worker information when only one tier exists. - * - * @param workerInfoList the worker info list to get info from - */ - private void printShortWorkerInfo(List workerInfoList) { - String tier = String.format("%-16s", mCapacityTierInfoMap.firstKey()); - String shortInfoFormat = getInfoFormat(workerInfoList, true); - print(String.format("%n" + shortInfoFormat, - "Worker Name", "State", "Last Heartbeat", "Storage", tier, "Version", "Revision")); - - for (WorkerInfo info : workerInfoList) { - long capacityBytes = info.getCapacityBytes(); - long usedBytes = info.getUsedBytes(); - - String usedPercentageInfo = ""; - if (capacityBytes != 0) { - int usedPercentage = (int) (100L * usedBytes / capacityBytes); - usedPercentageInfo = String.format(" (%s%%)", usedPercentage); - } - print(String.format(shortInfoFormat, info.getAddress().getHost(), info.getState(), - info.getLastContactSec(), "capacity", - String.format("%-16s", FormatUtils.getSizeFromBytes(capacityBytes)), - info.getVersion(), info.getRevision())); - print(String.format(shortInfoFormat, "", "", "", "used", - String.format("%-16s", FormatUtils.getSizeFromBytes(usedBytes) + usedPercentageInfo), - "", "")); - } - } - - /** - * Gets the info format according to the longest worker name. - * @param workerInfoList the worker info list to get info from - * @param isShort whether exists only one tier - * @return the info format for printing long/short worker info - */ - private String getInfoFormat(List workerInfoList, boolean isShort) { - int maxWorkerNameLength = workerInfoList.stream().map(w -> w.getAddress().getHost().length()) - .max(Comparator.comparing(Integer::intValue)).get(); - int firstIndent = 16; - if (firstIndent <= maxWorkerNameLength) { - // extend first indent according to the longest worker name by default 5 - firstIndent = maxWorkerNameLength + 5; - } - if (isShort) { - return "%-" + firstIndent + "s %-15s %-16s %-13s %s %-16s %-40s"; - } - return "%-" + firstIndent + "s %-15s %-16s %-13s %-16s %s %-16s %-40s"; - } - - /** - * Gets the worker info options. - * - * @param cl CommandLine that contains the client options - * @return GetWorkerReportOptions to get worker information - */ - private GetWorkerReportOptions getOptions(CommandLine cl) throws IOException { - if (cl.getOptions().length > 1) { - System.out.println(getUsage()); - throw new InvalidArgumentException("Too many arguments passed in."); - } - GetWorkerReportOptions workerOptions = GetWorkerReportOptions.defaults(); - - Set fieldRange = EnumSet.of(WorkerInfoField.ADDRESS, - WorkerInfoField.WORKER_CAPACITY_BYTES, WorkerInfoField.WORKER_CAPACITY_BYTES_ON_TIERS, - WorkerInfoField.LAST_CONTACT_SEC, WorkerInfoField.WORKER_USED_BYTES, - WorkerInfoField.WORKER_USED_BYTES_ON_TIERS, WorkerInfoField.BUILD_VERSION, - WorkerInfoField.ID, WorkerInfoField.STATE); - workerOptions.setFieldRange(fieldRange); - - if (cl.hasOption(ReportCommand.LIVE_OPTION_NAME)) { - workerOptions.setWorkerRange(WorkerRange.LIVE); - } else if (cl.hasOption(ReportCommand.LOST_OPTION_NAME)) { - workerOptions.setWorkerRange(WorkerRange.LOST); - } else if (cl.hasOption(ReportCommand.DECOMMISSIONED_OPTION_NAME)) { - workerOptions.setWorkerRange(WorkerRange.DECOMMISSIONED); - } else if (cl.hasOption(ReportCommand.SPECIFIED_OPTION_NAME)) { - workerOptions.setWorkerRange(WorkerRange.SPECIFIED); - String addressString = cl.getOptionValue(ReportCommand.SPECIFIED_OPTION_NAME); - String[] addressArray = addressString.split(","); - // Addresses in GetWorkerReportOptions is only used when WorkerRange is SPECIFIED - workerOptions.setAddresses(new HashSet<>(Arrays.asList(addressArray))); - } - return workerOptions; - } - - /** - * Gets the formatted tier values of a worker. - * - * @param map the map to get worker tier values from - * @param workerName name of the worker - * @return the formatted tier values of the input worker name - */ - private static String getWorkerFormattedTierValues(Map> map, - String workerName) { - return map.values().stream().map((tierMap) - -> (String.format("%-14s", tierMap.getOrDefault(workerName, "-")))) - .collect(Collectors.joining("")); - } - - /** - * Initializes member variables used to collect worker info. - */ - private void initVariables() { - mSumCapacityBytes = 0; - mSumUsedBytes = 0; - mSumCapacityBytesOnTierMap = new TreeMap<>(FileSystemAdminShellUtils::compareTierNames); - mSumUsedBytesOnTierMap = new TreeMap<>(FileSystemAdminShellUtils::compareTierNames); - - // TierInfoMap is of form Map> - mCapacityTierInfoMap = new TreeMap<>(FileSystemAdminShellUtils::compareTierNames); - mUsedTierInfoMap = new TreeMap<>(FileSystemAdminShellUtils::compareTierNames); - } - - /** - * Prints indented information. - * - * @param text information to print - */ - private void print(String text) { - String indent = Strings.repeat(" ", mIndentationLevel * INDENT_SIZE); - mPrintStream.println(indent + text); - } - - /** - * @return report capacity command usage - */ - public static String getUsage() { - return "alluxio fsadmin report capacity [filter arg]\n" - + "Report Alluxio capacity information.\n" - + "Where [filter arg] is an optional argument. If no arguments passed in, " - + "capacity information of all workers will be printed out.\n" - + "[filter arg] can be one of the following:\n" - + " -live Live workers\n" - + " -lost Lost workers\n" - + " -workers Specified workers, " - + "host names or ip addresses separated by \",\"\n"; - } -} \ No newline at end of file diff --git a/dora/shell/src/test/java/alluxio/cli/fsadmin/report/CapacityCommandTest.java b/dora/shell/src/test/java/alluxio/cli/fsadmin/report/CapacityCommandTest.java deleted file mode 100644 index d4aa63800ffe..000000000000 --- a/dora/shell/src/test/java/alluxio/cli/fsadmin/report/CapacityCommandTest.java +++ /dev/null @@ -1,307 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.cli.fsadmin.report; - -import alluxio.Constants; -import alluxio.client.block.BlockMasterClient; -import alluxio.client.block.options.GetWorkerReportOptions; -import alluxio.conf.Configuration; -import alluxio.master.WorkerState; -import alluxio.wire.WorkerInfo; -import alluxio.wire.WorkerNetAddress; - -import org.hamcrest.collection.IsIterableContainingInOrder; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; -import org.mockito.Mockito; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.PrintStream; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class CapacityCommandTest { - private BlockMasterClient mBlockMasterClient; - - @Before - public void prepareDependencies() throws IOException { - // Prepare mock block master client - mBlockMasterClient = Mockito.mock(BlockMasterClient.class); - } - - @Test - public void longCapacity() throws IOException { - List longInfoList = prepareLongInfoList(); - Mockito.when(mBlockMasterClient.getWorkerReport(Mockito.any())) - .thenReturn(longInfoList); - - try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - PrintStream printStream = new PrintStream(outputStream, true, "utf-8")) { - CapacityCommand capacityCommand = new CapacityCommand(mBlockMasterClient, - printStream); - capacityCommand.generateCapacityReport(GetWorkerReportOptions.defaults(), - Configuration.global()); - String output = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); - // CHECKSTYLE.OFF: LineLengthExceed - Much more readable - List expectedOutput = Arrays.asList("Capacity information for all workers: ", - " Total Capacity: 29.80GB", - " Tier: MEM Size: 8.38GB", - " Tier: SSD Size: 4768.37MB", - " Tier: HDD Size: 1907.35MB", - " Tier: DOM Size: 9.31GB", - " Tier: RAM Size: 5.59GB", - " Used Capacity: 10.24GB", - " Tier: MEM Size: 3051.76MB", - " Tier: SSD Size: 286.10MB", - " Tier: HDD Size: 1907.35MB", - " Tier: DOM Size: 476.84MB", - " Tier: RAM Size: 4768.37MB", - " Used Percentage: 34%", - " Free Percentage: 66%", - "", - "Worker Name State Last Heartbeat Storage Total MEM SSD HDD DOM RAM Version Revision ", - "216.239.33.96 ACTIVE 542 capacity 18.63GB 4768.37MB 4768.37MB - 9.31GB - 2.10.0-SNAPSHOT 0123456789abcdef0123456789abcdef01234567", - " used 953.67MB (5%) 190.73MB 286.10MB - 476.84MB - ", - "64.68.90.1 ACTIVE 3123 capacity 11.18GB 3814.70MB - 1907.35MB - 5.59GB 2.9.3 0123456789012345678901234567890123456789", - " used 9.31GB (83%) 2861.02MB - 1907.35MB - 4768.37MB "); - // CHECKSTYLE.ON: LineLengthExceed - List testOutput = Arrays.asList(output.split("\n")); - Assert.assertThat(testOutput, - IsIterableContainingInOrder.contains(expectedOutput.toArray())); - } - } - - @Test - public void shortCapacity() throws IOException { - List shortInfoList = prepareShortInfoList(); - Mockito.when(mBlockMasterClient.getWorkerReport(Mockito.any())) - .thenReturn(shortInfoList); - - try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - PrintStream printStream = new PrintStream(outputStream, true, "utf-8")) { - CapacityCommand capacityCommand = new CapacityCommand(mBlockMasterClient, - printStream); - capacityCommand.generateCapacityReport(GetWorkerReportOptions.defaults(), - Configuration.global()); - String output = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); - // CHECKSTYLE.OFF: LineLengthExceed - Much more readable - List expectedOutput = Arrays.asList("Capacity information for all workers: ", - " Total Capacity: 14.90GB", - " Tier: RAM Size: 14.90GB", - " Used Capacity: 5.12GB", - " Tier: RAM Size: 5.12GB", - " Used Percentage: 34%", - " Free Percentage: 66%", - "", - "Worker Name State Last Heartbeat Storage RAM Version Revision ", - "215.42.95.24 ACTIVE 953 capacity 9.31GB 2.2.4 000111222333444555666777888999aaabbbcccd", - " used 476.84MB (5%) ", - "29.53.5.124 LOST 6424122 capacity 5.59GB 2.2.3 00112233445566778899aabbccddeeff00112233", - " used 4768.37MB (83%) "); - List testOutput = Arrays.asList(output.split("\n")); - Assert.assertThat(testOutput, - IsIterableContainingInOrder.contains(expectedOutput.toArray())); - } - } - - @Test - public void longWorkerNameCapacity() throws IOException { - List longWorkerNameInfoList = prepareLongWorkerNameInfoList(); - Mockito.when(mBlockMasterClient.getWorkerReport(Mockito.any())) - .thenReturn(longWorkerNameInfoList); - - try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - PrintStream printStream = new PrintStream(outputStream, true, "utf-8")) { - CapacityCommand capacityCommand = new CapacityCommand(mBlockMasterClient, - printStream); - capacityCommand.generateCapacityReport(GetWorkerReportOptions.defaults(), - Configuration.global()); - String output = new String(outputStream.toByteArray(), StandardCharsets.UTF_8); - List testRst = Arrays.asList(output.split("\n")); - // CHECKSTYLE.OFF: LineLengthExceed - Much more readable - List expectedOutput = Arrays.asList("Capacity information for all workers: ", - " Total Capacity: 3051.76MB", - " Tier: MEM Size: 1144.41MB", - " Tier: SSD Size: 572.20MB", - " Tier: HDD Size: 190.73MB", - " Used Capacity: 1049.04MB", - " Tier: MEM Size: 305.18MB", - " Tier: SSD Size: 28.61MB", - " Tier: HDD Size: 190.73MB", - " Used Percentage: 34%", - " Free Percentage: 66%", - "", - "Worker Name State Last Heartbeat Storage Total MEM SSD HDD Version Revision ", - "org.apache.hdp1 ACTIVE 681 capacity 1907.35MB 572.20MB 572.20MB - 2.10.0-rc1 abababababababababababababababababababab", - " used 95.37MB (5%) 19.07MB 28.61MB - ", - "org.alluxio.long.host1 ACTIVE 6211 capacity 1144.41MB 572.20MB - 190.73MB 2.10.0-rc2 0101010101010101010101010101010101010101", - " used 953.67MB (83%) 286.10MB - 190.73MB "); - // CHECKSTYLE.ON: LineLengthExceed - List testOutput = Arrays.asList(output.split("\n")); - - Assert.assertThat(testOutput, - IsIterableContainingInOrder.contains(expectedOutput.toArray())); - } - } - - /** - * @return long worker info list to test - */ - private List prepareLongInfoList() { - List infoList = new ArrayList<>(); - Map capacityBytesOnTiersOne = new HashMap<>(); - capacityBytesOnTiersOne.put(Constants.MEDIUM_MEM, 4000000000L); - capacityBytesOnTiersOne.put("RAM", 6000000000L); - capacityBytesOnTiersOne.put(Constants.MEDIUM_HDD, 2000000000L); - Map usedBytesOnTiersOne = new HashMap<>(); - usedBytesOnTiersOne.put(Constants.MEDIUM_MEM, 3000000000L); - usedBytesOnTiersOne.put("RAM", 5000000000L); - usedBytesOnTiersOne.put(Constants.MEDIUM_HDD, 2000000000L); - WorkerInfo firstInfo = new WorkerInfo() - .setAddress(new WorkerNetAddress().setHost("64.68.90.1")) - .setCapacityBytes(12000000000L) - .setCapacityBytesOnTiers(capacityBytesOnTiersOne) - .setId(1) - .setLastContactSec(3123) - .setStartTimeMs(1331231121212L) - .setState(WorkerState.LIVE.toString()) - .setUsedBytes(10000000000L) - .setUsedBytesOnTiers(usedBytesOnTiersOne) - .setVersion("2.9.3") - .setRevision("0123456789012345678901234567890123456789"); - - Map capacityBytesOnTiersSec = new HashMap<>(); - capacityBytesOnTiersSec.put(Constants.MEDIUM_MEM, 5000000000L); - capacityBytesOnTiersSec.put(Constants.MEDIUM_SSD, 5000000000L); - capacityBytesOnTiersSec.put("DOM", 10000000000L); - Map usedBytesOnTiersSec = new HashMap<>(); - usedBytesOnTiersSec.put(Constants.MEDIUM_MEM, 200000000L); - usedBytesOnTiersSec.put(Constants.MEDIUM_SSD, 300000000L); - usedBytesOnTiersSec.put("DOM", 500000000L); - WorkerInfo secondInfo = new WorkerInfo() - .setAddress(new WorkerNetAddress().setHost("216.239.33.96")) - .setCapacityBytes(20000000000L) - .setCapacityBytesOnTiers(capacityBytesOnTiersSec) - .setId(2) - .setLastContactSec(542) - .setStartTimeMs(1131231121212L) - .setState(WorkerState.LIVE.toString()) - .setUsedBytes(1000000000L) - .setUsedBytesOnTiers(usedBytesOnTiersSec) - .setVersion("2.10.0-SNAPSHOT") - .setRevision("0123456789abcdef0123456789abcdef01234567"); - - infoList.add(firstInfo); - infoList.add(secondInfo); - return infoList; - } - - /** - * @return short worker info list that only one tier exists - */ - private List prepareShortInfoList() { - List infoList = new ArrayList<>(); - Map capacityBytesOnTiersOne = new HashMap<>(); - capacityBytesOnTiersOne.put("RAM", 6000000000L); - Map usedBytesOnTiersOne = new HashMap<>(); - usedBytesOnTiersOne.put("RAM", 5000000000L); - WorkerInfo firstInfo = new WorkerInfo() - .setAddress(new WorkerNetAddress().setHost("29.53.5.124")) - .setCapacityBytes(6000000000L) - .setCapacityBytesOnTiers(capacityBytesOnTiersOne) - .setId(1) - .setLastContactSec(6424122) - .setStartTimeMs(19365332L) - .setState(WorkerState.LOST.toString()) - .setUsedBytes(5000000000L) - .setUsedBytesOnTiers(usedBytesOnTiersOne) - .setVersion("2.2.3") - .setRevision("00112233445566778899aabbccddeeff00112233"); - - Map capacityBytesOnTiersSec = new HashMap<>(); - capacityBytesOnTiersSec.put("RAM", 10000000000L); - Map usedBytesOnTiersSec = new HashMap<>(); - usedBytesOnTiersSec.put("RAM", 500000000L); - WorkerInfo secondInfo = new WorkerInfo() - .setAddress(new WorkerNetAddress().setHost("215.42.95.24")) - .setCapacityBytes(10000000000L) - .setCapacityBytesOnTiers(capacityBytesOnTiersSec) - .setId(2) - .setLastContactSec(953) - .setStartTimeMs(112495222L) - .setState(WorkerState.LIVE.toString()) - .setUsedBytes(500000000L) - .setUsedBytesOnTiers(usedBytesOnTiersSec) - .setVersion("2.2.4") - .setRevision("000111222333444555666777888999aaabbbcccd"); - - infoList.add(firstInfo); - infoList.add(secondInfo); - return infoList; - } - - /** - * @return long worker name info list to test - */ - private List prepareLongWorkerNameInfoList() { - List infoList = new ArrayList<>(); - Map capacityBytesOnTiersOne = new HashMap<>(); - capacityBytesOnTiersOne.put(Constants.MEDIUM_MEM, 600000000L); - capacityBytesOnTiersOne.put(Constants.MEDIUM_HDD, 200000000L); - Map usedBytesOnTiersOne = new HashMap<>(); - usedBytesOnTiersOne.put(Constants.MEDIUM_MEM, 300000000L); - usedBytesOnTiersOne.put(Constants.MEDIUM_HDD, 200000000L); - WorkerInfo firstInfo = new WorkerInfo() - .setAddress(new WorkerNetAddress().setHost("org.alluxio.long.host1")) - .setCapacityBytes(1200000000L) - .setCapacityBytesOnTiers(capacityBytesOnTiersOne) - .setId(1) - .setLastContactSec(6211) - .setStartTimeMs(1529222699127L) - .setState(WorkerState.LIVE.toString()) - .setUsedBytes(1000000000L) - .setUsedBytesOnTiers(usedBytesOnTiersOne) - .setVersion("2.10.0-rc2") - .setRevision("0101010101010101010101010101010101010101"); - - Map capacityBytesOnTiersSec = new HashMap<>(); - capacityBytesOnTiersSec.put(Constants.MEDIUM_MEM, 600000000L); - capacityBytesOnTiersSec.put(Constants.MEDIUM_SSD, 600000000L); - Map usedBytesOnTiersSec = new HashMap<>(); - usedBytesOnTiersSec.put(Constants.MEDIUM_MEM, 20000000L); - usedBytesOnTiersSec.put(Constants.MEDIUM_SSD, 30000000L); - WorkerInfo secondInfo = new WorkerInfo() - .setAddress(new WorkerNetAddress().setHost("org.apache.hdp1")) - .setCapacityBytes(2000000000L) - .setCapacityBytesOnTiers(capacityBytesOnTiersSec) - .setId(2) - .setLastContactSec(681) - .setStartTimeMs(1529222699127L) - .setState(WorkerState.LIVE.toString()) - .setUsedBytes(100000000L) - .setUsedBytesOnTiers(usedBytesOnTiersSec) - .setVersion("2.10.0-rc1") - .setRevision("abababababababababababababababababababab"); - - infoList.add(firstInfo); - infoList.add(secondInfo); - return infoList; - } - - // TODO(elega) Add unit tests for the case where worker all master registration is enabled -} diff --git a/dora/tests/integration/src/test/java/alluxio/client/cli/fsadmin/command/CapacityCommandIntegrationTest.java b/dora/tests/integration/src/test/java/alluxio/client/cli/fsadmin/command/CapacityCommandIntegrationTest.java deleted file mode 100644 index 410b717c1773..000000000000 --- a/dora/tests/integration/src/test/java/alluxio/client/cli/fsadmin/command/CapacityCommandIntegrationTest.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 - * (the "License"). You may not use this work except in compliance with the License, which is - * available at www.apache.org/licenses/LICENSE-2.0 - * - * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied, as more fully set forth in the License. - * - * See the NOTICE file distributed with this work for information regarding copyright ownership. - */ - -package alluxio.client.cli.fsadmin.command; - -import alluxio.annotation.dora.DoraTestTodoItem; -import alluxio.cli.fsadmin.report.CapacityCommand; -import alluxio.client.cli.fsadmin.AbstractFsAdminShellTest; -import alluxio.util.FormatUtils; - -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Test; - -/** - * Tests for report capacity command. - */ -@Ignore -@DoraTestTodoItem(action = DoraTestTodoItem.Action.FIX, owner = "jiacheng", - comment = "rework on the register logic") -public final class CapacityCommandIntegrationTest extends AbstractFsAdminShellTest { - @Test - public void allCapacity() { - int ret = mFsAdminShell.run("report", "capacity"); - Assert.assertEquals(0, ret); - String output = mOutput.toString(); - String size = FormatUtils.getSizeFromBytes(SIZE_BYTES); - String[] lines = output.split("\n"); - Assert.assertEquals(11, lines.length); - Assert.assertEquals("Capacity information for all workers: ", lines[0]); - Assert.assertEquals(" Total Capacity: " + size, lines[1]); - Assert.assertEquals(" Tier: MEM Size: " + size, lines[2]); - Assert.assertEquals(" Used Capacity: 0B", lines[3]); - Assert.assertEquals(" Tier: MEM Size: 0B", lines[4]); - Assert.assertEquals(" Used Percentage: 0%", lines[5]); - Assert.assertEquals(" Free Percentage: 100%", lines[6]); - Assert.assertEquals("", lines[7]); - Assert.assertTrue(lines[8].matches( - "Worker Name {6,}State {11,}Last Heartbeat {3}Storage {7}MEM {14}Version {10}Revision *")); - Assert.assertTrue(lines[9].contains("ACTIVE")); - Assert.assertTrue(lines[9].contains("capacity " + size)); - Assert.assertTrue(lines[10].contains("used 0B (0%)")); - } - - @Test - public void lostCapacity() { - int ret = mFsAdminShell.run("report", "capacity", "-lost"); - Assert.assertEquals(0, ret); - Assert.assertEquals(mOutput.toString(), "No workers found.\n"); - } - - @Test - public void liveCapacity() { - int ret = mFsAdminShell.run("report", "capacity", "-live"); - Assert.assertEquals(0, ret); - String output = mOutput.toString(); - String size = FormatUtils.getSizeFromBytes(SIZE_BYTES); - String[] lines = output.split("\n"); - Assert.assertEquals(11, lines.length); - Assert.assertEquals("Capacity information for live workers: ", lines[0]); - Assert.assertEquals(" Total Capacity: " + size, lines[1]); - Assert.assertEquals(" Tier: MEM Size: " + size, lines[2]); - Assert.assertEquals(" Used Capacity: 0B", lines[3]); - Assert.assertEquals(" Tier: MEM Size: 0B", lines[4]); - Assert.assertEquals(" Used Percentage: 0%", lines[5]); - Assert.assertEquals(" Free Percentage: 100%", lines[6]); - Assert.assertEquals("", lines[7]); - Assert.assertTrue(lines[8].matches( - "Worker Name {6,}State {11,}Last Heartbeat {3}Storage {7}MEM {14}Version {10}Revision *")); - Assert.assertTrue(lines[9].contains("ACTIVE")); - Assert.assertTrue(lines[9].contains("capacity " + size)); - Assert.assertTrue(lines[10].contains("used 0B (0%)")); - } - - @Test - public void tooManyOptions() { - mFsAdminShell.run("report", "capacity", "-live", "-lost"); - String expected = CapacityCommand.getUsage() - + "\nToo many arguments passed in.\n"; - Assert.assertEquals(expected, mOutput.toString()); - } - - // TODO(elega) Add unit tests for the case where worker all master registration is enabled -} From d33280626eca2ded7e8d592f37913ba6d5d53058 Mon Sep 17 00:00:00 2001 From: twalluxio Date: Tue, 29 Aug 2023 15:34:48 +0800 Subject: [PATCH 13/21] add an empty line after SummaryOutput.java --- .../src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java index 5344c9b37ea5..8edaf8c507ab 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java @@ -255,4 +255,4 @@ public Map getUsedCapacityOnTiers() { public void setUsedCapacityOnTiers(Map usedCapacityOnTiers) { mUsedCapacityOnTiers = usedCapacityOnTiers; } -} \ No newline at end of file +} From 4b69c7e9ef87eba10981d3e2abcbb1992cb02e09 Mon Sep 17 00:00:00 2001 From: twalluxio Date: Mon, 4 Sep 2023 08:39:21 +0800 Subject: [PATCH 14/21] change logger to org.slf4j --- .../cli/fsadmin/report/JobServiceMetricsCommand.java | 6 ++++-- .../alluxio/cli/fsadmin/report/MetricsCommand.java | 10 ++++------ .../alluxio/cli/fsadmin/report/SummaryCommand.java | 6 ++++-- .../java/alluxio/cli/fsadmin/report/UfsCommand.java | 5 ++++- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java index 313cf75635f4..2054a6033c0c 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java @@ -15,7 +15,8 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import org.jline.utils.Log; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.PrintStream; @@ -32,6 +33,7 @@ public class JobServiceMetricsCommand { private final JobMasterClient mJobMasterClient; private final PrintStream mPrintStream; private final String mDateFormatPattern; + private static final Logger LOG = LoggerFactory.getLogger(JobServiceMetricsCommand.class); /** * Creates a new instance of {@link JobServiceMetricsCommand}. @@ -65,7 +67,7 @@ public int run() throws IOException { } catch (JsonProcessingException e) { mPrintStream.println("Failed to convert jobServiceInfo output to JSON. " + "Check the command line log for the detailed error message."); - Log.error("Failed to output JSON object {}", jobServiceInfo); + LOG.error("Failed to output JSON object {}", jobServiceInfo); e.printStackTrace(); return -1; } diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java index 0ae8c4dd92e3..d47fcef3cd06 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java @@ -12,17 +12,14 @@ package alluxio.cli.fsadmin.report; import alluxio.client.metrics.MetricsMasterClient; -import alluxio.grpc.MetricValue; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import org.jline.utils.Log; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.PrintStream; -import java.text.DecimalFormat; -import java.text.DecimalFormatSymbols; -import java.util.*; /** * Prints Alluxio metrics information. @@ -30,6 +27,7 @@ public class MetricsCommand { private final MetricsMasterClient mMetricsMasterClient; private final PrintStream mPrintStream; + private static final Logger LOG = LoggerFactory.getLogger(JobServiceMetricsCommand.class); /** * Creates a new instance of {@link MetricsCommand}. @@ -56,7 +54,7 @@ public int run() throws IOException { } catch (JsonProcessingException e) { mPrintStream.println("Failed to convert metricsInfo output to JSON. " + "Check the command line log for the detailed error message."); - Log.error("Failed to output JSON object {}", metricsInfo); + LOG.error("Failed to output JSON object {}", metricsInfo); e.printStackTrace(); return -1; } diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java index 88c7248381a1..3bce5f39deac 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java @@ -20,7 +20,8 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import org.jline.utils.Log; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.io.PrintStream; @@ -36,6 +37,7 @@ public class SummaryCommand { private final BlockMasterClient mBlockMasterClient; private final PrintStream mPrintStream; private final String mDateFormatPattern; + private static final Logger LOG = LoggerFactory.getLogger(JobServiceMetricsCommand.class); /** * Creates a new instance of {@link SummaryCommand}. @@ -83,7 +85,7 @@ public int run() throws IOException { } catch (JsonProcessingException e) { mPrintStream.println("Failed to convert summaryInfo output to JSON. " + "Check the command line log for the detailed error message."); - Log.error("Failed to output JSON object {}", summaryInfo); + LOG.error("Failed to output JSON object {}", summaryInfo); e.printStackTrace(); return -1; } diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java index 25821a2b7b30..7d1d66dacd33 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java @@ -16,6 +16,8 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import org.jline.utils.Log; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.IOException; import java.util.Map; @@ -25,6 +27,7 @@ */ public class UfsCommand { private FileSystemMasterClient mFileSystemMasterClient; + private static final Logger LOG = LoggerFactory.getLogger(JobServiceMetricsCommand.class); /** * Creates a new instance of {@link UfsCommand}. @@ -49,7 +52,7 @@ public int run() throws IOException { } catch (JsonProcessingException e) { System.out.println("Failed to convert mountTable output to JSON. " + "Check the command line log for the detailed error message."); - Log.error("Failed to output JSON object {}", mountTable); + LOG.error("Failed to output JSON object {}", mountTable); e.printStackTrace(); return -1; } From f9f74adb7ec774e47af05c29f0918d9cc2b8c0a6 Mon Sep 17 00:00:00 2001 From: twalluxio Date: Mon, 4 Sep 2023 15:29:59 +0800 Subject: [PATCH 15/21] pass style check & unit tests --- .../java/alluxio/wire/FileSizeSerializer.java | 18 - .../java/alluxio/wire/MountPointInfo.java | 3 - .../report/JobServiceMetricsCommand.java | 4 +- .../cli/fsadmin/report/JobServiceOutput.java | 584 +++++++++------- .../cli/fsadmin/report/MetricsCommand.java | 4 +- .../cli/fsadmin/report/MetricsOutput.java | 174 ++--- .../cli/fsadmin/report/SummaryCommand.java | 4 +- .../cli/fsadmin/report/SummaryOutput.java | 631 +++++++++++------- .../cli/fsadmin/report/UfsCommand.java | 6 +- 9 files changed, 839 insertions(+), 589 deletions(-) delete mode 100644 dora/core/common/src/main/java/alluxio/wire/FileSizeSerializer.java diff --git a/dora/core/common/src/main/java/alluxio/wire/FileSizeSerializer.java b/dora/core/common/src/main/java/alluxio/wire/FileSizeSerializer.java deleted file mode 100644 index b780ea691899..000000000000 --- a/dora/core/common/src/main/java/alluxio/wire/FileSizeSerializer.java +++ /dev/null @@ -1,18 +0,0 @@ -package alluxio.wire; - -import alluxio.util.FormatUtils; -import com.amazonaws.util.StringUtils; -import com.fasterxml.jackson.core.JsonGenerator; -import com.fasterxml.jackson.databind.JsonSerializer; -import com.fasterxml.jackson.databind.SerializerProvider; - -import java.io.IOException; - -public class FileSizeSerializer extends JsonSerializer { - @Override - public void serialize(Long value, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) - throws IOException { - String formattedSize = FormatUtils.getSizeFromBytes(value); - jsonGenerator.writeString(StringUtils.upperCase(formattedSize)); - } -} diff --git a/dora/core/common/src/main/java/alluxio/wire/MountPointInfo.java b/dora/core/common/src/main/java/alluxio/wire/MountPointInfo.java index dcd1fb41bf62..ebd859d5dda6 100644 --- a/dora/core/common/src/main/java/alluxio/wire/MountPointInfo.java +++ b/dora/core/common/src/main/java/alluxio/wire/MountPointInfo.java @@ -15,7 +15,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.google.common.base.MoreObjects; import com.google.common.base.Objects; @@ -39,10 +38,8 @@ public class MountPointInfo implements Serializable { private String mUfsType = ""; @JsonIgnore private long mMountId = 0; - @JsonSerialize(using = FileSizeSerializer.class) @JsonProperty("ufsCapacityBytes") private long mUfsCapacityBytes = UNKNOWN_CAPACITY_BYTES; - @JsonSerialize(using = FileSizeSerializer.class) @JsonProperty("ufsUsedBytes") private long mUfsUsedBytes = UNKNOWN_USED_BYTES; private boolean mReadOnly; diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java index 2054a6033c0c..a817332143bb 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java @@ -65,8 +65,8 @@ public int run() throws IOException { String json = objectMapper.writeValueAsString(jobServiceInfo); mPrintStream.println(json); } catch (JsonProcessingException e) { - mPrintStream.println("Failed to convert jobServiceInfo output to JSON. " + - "Check the command line log for the detailed error message."); + mPrintStream.println("Failed to convert jobServiceInfo output to JSON. " + + "Check the command line log for the detailed error message."); LOG.error("Failed to output JSON object {}", jobServiceInfo); e.printStackTrace(); return -1; diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceOutput.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceOutput.java index d7f552b28567..538a000a8748 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceOutput.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceOutput.java @@ -12,303 +12,373 @@ import java.util.ArrayList; import java.util.List; +/** + * An output class, describing the job service. + */ public class JobServiceOutput { - private List mMasterStatus; - private List mWorkerHealth; - private List mStatusSummary; - private List mRecentModifiedJobs; - private List mRecentFailedJobs; - private List mLongestRunningJobs; - - private static class SerializableJobMasterStatus { - private String mHost; - private int mPort; - private String mState; - private String mStartTime; - private String mVersion; - private String mRevision; - - public SerializableJobMasterStatus(JobMasterStatus jobMasterStatus, String dateFormat) { - mHost = jobMasterStatus.getMasterAddress().getHost(); - mPort = jobMasterStatus.getMasterAddress().getRpcPort(); - mState = jobMasterStatus.getState(); - mStartTime = CommonUtils.convertMsToDate(jobMasterStatus.getStartTime(), dateFormat); - mVersion = jobMasterStatus.getVersion().getVersion(); - mRevision = jobMasterStatus.getVersion().getRevision(); - } - - public String getHost() { - return mHost; - } - - public void setHost(String host) { - mHost = host; - } - - public int getPort() { - return mPort; - } - - public void setPort(int port) { - mPort = port; - } - - public String getState() { - return mState; - } - - public void setState(String state) { - mState = state; - } - - public String getStartTime() { - return mStartTime; - } - - public void setStartTime(String startTime) { - mStartTime = startTime; - } - - public String getVersion() { - return mVersion; - } - - public void setVersion(String version) { - mVersion = version; - } - - public String getRevision() { - return mRevision; - } - - public void setRevision(String revision) { - mRevision = revision; - } - } - - private static class SerializableWorkerHealth { - private String mHost; - private String mVersion; - private String mRevision; - private int mTaskPoolSize; - private int mUnfinishedTasks; - private int mActiveTasks; - private List mLoadAverage; - - public SerializableWorkerHealth(JobWorkerHealth workerHealth) { - mHost = workerHealth.getHostname(); - mVersion = workerHealth.getVersion().getVersion(); - mRevision = workerHealth.getVersion().getRevision(); - mTaskPoolSize = workerHealth.getTaskPoolSize(); - mUnfinishedTasks = workerHealth.getUnfinishedTasks(); - mActiveTasks = workerHealth.getNumActiveTasks(); - mLoadAverage = workerHealth.getLoadAverage(); - } - public String getHost() { - return mHost; - } - - public void setHost(String host) { - mHost = host; - } - - public String getVersion() { - return mVersion; - } - - public void setVersion(String version) { - mVersion = version; - } - - public String getRevision() { - return mRevision; - } - - public void setRevision(String revision) { - mRevision = revision; - } - - public int getTaskPoolSize() { - return mTaskPoolSize; - } - - public void setTaskPoolSize(int taskPoolSize) { - mTaskPoolSize = taskPoolSize; - } - - public int getUnfinishedTasks() { - return mUnfinishedTasks; - } - - public void setUnfinishedTasks(int unfinishedTasks) { - mUnfinishedTasks = unfinishedTasks; - } - - public int getActiveTasks() { - return mActiveTasks; - } - - public void setActiveTasks(int activeTasks) { - mActiveTasks = activeTasks; - } - - public List getLoadAverage() { - return mLoadAverage; - } - - public void setLoadAverage(List loadAverage) { - mLoadAverage = loadAverage; - } - - } - - private static class SerializableStatusSummary { - private String mStatus; - private long mCount; - - public SerializableStatusSummary(StatusSummary statusSummary){ - mStatus = statusSummary.getStatus().toString(); - mCount = statusSummary.getCount(); - } - - public String getStatus() { - return mStatus; - } - - public void setStatus(String status) { - mStatus = status; - } - - public long getCount(){ - return mCount; - } + private List mMasterStatus; + private List mWorkerHealth; + private List mStatusSummary; + private List mRecentModifiedJobs; + private List mRecentFailedJobs; + private List mLongestRunningJobs; + + private static class SerializableJobMasterStatus { + private String mHost; + private int mPort; + private String mState; + private String mStartTime; + private String mVersion; + private String mRevision; + + public SerializableJobMasterStatus(JobMasterStatus jobMasterStatus, String dateFormat) { + mHost = jobMasterStatus.getMasterAddress().getHost(); + mPort = jobMasterStatus.getMasterAddress().getRpcPort(); + mState = jobMasterStatus.getState(); + mStartTime = CommonUtils.convertMsToDate(jobMasterStatus.getStartTime(), dateFormat); + mVersion = jobMasterStatus.getVersion().getVersion(); + mRevision = jobMasterStatus.getVersion().getRevision(); + } + + public String getHost() { + return mHost; + } + + public void setHost(String host) { + mHost = host; + } + + public int getPort() { + return mPort; + } + + public void setPort(int port) { + mPort = port; + } + + public String getState() { + return mState; + } + + public void setState(String state) { + mState = state; + } + + public String getStartTime() { + return mStartTime; + } + + public void setStartTime(String startTime) { + mStartTime = startTime; + } + + public String getVersion() { + return mVersion; + } + + public void setVersion(String version) { + mVersion = version; + } + + public String getRevision() { + return mRevision; + } + + public void setRevision(String revision) { + mRevision = revision; + } + } + + private static class SerializableWorkerHealth { + private String mHost; + private String mVersion; + private String mRevision; + private int mTaskPoolSize; + private int mUnfinishedTasks; + private int mActiveTasks; + private List mLoadAverage; + + public SerializableWorkerHealth(JobWorkerHealth workerHealth) { + mHost = workerHealth.getHostname(); + mVersion = workerHealth.getVersion().getVersion(); + mRevision = workerHealth.getVersion().getRevision(); + mTaskPoolSize = workerHealth.getTaskPoolSize(); + mUnfinishedTasks = workerHealth.getUnfinishedTasks(); + mActiveTasks = workerHealth.getNumActiveTasks(); + mLoadAverage = workerHealth.getLoadAverage(); + } - public void setCount(long count){ - mCount = count; - } + public String getHost() { + return mHost; } - private static class SerializableJobInfo { - private String mTimestamp; - private long mId; - private String mName; - private String mStatus; - public SerializableJobInfo(JobInfo jobInfo, String dateFormat) { - mTimestamp = CommonUtils.convertMsToDate(jobInfo.getLastUpdated(), dateFormat); - mId = jobInfo.getId(); - mName = jobInfo.getName(); - mStatus = jobInfo.getStatus().toString(); - } + public void setHost(String host) { + mHost = host; + } - public String getTimestamp() { - return mTimestamp; - } + public String getVersion() { + return mVersion; + } - public void setTimestamp(String timestamp) { - mTimestamp = timestamp; - } + public void setVersion(String version) { + mVersion = version; + } - public long getId() { - return mId; - } + public String getRevision() { + return mRevision; + } - public void setId(long id) { - mId = id; - } + public void setRevision(String revision) { + mRevision = revision; + } - public String getName() { - return mName; - } + public int getTaskPoolSize() { + return mTaskPoolSize; + } - public void setName(String name) { - mName = name; - } + public void setTaskPoolSize(int taskPoolSize) { + mTaskPoolSize = taskPoolSize; + } - public String getStatus() { - return mStatus; - } + public int getUnfinishedTasks() { + return mUnfinishedTasks; + } - public void setStatus(String status) { - mStatus = status; - } + public void setUnfinishedTasks(int unfinishedTasks) { + mUnfinishedTasks = unfinishedTasks; } - public JobServiceOutput(JobMasterClient jobMasterClient, String dateFormat) throws IOException { - mMasterStatus = new ArrayList<>(); - for (JobMasterStatus masterStatus : jobMasterClient.getAllMasterStatus()) { - mMasterStatus.add(new SerializableJobMasterStatus(masterStatus, dateFormat)); - } + public int getActiveTasks() { + return mActiveTasks; + } - mWorkerHealth = new ArrayList<>(); - for (JobWorkerHealth workerHealth : jobMasterClient.getAllWorkerHealth()) { - mWorkerHealth.add(new SerializableWorkerHealth(workerHealth)); - } + public void setActiveTasks(int activeTasks) { + mActiveTasks = activeTasks; + } - JobServiceSummary jobServiceSummary = jobMasterClient.getJobServiceSummary(); - mStatusSummary = new ArrayList<>(); - for (StatusSummary statusSummary : jobServiceSummary.getSummaryPerStatus()) { - mStatusSummary.add(new SerializableStatusSummary(statusSummary)); - } + public List getLoadAverage() { + return mLoadAverage; + } - mRecentModifiedJobs = new ArrayList<>(); - mRecentFailedJobs = new ArrayList<>(); - mLongestRunningJobs = new ArrayList<>(); - for (JobInfo jobInfo : jobServiceSummary.getRecentActivities()){ - mRecentModifiedJobs.add(new SerializableJobInfo(jobInfo, dateFormat)); - } - for (JobInfo jobInfo : jobServiceSummary.getRecentFailures()){ - mRecentFailedJobs.add(new SerializableJobInfo(jobInfo, dateFormat)); - } - for (JobInfo jobInfo : jobServiceSummary.getLongestRunning()) { - mLongestRunningJobs.add(new SerializableJobInfo(jobInfo, dateFormat)); - } + public void setLoadAverage(List loadAverage) { + mLoadAverage = loadAverage; } + } + + private static class SerializableStatusSummary { + private String mStatus; + private long mCount; - public List getMasterStatus() { - return mMasterStatus; + public SerializableStatusSummary(StatusSummary statusSummary) { + mStatus = statusSummary.getStatus().toString(); + mCount = statusSummary.getCount(); } - public void setMasterStatus(List masterStatus) { - mMasterStatus = masterStatus; + public String getStatus() { + return mStatus; } - public List getWorkerHealth() { - return mWorkerHealth; + public void setStatus(String status) { + mStatus = status; } - public void setWorkerHealth(List workerHealth) { - mWorkerHealth = workerHealth; + public long getCount() { + return mCount; } - public List getStatusSummary() { - return mStatusSummary; + public void setCount(long count) { + mCount = count; + } + } + + private static class SerializableJobInfo { + private String mTimestamp; + private long mId; + private String mName; + private String mStatus; + + public SerializableJobInfo(JobInfo jobInfo, String dateFormat) { + mTimestamp = CommonUtils.convertMsToDate(jobInfo.getLastUpdated(), dateFormat); + mId = jobInfo.getId(); + mName = jobInfo.getName(); + mStatus = jobInfo.getStatus().toString(); + } + + public String getTimestamp() { + return mTimestamp; + } + + public void setTimestamp(String timestamp) { + mTimestamp = timestamp; + } + + public long getId() { + return mId; + } + + public void setId(long id) { + mId = id; } - public void setStatusSummary(List statusSummary) { - mStatusSummary = statusSummary; + public String getName() { + return mName; } - public List getRecentModifiedJobs() { - return mRecentModifiedJobs; + public void setName(String name) { + mName = name; } - public void setRecentModifiedJobs(List recentModifiedJobs) { - mRecentModifiedJobs = recentModifiedJobs; + public String getStatus() { + return mStatus; } - public List getRecentFailedJobs() { - return mRecentFailedJobs; + public void setStatus(String status) { + mStatus = status; + } + } + + /** + * Creates a new instance of {@link JobServiceOutput}. + * + * @param jobMasterClient job service info to parse from + * @param dateFormat specify the pattern of dates + */ + public JobServiceOutput(JobMasterClient jobMasterClient, String dateFormat) throws IOException { + mMasterStatus = new ArrayList<>(); + for (JobMasterStatus masterStatus : jobMasterClient.getAllMasterStatus()) { + mMasterStatus.add(new SerializableJobMasterStatus(masterStatus, dateFormat)); } - public void setRecentFailedJobs(List recentFailedJobs) { - mRecentFailedJobs = recentFailedJobs; + mWorkerHealth = new ArrayList<>(); + for (JobWorkerHealth workerHealth : jobMasterClient.getAllWorkerHealth()) { + mWorkerHealth.add(new SerializableWorkerHealth(workerHealth)); } - public List getLongestRunningJobs() { - return mLongestRunningJobs; + JobServiceSummary jobServiceSummary = jobMasterClient.getJobServiceSummary(); + mStatusSummary = new ArrayList<>(); + for (StatusSummary statusSummary : jobServiceSummary.getSummaryPerStatus()) { + mStatusSummary.add(new SerializableStatusSummary(statusSummary)); } - public void setLongestRunningJobs(List longestRunningJobs) { - mLongestRunningJobs = longestRunningJobs; + mRecentModifiedJobs = new ArrayList<>(); + mRecentFailedJobs = new ArrayList<>(); + mLongestRunningJobs = new ArrayList<>(); + for (JobInfo jobInfo : jobServiceSummary.getRecentActivities()) { + mRecentModifiedJobs.add(new SerializableJobInfo(jobInfo, dateFormat)); + } + for (JobInfo jobInfo : jobServiceSummary.getRecentFailures()) { + mRecentFailedJobs.add(new SerializableJobInfo(jobInfo, dateFormat)); + } + for (JobInfo jobInfo : jobServiceSummary.getLongestRunning()) { + mLongestRunningJobs.add(new SerializableJobInfo(jobInfo, dateFormat)); } + } + + /** + * Get master status. + * + * @return master status + */ + public List getMasterStatus() { + return mMasterStatus; + } + + /** + * Set master status. + * + * @param masterStatus master status + */ + public void setMasterStatus(List masterStatus) { + mMasterStatus = masterStatus; + } + + /** + * Get worker health info. + * + * @return worker health info + */ + public List getWorkerHealth() { + return mWorkerHealth; + } + + /** + * Set worker health info. + * + * @param workerHealth worker health info + */ + public void setWorkerHealth(List workerHealth) { + mWorkerHealth = workerHealth; + } + + /** + * Get status summary. + * + * @return status summary + */ + public List getStatusSummary() { + return mStatusSummary; + } + + /** + * Set status summary. + * + * @param statusSummary status summary + */ + public void setStatusSummary(List statusSummary) { + mStatusSummary = statusSummary; + } + + /** + * Get recent modified jobs. + * + * @return recent modified jobs + */ + public List getRecentModifiedJobs() { + return mRecentModifiedJobs; + } + + /** + * Set recent modified jobs. + * + * @param recentModifiedJobs recent modified jobs + */ + public void setRecentModifiedJobs(List recentModifiedJobs) { + mRecentModifiedJobs = recentModifiedJobs; + } + + /** + * Get recent failed jobs. + * + * @return recent failed jobs + */ + public List getRecentFailedJobs() { + return mRecentFailedJobs; + } + + /** + * Set recent failed jobs. + * + * @param recentFailedJobs recent failed jobs + */ + public void setRecentFailedJobs(List recentFailedJobs) { + mRecentFailedJobs = recentFailedJobs; + } + + /** + * Get the longest running jobs. + * + * @return longest running jobs + */ + public List getLongestRunningJobs() { + return mLongestRunningJobs; + } + + /** + * Set the longest running jobs. + * + * @param longestRunningJobs longest running jobs + */ + public void setLongestRunningJobs(List longestRunningJobs) { + mLongestRunningJobs = longestRunningJobs; + } } diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java index d47fcef3cd06..11fd638aeeb8 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsCommand.java @@ -52,8 +52,8 @@ public int run() throws IOException { String json = objectMapper.writeValueAsString(metricsInfo); mPrintStream.println(json); } catch (JsonProcessingException e) { - mPrintStream.println("Failed to convert metricsInfo output to JSON. " + - "Check the command line log for the detailed error message."); + mPrintStream.println("Failed to convert metricsInfo output to JSON. " + + "Check the command line log for the detailed error message."); LOG.error("Failed to output JSON object {}", metricsInfo); e.printStackTrace(); return -1; diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java index 2c410c64885c..ea0ce5789ee5 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java @@ -3,6 +3,7 @@ import alluxio.grpc.MetricValue; import alluxio.metrics.MetricsSystem; import alluxio.util.FormatUtils; + import com.google.common.math.DoubleMath; import java.text.DecimalFormat; @@ -12,96 +13,111 @@ import java.util.Locale; import java.util.Map; +/** + * An output class, describing the metrics. + */ public class MetricsOutput { - private static final String BYTES_METRIC_IDENTIFIER = "Bytes"; - private static final String THROUGHPUT_METRIC_IDENTIFIER = "Throughput"; - private static final DecimalFormat DECIMAL_FORMAT - = new DecimalFormat("###,###.#####", new DecimalFormatSymbols(Locale.US)); - private List mMetricsInfo; - - private class SerializableMetricInfo { - private String mKey; - private String mType; - private String mValue; - - public SerializableMetricInfo(Map.Entry entry) { - mKey = entry.getKey(); - mType = entry.getValue().getMetricType().toString(); - if (entry.getValue().hasStringValue()) { - mValue = entry.getValue().getStringValue(); - } else { - double doubleValue = entry.getValue().getDoubleValue(); - if (mKey.contains(BYTES_METRIC_IDENTIFIER)) { - // Bytes long can be transformed to human-readable format - mValue = FormatUtils.getSizeFromBytes((long) doubleValue); - if (mKey.contains(THROUGHPUT_METRIC_IDENTIFIER)) { - mValue = mValue + "/MIN"; - } - } else if (DoubleMath.isMathematicalInteger(doubleValue)) { - mValue = DECIMAL_FORMAT.format((long) doubleValue); - } else { - mValue = String.valueOf(doubleValue); - } - } - } - - public String getKey() { - return mKey; - } - - public void setKey(String key) { - mKey = key; - } - - public String getType() { - return mType; - } - - public void setType(String type) { - mType = type; - } - - public String getValue() { - return mValue; + private static final String BYTES_METRIC_IDENTIFIER = "Bytes"; + private static final String THROUGHPUT_METRIC_IDENTIFIER = "Throughput"; + private static final DecimalFormat DECIMAL_FORMAT + = new DecimalFormat("###,###.#####", new DecimalFormatSymbols(Locale.US)); + private List mMetricsInfo; + + private class SerializableMetricInfo { + private String mKey; + private String mType; + private String mValue; + + public SerializableMetricInfo(Map.Entry entry) { + mKey = entry.getKey(); + mType = entry.getValue().getMetricType().toString(); + if (entry.getValue().hasStringValue()) { + mValue = entry.getValue().getStringValue(); + } else { + double doubleValue = entry.getValue().getDoubleValue(); + if (mKey.contains(BYTES_METRIC_IDENTIFIER)) { + // Bytes long can be transformed to human-readable format + mValue = FormatUtils.getSizeFromBytes((long) doubleValue); + if (mKey.contains(THROUGHPUT_METRIC_IDENTIFIER)) { + mValue = mValue + "/MIN"; + } + } else if (DoubleMath.isMathematicalInteger(doubleValue)) { + mValue = DECIMAL_FORMAT.format((long) doubleValue); + } else { + mValue = String.valueOf(doubleValue); } + } + } - public void setValue(String value) { - mValue = value; - } + public String getKey() { + return mKey; } - public MetricsOutput(Map metrics){ - mMetricsInfo = new ArrayList<>(); - for (Map.Entry entry : metrics.entrySet()){ - String key = entry.getKey(); - if (!isAlluxioMetric(key)) { - continue; - } - mMetricsInfo.add(new SerializableMetricInfo(entry)); - } + public void setKey(String key) { + mKey = key; } - /** - * Checks if a metric is Alluxio metric. - * - * @param name name of the metrics to check - * @return true if a metric is an Alluxio metric, false otherwise - */ - private boolean isAlluxioMetric(String name) { - for (MetricsSystem.InstanceType instance : MetricsSystem.InstanceType.values()) { - if (name.startsWith(instance.toString())) { - return true; - } - } - return false; + public String getType() { + return mType; } - public List getMetricsInfo() { - return mMetricsInfo; + public void setType(String type) { + mType = type; } - public void setMetricsInfo(List metricsInfo) { - mMetricsInfo = metricsInfo; + public String getValue() { + return mValue; } + public void setValue(String value) { + mValue = value; + } + } + + /** + * Creates a new instance of {@link MetricsOutput}. + * + * @param metrics metric to parse output from + */ + public MetricsOutput(Map metrics) { + mMetricsInfo = new ArrayList<>(); + for (Map.Entry entry : metrics.entrySet()) { + String key = entry.getKey(); + if (!isAlluxioMetric(key)) { + continue; + } + mMetricsInfo.add(new SerializableMetricInfo(entry)); + } + } + + /** + * Checks if a metric is Alluxio metric. + * + * @param name name of the metrics to check + * @return true if a metric is an Alluxio metric, false otherwise + */ + private boolean isAlluxioMetric(String name) { + for (MetricsSystem.InstanceType instance : MetricsSystem.InstanceType.values()) { + if (name.startsWith(instance.toString())) { + return true; + } + } + return false; + } + + /** + * Get metrics info. + * @return metrics info + */ + public List getMetricsInfo() { + return mMetricsInfo; + } + + /** + * Set metrics info. + * @param metricsInfo metrics info + */ + public void setMetricsInfo(List metricsInfo) { + mMetricsInfo = metricsInfo; + } } diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java index 3bce5f39deac..4787ab98056f 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java @@ -83,8 +83,8 @@ public int run() throws IOException { String json = objectMapper.writeValueAsString(summaryInfo); mPrintStream.println(json); } catch (JsonProcessingException e) { - mPrintStream.println("Failed to convert summaryInfo output to JSON. " + - "Check the command line log for the detailed error message."); + mPrintStream.println("Failed to convert summaryInfo output to JSON. " + + "Check the command line log for the detailed error message."); LOG.error("Failed to output JSON object {}", summaryInfo); e.printStackTrace(); return -1; diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java index 8edaf8c507ab..25a8399d9859 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java @@ -3,256 +3,441 @@ import alluxio.cli.fsadmin.FileSystemAdminShellUtils; import alluxio.grpc.MasterInfo; import alluxio.grpc.MasterVersion; -import alluxio.wire.BlockMasterInfo; import alluxio.util.CommonUtils; import alluxio.util.FormatUtils; +import alluxio.wire.BlockMasterInfo; import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.TreeMap; +/** + * An output class, describing the summary. + */ public class SummaryOutput { - private String mMasterAddress; - private int mWebPort; - private int mRpcPort; - private String mStarted; - private String mUptime; + private String mMasterAddress; + private int mWebPort; + private int mRpcPort; + private String mStarted; + private String mUptime; + private String mVersion; + private boolean mSafeMode; + + private List mZookeeperAddress; + private boolean mUseZookeeper; + private List mRaftJournalAddress; + private boolean mUseRaftJournal; + private List mMasterVersions; + + private int mLiveWorkers; + private int mLostWorkers; + private Map mTotalCapacityOnTiers; + private Map mUsedCapacityOnTiers; + private String mFreeCapacity; + + private static class SerializableMasterVersion { + private String mHost; + private int mPort; + private String mState; private String mVersion; - private boolean mSafeMode; - - private List mZookeeperAddress; - private boolean mUseZookeeper; - private List mRaftJournalAddress; - private boolean mUseRaftJournal; - private List mMasterVersions; - - private int mLiveWorkers; - private int mLostWorkers; - private Map mTotalCapacityOnTiers; - private Map mUsedCapacityOnTiers; - private String mFreeCapacity; - - private static class SerializableMasterVersion { - private String mHost; - private int mPort; - private String mState; - private String mVersion; - - public SerializableMasterVersion(MasterVersion masterVersion) { - mHost = masterVersion.getAddresses().getHost(); - mPort = masterVersion.getAddresses().getRpcPort(); - mState = masterVersion.getState(); - mVersion = masterVersion.getVersion(); - } - - public String getHost() { - return mHost; - } - - public void setHost(String host) { - mHost = host; - } - - public int getPort() { - return mPort; - } - - public void setPort(int port) { - mPort = port; - } - - public String getState() { - return mState; - } - - public void setState(String state) { - mState = state; - } - - public String getVersion() { - return mVersion; - } - - public void setVersion(String version) { - mVersion = version; - } - } - - public SummaryOutput(MasterInfo masterInfo, BlockMasterInfo blockMasterInfo, String dateFormatPattern) { - // give values to internal properties - mMasterAddress = masterInfo.getLeaderMasterAddress(); - mWebPort = masterInfo.getWebPort(); - mRpcPort = masterInfo.getRpcPort(); - mStarted = CommonUtils.convertMsToDate(masterInfo.getStartTimeMs(), dateFormatPattern); - mUptime = CommonUtils.convertMsToClockTime(masterInfo.getUpTimeMs()); - mVersion = masterInfo.getVersion(); - mSafeMode = masterInfo.getSafeMode(); - - mZookeeperAddress = masterInfo.getZookeeperAddressesList(); - mUseZookeeper = !mZookeeperAddress.isEmpty(); - mUseRaftJournal = masterInfo.getRaftJournal(); - if (mUseRaftJournal) { - mRaftJournalAddress = masterInfo.getRaftAddressList(); - } else { - mRaftJournalAddress = new ArrayList<>(); - } - mMasterVersions = new ArrayList<>(); - for (MasterVersion masterVersion : masterInfo.getMasterVersionsList()) { - mMasterVersions.add(new SerializableMasterVersion(masterVersion)); - } - - mLiveWorkers = blockMasterInfo.getLiveWorkerNum(); - mLostWorkers = blockMasterInfo.getLostWorkerNum(); - - mTotalCapacityOnTiers = new TreeMap<>(); - mUsedCapacityOnTiers = new TreeMap<>(); - Map totalBytesOnTiers = new TreeMap<>(FileSystemAdminShellUtils::compareTierNames); - totalBytesOnTiers.putAll(blockMasterInfo.getCapacityBytesOnTiers()); - for (Map.Entry totalBytesOnTier : totalBytesOnTiers.entrySet()) { - mTotalCapacityOnTiers.put(totalBytesOnTier.getKey(), FormatUtils.getSizeFromBytes(totalBytesOnTier.getValue())); - } - Map usedBytesOnTiers = new TreeMap<>(FileSystemAdminShellUtils::compareTierNames); - usedBytesOnTiers.putAll(blockMasterInfo.getUsedBytesOnTiers()); - for (Map.Entry usedBytesOnTier : usedBytesOnTiers.entrySet()) { - mUsedCapacityOnTiers.put(usedBytesOnTier.getKey(), FormatUtils.getSizeFromBytes(usedBytesOnTier.getValue())); - } - mFreeCapacity = FormatUtils.getSizeFromBytes(blockMasterInfo.getFreeBytes()); - } - - public String getMasterAddress() { - return mMasterAddress; - } - public void setMasterAddress(String masterAddress) { - mMasterAddress = masterAddress; + public SerializableMasterVersion(MasterVersion masterVersion) { + mHost = masterVersion.getAddresses().getHost(); + mPort = masterVersion.getAddresses().getRpcPort(); + mState = masterVersion.getState(); + mVersion = masterVersion.getVersion(); } - public int getWebPort() { - return mWebPort; + public String getHost() { + return mHost; } - public void setWebPort(int webPort) { - mWebPort = webPort; + public void setHost(String host) { + mHost = host; } - public int getRpcPort() { - return mRpcPort; + public int getPort() { + return mPort; } - public void setRpcPort(int rpcPort) { - mRpcPort = rpcPort; + public void setPort(int port) { + mPort = port; } - public String getStarted() { - return mStarted; + public String getState() { + return mState; } - public void setStarted(String started) { - mStarted = started; - } - - public String getUptime() { - return mUptime; - } - - public void setUptime(String uptime) { - mUptime = uptime; + public void setState(String state) { + mState = state; } public String getVersion() { - return mVersion; + return mVersion; } public void setVersion(String version) { - mVersion = version; - } - - public boolean ismSafeMode() { - return mSafeMode; - } - - public void setSafeMode(boolean safeMode) { - mSafeMode = safeMode; - } - - public List getZookeeperAddress() { - return mZookeeperAddress; - } - - public void setZookeeperAddress(List zookeeperAddress) { - mZookeeperAddress = zookeeperAddress; - } - - public boolean isUseZookeeper() { - return mUseZookeeper; - } - - public void setUseZookeeper(boolean useZookeeper) { - mUseZookeeper = useZookeeper; - } - - public List getRaftJournalAddress() { - return mRaftJournalAddress; - } - - public void setRaftJournalAddress(List raftJournalAddress) { - mRaftJournalAddress = raftJournalAddress; - } - - public boolean isUseRaftJournal() { - return mUseRaftJournal; - } - - public void setUseRaftJournal(boolean useRaftJournal) { - mUseRaftJournal = useRaftJournal; - } - - public List getMasterVersions() { - return mMasterVersions; - } - - public void setMasterVersions(List masterVersions) { - mMasterVersions = masterVersions; - } - - public int getLiveWorkers() { - return mLiveWorkers; - } - - public void setLiveWorkers(int liveWorkers) { - mLiveWorkers = liveWorkers; - } - - public int getLostWorkers() { - return mLostWorkers; - } - - public void setLostWorkers(int lostWorkers) { - mLostWorkers = lostWorkers; - } - - public String getFreeCapacity() { - return mFreeCapacity; - } - - public void setFreeCapacity(String freeCapacity) { - mFreeCapacity = freeCapacity; - } - - public Map getTotalCapacityOnTiers() { - return mTotalCapacityOnTiers; - } - - public void setTotalCapacityOnTiers(Map totalCapacityOnTiers) { - mTotalCapacityOnTiers = totalCapacityOnTiers; - } - - public Map getUsedCapacityOnTiers() { - return mUsedCapacityOnTiers; - } - - public void setUsedCapacityOnTiers(Map usedCapacityOnTiers) { - mUsedCapacityOnTiers = usedCapacityOnTiers; - } + mVersion = version; + } + } + + /** + * Creates a new instance of {@link SummaryOutput}. + * + * @param masterInfo given master info + * @param blockMasterInfo given block master info + * @param dateFormatPattern specify the pattern of dates + */ + public SummaryOutput(MasterInfo masterInfo, BlockMasterInfo blockMasterInfo, + String dateFormatPattern) { + // give values to internal properties + mMasterAddress = masterInfo.getLeaderMasterAddress(); + mWebPort = masterInfo.getWebPort(); + mRpcPort = masterInfo.getRpcPort(); + mStarted = CommonUtils.convertMsToDate(masterInfo.getStartTimeMs(), dateFormatPattern); + mUptime = CommonUtils.convertMsToClockTime(masterInfo.getUpTimeMs()); + mVersion = masterInfo.getVersion(); + mSafeMode = masterInfo.getSafeMode(); + + mZookeeperAddress = masterInfo.getZookeeperAddressesList(); + mUseZookeeper = !mZookeeperAddress.isEmpty(); + mUseRaftJournal = masterInfo.getRaftJournal(); + if (mUseRaftJournal) { + mRaftJournalAddress = masterInfo.getRaftAddressList(); + } else { + mRaftJournalAddress = new ArrayList<>(); + } + mMasterVersions = new ArrayList<>(); + for (MasterVersion masterVersion : masterInfo.getMasterVersionsList()) { + mMasterVersions.add(new SerializableMasterVersion(masterVersion)); + } + + mLiveWorkers = blockMasterInfo.getLiveWorkerNum(); + mLostWorkers = blockMasterInfo.getLostWorkerNum(); + + mTotalCapacityOnTiers = new TreeMap<>(); + mUsedCapacityOnTiers = new TreeMap<>(); + Map totalBytesOnTiers = + new TreeMap<>(FileSystemAdminShellUtils::compareTierNames); + totalBytesOnTiers.putAll(blockMasterInfo.getCapacityBytesOnTiers()); + for (Map.Entry totalBytesOnTier : totalBytesOnTiers.entrySet()) { + mTotalCapacityOnTiers.put(totalBytesOnTier.getKey(), + FormatUtils.getSizeFromBytes(totalBytesOnTier.getValue())); + } + Map usedBytesOnTiers = + new TreeMap<>(FileSystemAdminShellUtils::compareTierNames); + usedBytesOnTiers.putAll(blockMasterInfo.getUsedBytesOnTiers()); + for (Map.Entry usedBytesOnTier : usedBytesOnTiers.entrySet()) { + mUsedCapacityOnTiers.put(usedBytesOnTier.getKey(), + FormatUtils.getSizeFromBytes(usedBytesOnTier.getValue())); + } + mFreeCapacity = FormatUtils.getSizeFromBytes(blockMasterInfo.getFreeBytes()); + } + + /** + * Get master address. + * + * @return master address + */ + public String getMasterAddress() { + return mMasterAddress; + } + + /** + * Set master address. + * + * @param masterAddress master address + */ + public void setMasterAddress(String masterAddress) { + mMasterAddress = masterAddress; + } + + /** + * Get web port. + * + * @return web port + */ + public int getWebPort() { + return mWebPort; + } + + /** + * Set web port. + * + * @param webPort web port + */ + public void setWebPort(int webPort) { + mWebPort = webPort; + } + + /** + * Get rpc port. + * + * @return rpc port + */ + public int getRpcPort() { + return mRpcPort; + } + + /** + * Set rpc port. + * + * @param rpcPort rpc port + */ + public void setRpcPort(int rpcPort) { + mRpcPort = rpcPort; + } + + /** + * Get started time. + * + * @return started time + */ + public String getStarted() { + return mStarted; + } + + /** + * Set started time. + * + * @param started started time + */ + public void setStarted(String started) { + mStarted = started; + } + + /** + * Get time running. + * + * @return time running + */ + public String getUptime() { + return mUptime; + } + + /** + * Set time running. + * + * @param uptime time running + */ + public void setUptime(String uptime) { + mUptime = uptime; + } + + /** + * Get Alluxio version. + * + * @return Alluxio version + */ + public String getVersion() { + return mVersion; + } + + /** + * Set Alluxio version. + * + * @param version Alluxio version + */ + public void setVersion(String version) { + mVersion = version; + } + + /** + * Get if in safe mode. + * + * @return if in safe mode + */ + public boolean isSafeMode() { + return mSafeMode; + } + + /** + * Set if in safe mode. + * + * @param safeMode if in safe mode + */ + public void setSafeMode(boolean safeMode) { + mSafeMode = safeMode; + } + + /** + * Get zookeeper address. + * + * @return zookeeper address + */ + public List getZookeeperAddress() { + return mZookeeperAddress; + } + + /** + * Set zookeeper address. + * + * @param zookeeperAddress zookeeper address + */ + public void setZookeeperAddress(List zookeeperAddress) { + mZookeeperAddress = zookeeperAddress; + } + + /** + * Get if zookeeper is running. + * + * @return if zookeeper is running + */ + public boolean isUseZookeeper() { + return mUseZookeeper; + } + + /** + * Set if zookeeper is running. + * + * @param useZookeeper if zookeeper is running + */ + public void setUseZookeeper(boolean useZookeeper) { + mUseZookeeper = useZookeeper; + } + + /** + * Get raft journal address. + * + * @return raft journal address + */ + public List getRaftJournalAddress() { + return mRaftJournalAddress; + } + + /** + * Set raft journal address. + * + * @param raftJournalAddress raft journal address + */ + public void setRaftJournalAddress(List raftJournalAddress) { + mRaftJournalAddress = raftJournalAddress; + } + + /** + * Get if Alluxio uses raft journal. + * + * @return if Alluxio uses raft journal + */ + public boolean isUseRaftJournal() { + return mUseRaftJournal; + } + + /** + * Set if Alluxio uses raft journal. + * + * @param useRaftJournal if Alluxio uses raft journal + */ + public void setUseRaftJournal(boolean useRaftJournal) { + mUseRaftJournal = useRaftJournal; + } + + /** + * Get master versions. + * + * @return master versions + */ + public List getMasterVersions() { + return mMasterVersions; + } + + /** + * Set master versions. + * + * @param masterVersions master versions + */ + public void setMasterVersions(List masterVersions) { + mMasterVersions = masterVersions; + } + + /** + * Get live workers. + * + * @return live workers + */ + public int getLiveWorkers() { + return mLiveWorkers; + } + + /** + * Set live workers. + * + * @param liveWorkers live workers + */ + public void setLiveWorkers(int liveWorkers) { + mLiveWorkers = liveWorkers; + } + + /** + * Get lost workers. + * + * @return lost workers + */ + public int getLostWorkers() { + return mLostWorkers; + } + + /** + * Set lost workers. + * + * @param lostWorkers lost workers + */ + public void setLostWorkers(int lostWorkers) { + mLostWorkers = lostWorkers; + } + + /** + * Get free capacity. + * + * @return free capacity + */ + public String getFreeCapacity() { + return mFreeCapacity; + } + + /** + * Set free capacity. + * + * @param freeCapacity free capacity + */ + public void setFreeCapacity(String freeCapacity) { + mFreeCapacity = freeCapacity; + } + + /** + * Get capacity by tiers. + * + * @return capacity by tiers + */ + public Map getTotalCapacityOnTiers() { + return mTotalCapacityOnTiers; + } + + /** + * Get capacity by tiers. + * + * @param totalCapacityOnTiers capacity by tiers + */ + public void setTotalCapacityOnTiers(Map totalCapacityOnTiers) { + mTotalCapacityOnTiers = totalCapacityOnTiers; + } + + /** + * Get used capacity by tiers. + * + * @return used capacity by tiers + */ + public Map getUsedCapacityOnTiers() { + return mUsedCapacityOnTiers; + } + + /** + * Set used capacity by tiers. + * + * @param usedCapacityOnTiers used capacity by tiers + */ + public void setUsedCapacityOnTiers(Map usedCapacityOnTiers) { + mUsedCapacityOnTiers = usedCapacityOnTiers; + } } diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java index 7d1d66dacd33..a784acab3a40 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/UfsCommand.java @@ -13,9 +13,9 @@ import alluxio.client.file.FileSystemMasterClient; import alluxio.wire.MountPointInfo; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import org.jline.utils.Log; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,8 +50,8 @@ public int run() throws IOException { String json = objectMapper.writeValueAsString(mountTable); System.out.println(json); } catch (JsonProcessingException e) { - System.out.println("Failed to convert mountTable output to JSON. " + - "Check the command line log for the detailed error message."); + System.out.println("Failed to convert mountTable output to JSON. " + + "Check the command line log for the detailed error message."); LOG.error("Failed to output JSON object {}", mountTable); e.printStackTrace(); return -1; From 2da6e2939c8fbc988aeb227f1aaa1fa3a33e2adb Mon Sep 17 00:00:00 2001 From: twalluxio Date: Mon, 4 Sep 2023 15:56:17 +0800 Subject: [PATCH 16/21] add license --- .../alluxio/cli/fsadmin/report/JobServiceOutput.java | 11 +++++++++++ .../alluxio/cli/fsadmin/report/MetricsOutput.java | 11 +++++++++++ .../alluxio/cli/fsadmin/report/SummaryOutput.java | 11 +++++++++++ 3 files changed, 33 insertions(+) diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceOutput.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceOutput.java index 538a000a8748..6f3adfc16956 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceOutput.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceOutput.java @@ -1,3 +1,14 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + package alluxio.cli.fsadmin.report; import alluxio.client.job.JobMasterClient; diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java index ea0ce5789ee5..d2a05db9fc86 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java @@ -1,3 +1,14 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + package alluxio.cli.fsadmin.report; import alluxio.grpc.MetricValue; diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java index 25a8399d9859..50fc49793398 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryOutput.java @@ -1,3 +1,14 @@ +/* + * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 + * (the "License"). You may not use this work except in compliance with the License, which is + * available at www.apache.org/licenses/LICENSE-2.0 + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, + * either express or implied, as more fully set forth in the License. + * + * See the NOTICE file distributed with this work for information regarding copyright ownership. + */ + package alluxio.cli.fsadmin.report; import alluxio.cli.fsadmin.FileSystemAdminShellUtils; From 92db746b9b77dae57b76033f556b8977bc1ac976 Mon Sep 17 00:00:00 2001 From: twalluxio Date: Tue, 5 Sep 2023 12:05:33 +0800 Subject: [PATCH 17/21] fix unit tests --- .../report/JobServiceMetricsCommandTest.java | 119 ++++++++++-------- .../fsadmin/report/MetricsCommandTest.java | 113 ++++++++++++----- .../fsadmin/report/SummaryCommandTest.java | 101 +++++++++------ 3 files changed, 216 insertions(+), 117 deletions(-) diff --git a/dora/shell/src/test/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommandTest.java b/dora/shell/src/test/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommandTest.java index c279a9e75614..4657d401bc79 100644 --- a/dora/shell/src/test/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommandTest.java +++ b/dora/shell/src/test/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommandTest.java @@ -12,7 +12,6 @@ package alluxio.cli.fsadmin.report; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import alluxio.client.job.JobMasterClient; import alluxio.grpc.BuildVersion; @@ -24,8 +23,9 @@ import alluxio.job.wire.PlanInfo; import alluxio.job.wire.Status; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Lists; -import org.apache.commons.lang3.ArrayUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -101,67 +101,80 @@ public void testBasic() throws IOException, ParseException { Mockito.when(mJobMasterClient.getJobServiceSummary()) .thenReturn(new JobServiceSummary(jobInfos)); - new JobServiceMetricsCommand(mJobMasterClient, mPrintStream, "MM-dd-yyyy HH:mm:ss:SSS").run(); + new JobServiceMetricsCommand(mJobMasterClient, mPrintStream, "yyyyMMdd-HHmmss").run(); String output = new String(mOutputStream.toByteArray(), StandardCharsets.UTF_8); - - String[] lineByLine = output.split("\n"); + ObjectMapper mapper = new ObjectMapper(); + JsonNode jsonNode = mapper.readTree(output); // Master Status Section - assertTrue(lineByLine[0].contains("Master Address State Start Time " - + "Version Revision")); - assertTrue(lineByLine[1].contains("master-node-1:19998 PRIMARY")); - assertTrue(lineByLine[1].contains(startTimeStr)); - assertTrue(lineByLine[1].contains("alluxio-version-2.9 abcdef")); - assertTrue(lineByLine[2].contains("master-node-0:19998 STANDBY")); - assertTrue(lineByLine[2].contains(startTimeStr)); - assertTrue(lineByLine[2].contains("alluxio-version-2.10 abcdef")); - assertTrue(lineByLine[3].contains("master-node-2:19998 STANDBY")); - assertTrue(lineByLine[3].contains(startTimeStr)); - assertTrue(lineByLine[3].contains("alluxio-version-2.10 bcdefg")); + JsonNode masterStatuses = jsonNode.get("masterStatus"); + assertEquals(startTimeStr, masterStatuses.get(0).get("startTime").asText()); + assertEquals("abcdef", masterStatuses.get(0).get("revision").asText()); + assertEquals("19998", masterStatuses.get(0).get("port").asText()); + assertEquals("master-node-1", masterStatuses.get(0).get("host").asText()); + assertEquals("alluxio-version-2.9", masterStatuses.get(0).get("version").asText()); + assertEquals("PRIMARY", masterStatuses.get(0).get("state").asText()); + assertEquals(startTimeStr, masterStatuses.get(1).get("startTime").asText()); + assertEquals("abcdef", masterStatuses.get(1).get("revision").asText()); + assertEquals("19998", masterStatuses.get(1).get("port").asText()); + assertEquals("master-node-0", masterStatuses.get(1).get("host").asText()); + assertEquals("alluxio-version-2.10", masterStatuses.get(1).get("version").asText()); + assertEquals("STANDBY", masterStatuses.get(1).get("state").asText()); + assertEquals(startTimeStr, masterStatuses.get(2).get("startTime").asText()); + assertEquals("bcdefg", masterStatuses.get(2).get("revision").asText()); + assertEquals("19998", masterStatuses.get(2).get("port").asText()); + assertEquals("master-node-2", masterStatuses.get(2).get("host").asText()); + assertEquals("alluxio-version-2.10", masterStatuses.get(2).get("version").asText()); + assertEquals("STANDBY", masterStatuses.get(2).get("state").asText()); // Worker Health Section - assertTrue(lineByLine[5].contains("Job Worker Version " - + "Revision Task Pool Size Unfinished Tasks Active Tasks Load Avg")); - assertTrue(lineByLine[6].contains("testHost 2.10.0-SNAPSHOT " - + "ac6a0616")); - assertTrue(lineByLine[6].contains("10 2 2 " - + "1.2, 0.9, 0.7")); + JsonNode workerHealth = jsonNode.get("workerHealth"); + assertEquals("ac6a0616", workerHealth.get(0).get("revision").asText()); + assertEquals("2", workerHealth.get(0).get("activeTasks").asText()); + assertEquals("1.2", workerHealth.get(0).get("loadAverage").get(0).asText()); + assertEquals("0.9", workerHealth.get(0).get("loadAverage").get(1).asText()); + assertEquals("0.7", workerHealth.get(0).get("loadAverage").get(2).asText()); + assertEquals("10", workerHealth.get(0).get("taskPoolSize").asText()); + assertEquals("2", workerHealth.get(0).get("unfinishedTasks").asText()); + assertEquals("testHost", workerHealth.get(0).get("host").asText()); + assertEquals("2.10.0-SNAPSHOT", workerHealth.get(0).get("version").asText()); // Group By Status - lineByLine = ArrayUtils.subarray(lineByLine, 8, lineByLine.length); - - assertEquals("Status: CREATED Count: 0", lineByLine[0]); - assertEquals("Status: CANCELED Count: 0", lineByLine[1]); - assertEquals("Status: FAILED Count: 1", lineByLine[2]); - assertEquals("Status: RUNNING Count: 1", lineByLine[3]); - assertEquals("Status: COMPLETED Count: 0", lineByLine[4]); - assertEquals("", lineByLine[5]); + JsonNode statusSummary = jsonNode.get("statusSummary"); + assertEquals("CREATED", statusSummary.get(0).get("status").asText()); + assertEquals("0", statusSummary.get(0).get("count").asText()); + assertEquals("CANCELED", statusSummary.get(1).get("status").asText()); + assertEquals("0", statusSummary.get(1).get("count").asText()); + assertEquals("FAILED", statusSummary.get(2).get("status").asText()); + assertEquals("1", statusSummary.get(2).get("count").asText()); + assertEquals("RUNNING", statusSummary.get(3).get("status").asText()); + assertEquals("1", statusSummary.get(3).get("count").asText()); + assertEquals("COMPLETED", statusSummary.get(4).get("status").asText()); + assertEquals("0", statusSummary.get(4).get("count").asText()); // Top 10 - lineByLine = ArrayUtils.subarray(lineByLine, 6, lineByLine.length); - - assertEquals("10 Most Recently Modified Jobs:", lineByLine[0]); - assertEquals( - "Timestamp: 01-17-2019 12:30:15:000 Id: 2 Name: Test2" - + " Status: FAILED", - lineByLine[1]); - assertEquals( - "Timestamp: 01-17-2019 12:00:00:000 Id: 1 Name: Test1" - + " Status: RUNNING", - lineByLine[2]); - assertEquals("", lineByLine[3]); - assertEquals("10 Most Recently Failed Jobs:", lineByLine[4]); - assertEquals( - "Timestamp: 01-17-2019 12:30:15:000 Id: 2 Name: Test2" - + " Status: FAILED", - lineByLine[5]); - assertEquals("", lineByLine[6]); - assertEquals("10 Longest Running Jobs:", lineByLine[7]); - assertEquals( - "Timestamp: 01-17-2019 12:00:00:000 Id: 1 Name: Test1" - + " Status: RUNNING", - lineByLine[8]); + JsonNode recentModifiedJobs = jsonNode.get("recentModifiedJobs"); + assertEquals("2", recentModifiedJobs.get(0).get("id").asText()); + assertEquals("FAILED", recentModifiedJobs.get(0).get("status").asText()); + assertEquals("20190117-123015", recentModifiedJobs.get(0).get("timestamp").asText()); + assertEquals("Test2", recentModifiedJobs.get(0).get("name").asText()); + assertEquals("1", recentModifiedJobs.get(1).get("id").asText()); + assertEquals("RUNNING", recentModifiedJobs.get(1).get("status").asText()); + assertEquals("20190117-120000", recentModifiedJobs.get(1).get("timestamp").asText()); + assertEquals("Test1", recentModifiedJobs.get(1).get("name").asText()); + + JsonNode recentFailedJobs = jsonNode.get("recentFailedJobs"); + assertEquals("2", recentFailedJobs.get(0).get("id").asText()); + assertEquals("FAILED", recentFailedJobs.get(0).get("status").asText()); + assertEquals("20190117-123015", recentFailedJobs.get(0).get("timestamp").asText()); + assertEquals("Test2", recentFailedJobs.get(0).get("name").asText()); + + JsonNode longestRunningJobs = jsonNode.get("longestRunningJobs"); + assertEquals("1", longestRunningJobs.get(0).get("id").asText()); + assertEquals("RUNNING", longestRunningJobs.get(0).get("status").asText()); + assertEquals("20190117-120000", longestRunningJobs.get(0).get("timestamp").asText()); + assertEquals("Test1", longestRunningJobs.get(0).get("name").asText()); } private JobInfo createJobInfo(int id, String name, Status status, String datetime) diff --git a/dora/shell/src/test/java/alluxio/cli/fsadmin/report/MetricsCommandTest.java b/dora/shell/src/test/java/alluxio/cli/fsadmin/report/MetricsCommandTest.java index a2cff516448b..d2f59ba5d999 100644 --- a/dora/shell/src/test/java/alluxio/cli/fsadmin/report/MetricsCommandTest.java +++ b/dora/shell/src/test/java/alluxio/cli/fsadmin/report/MetricsCommandTest.java @@ -11,14 +11,19 @@ package alluxio.cli.fsadmin.report; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + import alluxio.client.metrics.MetricsMasterClient; import alluxio.grpc.MetricType; import alluxio.grpc.MetricValue; import alluxio.metrics.MetricKey; -import org.hamcrest.collection.IsIterableContainingInOrder; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.Mockito; @@ -27,9 +32,7 @@ import java.io.IOException; import java.io.PrintStream; import java.nio.charset.StandardCharsets; -import java.util.Arrays; import java.util.HashMap; -import java.util.List; import java.util.Map; public class MetricsCommandTest { @@ -117,30 +120,84 @@ private Map generateMetricsMap() { /** * Checks if the output is expected. */ - private void checkIfOutputValid() { + private void checkIfOutputValid() throws JsonProcessingException { String output = new String(mOutputStream.toByteArray(), StandardCharsets.UTF_8); - // CHECKSTYLE.OFF: LineLengthExceed - Much more readable - List expectedOutput = Arrays.asList( - "Cluster.BytesReadDomain (Type: GAUGE, Value: 4145.73KB)", - "Cluster.BytesReadDomainThroughput (Type: GAUGE, Value: 29.97MB/MIN)", - "Cluster.BytesReadRemote (Type: GAUGE, Value: 401.79MB)", - "Cluster.BytesReadRemoteThroughput (Type: GAUGE, Value: 518.36MB/MIN)", - "Cluster.BytesReadUfsAll (Type: GAUGE, Value: 509.47MB)", - "Cluster.BytesReadUfsThroughput (Type: GAUGE, Value: 728.16KB/MIN)", - "Cluster.BytesWrittenDomain (Type: GAUGE, Value: 62.43MB)", - "Cluster.BytesWrittenDomainThroughput (Type: GAUGE, Value: 1202.37KB/MIN)", - "Cluster.BytesWrittenRemote (Type: GAUGE, Value: 22.98KB)", - "Cluster.BytesWrittenRemoteThroughput (Type: GAUGE, Value: 8.03MB/MIN)", - "Cluster.BytesWrittenUfsAll (Type: GAUGE, Value: 317.70KB)", - "Cluster.BytesWrittenUfsThroughput (Type: GAUGE, Value: 33.46KB/MIN)", - "Cluster.CapacityTotal (Type: GAUGE, Value: 1,154,531,246,129,122)", - "Master.CompleteFileOps (Type: COUNTER, Value: 813)", - "Master.UfsSessionCount-Ufs:_alluxio_underFSStorage (Type: COUNTER, Value: 8,535)", - "Master.UfsSessionCount-Ufs:file:___Users_alluxio_alluxioMountedFolder (Type: COUNTER, Value: 1,231)", - "Master.getMetrics.User:alluxio (Type: TIMER, Value: 4)"); - // CHECKSTYLE.ON: LineLengthExceed - List testOutput = Arrays.asList(output.split("\n")); - Assert.assertThat(testOutput, - IsIterableContainingInOrder.contains(expectedOutput.toArray())); + ObjectMapper mapper = new ObjectMapper(); + JsonNode metricsInfo = mapper.readTree(output).get("metricsInfo"); + assertEquals(17, metricsInfo.size()); + for (JsonNode metricInfo : metricsInfo) { + switch (metricInfo.get("key").asText()) { + case "Cluster.BytesReadDomain": + assertEquals("GAUGE", metricInfo.get("type").asText()); + assertEquals("4145.73KB", metricInfo.get("value").asText()); + break; + case "Cluster.BytesReadDomainThroughput": + assertEquals("GAUGE", metricInfo.get("type").asText()); + assertEquals("29.97MB/MIN", metricInfo.get("value").asText()); + break; + case "Cluster.BytesReadRemote": + assertEquals("GAUGE", metricInfo.get("type").asText()); + assertEquals("401.79MB", metricInfo.get("value").asText()); + break; + case "Cluster.BytesReadRemoteThroughput": + assertEquals("GAUGE", metricInfo.get("type").asText()); + assertEquals("518.36MB/MIN", metricInfo.get("value").asText()); + break; + case "Cluster.BytesReadUfsAll": + assertEquals("GAUGE", metricInfo.get("type").asText()); + assertEquals("509.47MB", metricInfo.get("value").asText()); + break; + case "Cluster.BytesReadUfsThroughput": + assertEquals("GAUGE", metricInfo.get("type").asText()); + assertEquals("728.16KB/MIN", metricInfo.get("value").asText()); + break; + case "Cluster.BytesWrittenDomain": + assertEquals("GAUGE", metricInfo.get("type").asText()); + assertEquals("62.43MB", metricInfo.get("value").asText()); + break; + case "Cluster.BytesWrittenDomainThroughput": + assertEquals("GAUGE", metricInfo.get("type").asText()); + assertEquals("1202.37KB/MIN", metricInfo.get("value").asText()); + break; + case "Cluster.BytesWrittenRemote": + assertEquals("GAUGE", metricInfo.get("type").asText()); + assertEquals("22.98KB", metricInfo.get("value").asText()); + break; + case "Cluster.BytesWrittenRemoteThroughput": + assertEquals("GAUGE", metricInfo.get("type").asText()); + assertEquals("8.03MB/MIN", metricInfo.get("value").asText()); + break; + case "Cluster.BytesWrittenUfsAll": + assertEquals("GAUGE", metricInfo.get("type").asText()); + assertEquals("317.70KB", metricInfo.get("value").asText()); + break; + case "Cluster.BytesWrittenUfsThroughput": + assertEquals("GAUGE", metricInfo.get("type").asText()); + assertEquals("33.46KB/MIN", metricInfo.get("value").asText()); + break; + case "Cluster.CapacityTotal": + assertEquals("GAUGE", metricInfo.get("type").asText()); + assertEquals("1,154,531,246,129,122", metricInfo.get("value").asText()); + break; + case "Master.CompleteFileOps": + assertEquals("COUNTER", metricInfo.get("type").asText()); + assertEquals("813", metricInfo.get("value").asText()); + break; + case "Master.UfsSessionCount-Ufs:_alluxio_underFSStorage": + assertEquals("COUNTER", metricInfo.get("type").asText()); + assertEquals("8,535", metricInfo.get("value").asText()); + break; + case "Master.UfsSessionCount-Ufs:file:___Users_alluxio_alluxioMountedFolder": + assertEquals("COUNTER", metricInfo.get("type").asText()); + assertEquals("1,231", metricInfo.get("value").asText()); + break; + case "Master.getMetrics.User:alluxio": + assertEquals("TIMER", metricInfo.get("type").asText()); + assertEquals("4", metricInfo.get("value").asText()); + break; + default: + fail("Invalid key: " + metricInfo.get("key").asText()); + } + } } } diff --git a/dora/shell/src/test/java/alluxio/cli/fsadmin/report/SummaryCommandTest.java b/dora/shell/src/test/java/alluxio/cli/fsadmin/report/SummaryCommandTest.java index 303fd4add38c..b5b685a5c890 100644 --- a/dora/shell/src/test/java/alluxio/cli/fsadmin/report/SummaryCommandTest.java +++ b/dora/shell/src/test/java/alluxio/cli/fsadmin/report/SummaryCommandTest.java @@ -11,6 +11,8 @@ package alluxio.cli.fsadmin.report; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -28,6 +30,10 @@ import alluxio.util.CommonUtils; import alluxio.wire.BlockMasterInfo; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; + import org.hamcrest.collection.IsIterableContainingInOrder; import org.junit.After; import org.junit.Assert; @@ -38,11 +44,7 @@ import java.io.IOException; import java.io.PrintStream; import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; public class SummaryCommandTest { @@ -157,7 +159,7 @@ public void ZkHaSummary() throws IOException { " [zookeeper_hostname3]:2181", " Raft-based Journal: false")); summaryCommand.run(); - checkIfOutputValid(sConf.getString(PropertyKey.USER_DATE_FORMAT_PATTERN), zkHAPattern); + checkIfOutputValid(sConf.getString(PropertyKey.USER_DATE_FORMAT_PATTERN), "zk"); } @Test @@ -174,43 +176,70 @@ public void RaftHaSummary() throws IOException { " [raftJournal_hostname3]:19200")); summaryCommand.run(); - checkIfOutputValid(sConf.getString(PropertyKey.USER_DATE_FORMAT_PATTERN), raftHaPattern); + checkIfOutputValid(sConf.getString(PropertyKey.USER_DATE_FORMAT_PATTERN), "raft"); } /** * Checks if the output is expected. */ - private void checkIfOutputValid(String dateFormatPattern, List HAPattern) { + private void checkIfOutputValid(String dateFormatPattern, String HAPattern) + throws JsonProcessingException { String output = new String(mOutputStream.toByteArray(), StandardCharsets.UTF_8); + ObjectMapper mapper = new ObjectMapper(); + JsonNode jsonNode = mapper.readTree(output); + + // check master + String versionStr = String.format("%s", RuntimeConstants.VERSION); + assertEquals("testAddress", jsonNode.get("masterAddress").asText()); + assertEquals("10000", jsonNode.get("masterVersions").get(0).get("port").asText()); + assertEquals(versionStr, jsonNode.get("masterVersions").get(0).get("version").asText()); + assertEquals("hostname1", jsonNode.get("masterVersions").get(0).get("host").asText()); + assertEquals("Primary", jsonNode.get("masterVersions").get(0).get("state").asText()); + assertEquals("10001", jsonNode.get("masterVersions").get(1).get("port").asText()); + assertEquals(versionStr, jsonNode.get("masterVersions").get(1).get("version").asText()); + assertEquals("hostname2", jsonNode.get("masterVersions").get(1).get("host").asText()); + assertEquals("Standby", jsonNode.get("masterVersions").get(1).get("state").asText()); + assertEquals("10002", jsonNode.get("masterVersions").get(2).get("port").asText()); + assertEquals(versionStr, jsonNode.get("masterVersions").get(2).get("version").asText()); + assertEquals("hostname3", jsonNode.get("masterVersions").get(2).get("host").asText()); + assertEquals("Standby", jsonNode.get("masterVersions").get(2).get("state").asText()); + + // check cluster summary // Skip checking startTime which relies on system time zone String startTime = CommonUtils.convertMsToDate(1131242343122L, dateFormatPattern); - List expectedOutput = new ArrayList<>(Arrays.asList("Alluxio cluster summary: ", - " Master Address: testAddress", - " Web Port: 1231", - " Rpc Port: 8462", - " Started: " + startTime, - " Uptime: 143 day(s), 15 hour(s), 53 minute(s), and 32 second(s)", - " Version: testVersion", - " Safe Mode: false")); - expectedOutput.addAll(HAPattern); - String versionStr = String.format("%-32s", RuntimeConstants.VERSION); - expectedOutput.addAll(new ArrayList<>(Arrays.asList( - " Master Address State Version ", - " hostname1:10000 Primary " + versionStr, - " hostname2:10001 Standby " + versionStr, - " hostname3:10002 Standby " + versionStr, - " Live Workers: 12", - " Lost Workers: 4", - " Total Capacity: 1309.92KB", - " Tier: MEM Size: 1309.92KB", - " Tier: DOM Size: 230.96KB", - " Tier: RAM Size: 22.57KB", - " Used Capacity: 60.97KB", - " Tier: MEM Size: 60.97KB", - " Tier: DOM Size: 72.50KB", - " Tier: RAM Size: 6.10KB", - " Free Capacity: 1248.94KB"))); - List testOutput = Arrays.asList(output.split("\n")); - Assert.assertThat(testOutput, IsIterableContainingInOrder.contains(expectedOutput.toArray())); + assertEquals("1231", jsonNode.get("webPort").asText()); + assertEquals("8462", jsonNode.get("rpcPort").asText()); + assertEquals("testVersion", jsonNode.get("version").asText()); + assertEquals(startTime, jsonNode.get("started").asText()); + assertEquals("143 day(s), 15 hour(s), 53 minute(s), and 32 second(s)", jsonNode.get("uptime").asText()); + assertEquals("false", jsonNode.get("safeMode").asText()); + + // check zookeeper and raft + if (Objects.equals(HAPattern, "zk")) { + assertEquals("true", jsonNode.get("useZookeeper").asText()); + assertEquals("false", jsonNode.get("useRaftJournal").asText()); + assertEquals("[zookeeper_hostname1]:2181", jsonNode.get("zookeeperAddress").get(0).asText()); + assertEquals("[zookeeper_hostname2]:2181", jsonNode.get("zookeeperAddress").get(1).asText()); + assertEquals("[zookeeper_hostname3]:2181", jsonNode.get("zookeeperAddress").get(2).asText()); + } else if (Objects.equals(HAPattern, "raft")) { + assertEquals("false", jsonNode.get("useZookeeper").asText()); + assertEquals("true", jsonNode.get("useRaftJournal").asText()); + assertEquals("[raftJournal_hostname1]:19200", jsonNode.get("raftJournalAddress").get(0).asText()); + assertEquals("[raftJournal_hostname2]:19200", jsonNode.get("raftJournalAddress").get(1).asText()); + assertEquals("[raftJournal_hostname3]:19200", jsonNode.get("raftJournalAddress").get(2).asText()); + } else { + fail("HAPattern is neither zk nor raft"); + } + + // check worker + assertEquals("12", jsonNode.get("liveWorkers").asText()); + assertEquals("4", jsonNode.get("lostWorkers").asText()); + assertEquals("1248.94KB", jsonNode.get("freeCapacity").asText()); + assertEquals("230.96KB", jsonNode.get("totalCapacityOnTiers").get("DOM").asText()); + assertEquals("1309.92KB", jsonNode.get("totalCapacityOnTiers").get("MEM").asText()); + assertEquals("22.57KB", jsonNode.get("totalCapacityOnTiers").get("RAM").asText()); + assertEquals("72.50KB", jsonNode.get("usedCapacityOnTiers").get("DOM").asText()); + assertEquals("60.97KB", jsonNode.get("usedCapacityOnTiers").get("MEM").asText()); + assertEquals("6.10KB", jsonNode.get("usedCapacityOnTiers").get("RAM").asText()); } } From 3c20ea1073f86189c0aef094a8b9bb6b2f8348e3 Mon Sep 17 00:00:00 2001 From: twalluxio Date: Tue, 5 Sep 2023 12:12:44 +0800 Subject: [PATCH 18/21] checkstyle fix --- .../fsadmin/report/MetricsCommandTest.java | 1 - .../fsadmin/report/SummaryCommandTest.java | 44 ++++++++----------- 2 files changed, 18 insertions(+), 27 deletions(-) diff --git a/dora/shell/src/test/java/alluxio/cli/fsadmin/report/MetricsCommandTest.java b/dora/shell/src/test/java/alluxio/cli/fsadmin/report/MetricsCommandTest.java index d2f59ba5d999..cc8d7fed0696 100644 --- a/dora/shell/src/test/java/alluxio/cli/fsadmin/report/MetricsCommandTest.java +++ b/dora/shell/src/test/java/alluxio/cli/fsadmin/report/MetricsCommandTest.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; - import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/dora/shell/src/test/java/alluxio/cli/fsadmin/report/SummaryCommandTest.java b/dora/shell/src/test/java/alluxio/cli/fsadmin/report/SummaryCommandTest.java index b5b685a5c890..93f06d0ae030 100644 --- a/dora/shell/src/test/java/alluxio/cli/fsadmin/report/SummaryCommandTest.java +++ b/dora/shell/src/test/java/alluxio/cli/fsadmin/report/SummaryCommandTest.java @@ -33,10 +33,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; - -import org.hamcrest.collection.IsIterableContainingInOrder; import org.junit.After; -import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -44,7 +41,10 @@ import java.io.IOException; import java.io.PrintStream; import java.nio.charset.StandardCharsets; -import java.util.*; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; public class SummaryCommandTest { @@ -151,13 +151,6 @@ public void ZkHaSummary() throws IOException { prepareZKHADependencies(); SummaryCommand summaryCommand = new SummaryCommand(mMetaMasterClient, mBlockMasterClient, sConf.getString(PropertyKey.USER_DATE_FORMAT_PATTERN), mPrintStream); - ArrayList zkHAPattern = new ArrayList<>(Arrays.asList( - " Zookeeper Enabled: true", - " Zookeeper Addresses: ", - " [zookeeper_hostname1]:2181", - " [zookeeper_hostname2]:2181", - " [zookeeper_hostname3]:2181", - " Raft-based Journal: false")); summaryCommand.run(); checkIfOutputValid(sConf.getString(PropertyKey.USER_DATE_FORMAT_PATTERN), "zk"); } @@ -167,15 +160,7 @@ public void RaftHaSummary() throws IOException { prepareRaftHaDependencies(); SummaryCommand summaryCommand = new SummaryCommand(mMetaMasterClient, mBlockMasterClient, sConf.getString(PropertyKey.USER_DATE_FORMAT_PATTERN), mPrintStream); - ArrayList raftHaPattern = new ArrayList<>(Arrays.asList( - " Zookeeper Enabled: false", - " Raft-based Journal: true", - " Raft Journal Addresses: ", - " [raftJournal_hostname1]:19200", - " [raftJournal_hostname2]:19200", - " [raftJournal_hostname3]:19200")); summaryCommand.run(); - checkIfOutputValid(sConf.getString(PropertyKey.USER_DATE_FORMAT_PATTERN), "raft"); } @@ -211,22 +196,29 @@ private void checkIfOutputValid(String dateFormatPattern, String HAPattern) assertEquals("8462", jsonNode.get("rpcPort").asText()); assertEquals("testVersion", jsonNode.get("version").asText()); assertEquals(startTime, jsonNode.get("started").asText()); - assertEquals("143 day(s), 15 hour(s), 53 minute(s), and 32 second(s)", jsonNode.get("uptime").asText()); + assertEquals("143 day(s), 15 hour(s), 53 minute(s), and 32 second(s)", + jsonNode.get("uptime").asText()); assertEquals("false", jsonNode.get("safeMode").asText()); // check zookeeper and raft if (Objects.equals(HAPattern, "zk")) { assertEquals("true", jsonNode.get("useZookeeper").asText()); assertEquals("false", jsonNode.get("useRaftJournal").asText()); - assertEquals("[zookeeper_hostname1]:2181", jsonNode.get("zookeeperAddress").get(0).asText()); - assertEquals("[zookeeper_hostname2]:2181", jsonNode.get("zookeeperAddress").get(1).asText()); - assertEquals("[zookeeper_hostname3]:2181", jsonNode.get("zookeeperAddress").get(2).asText()); + assertEquals("[zookeeper_hostname1]:2181", + jsonNode.get("zookeeperAddress").get(0).asText()); + assertEquals("[zookeeper_hostname2]:2181", + jsonNode.get("zookeeperAddress").get(1).asText()); + assertEquals("[zookeeper_hostname3]:2181", + jsonNode.get("zookeeperAddress").get(2).asText()); } else if (Objects.equals(HAPattern, "raft")) { assertEquals("false", jsonNode.get("useZookeeper").asText()); assertEquals("true", jsonNode.get("useRaftJournal").asText()); - assertEquals("[raftJournal_hostname1]:19200", jsonNode.get("raftJournalAddress").get(0).asText()); - assertEquals("[raftJournal_hostname2]:19200", jsonNode.get("raftJournalAddress").get(1).asText()); - assertEquals("[raftJournal_hostname3]:19200", jsonNode.get("raftJournalAddress").get(2).asText()); + assertEquals("[raftJournal_hostname1]:19200", + jsonNode.get("raftJournalAddress").get(0).asText()); + assertEquals("[raftJournal_hostname2]:19200", + jsonNode.get("raftJournalAddress").get(1).asText()); + assertEquals("[raftJournal_hostname3]:19200", + jsonNode.get("raftJournalAddress").get(2).asText()); } else { fail("HAPattern is neither zk nor raft"); } From fb6f842175c6610a6c76ffe8702826dedab9eaab Mon Sep 17 00:00:00 2001 From: twalluxio Date: Tue, 5 Sep 2023 13:56:47 +0800 Subject: [PATCH 19/21] re-submit unit test changes --- .../fsadmin/report/SummaryCommandTest.java | 34 +++++++------------ 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/dora/shell/src/test/java/alluxio/cli/fsadmin/report/SummaryCommandTest.java b/dora/shell/src/test/java/alluxio/cli/fsadmin/report/SummaryCommandTest.java index 93f06d0ae030..33933e63dfa8 100644 --- a/dora/shell/src/test/java/alluxio/cli/fsadmin/report/SummaryCommandTest.java +++ b/dora/shell/src/test/java/alluxio/cli/fsadmin/report/SummaryCommandTest.java @@ -97,7 +97,13 @@ public void prepareBaseDependencies() throws IOException { mPrintStream = new PrintStream(mOutputStream, true, "utf-8"); } - void prepareZKHADependencies() throws IOException { + @After + public void after() { + mPrintStream.close(); + } + + @Test + public void ZkHaSummary() throws IOException { MasterVersion primaryVersion = MasterVersion.newBuilder() .setVersion(RuntimeConstants.VERSION).setState("Primary").setAddresses( NetAddress.newBuilder().setHost("hostname1").setRpcPort(10000).build() @@ -117,9 +123,14 @@ void prepareZKHADependencies() throws IOException { .setRaftJournal(false) .build(); when(mMetaMasterClient.getMasterInfo(any())).thenReturn(mMasterInfo); + SummaryCommand summaryCommand = new SummaryCommand(mMetaMasterClient, + mBlockMasterClient, sConf.getString(PropertyKey.USER_DATE_FORMAT_PATTERN), mPrintStream); + summaryCommand.run(); + checkIfOutputValid(sConf.getString(PropertyKey.USER_DATE_FORMAT_PATTERN), "zk"); } - void prepareRaftHaDependencies() throws IOException { + @Test + public void RaftHaSummary() throws IOException { MasterVersion primaryVersion = MasterVersion.newBuilder() .setVersion(RuntimeConstants.VERSION).setState("Primary").setAddresses( NetAddress.newBuilder().setHost("hostname1").setRpcPort(10000).build() @@ -139,25 +150,6 @@ void prepareRaftHaDependencies() throws IOException { .addAllMasterVersions(Arrays.asList(primaryVersion, standby1Version, standby2Version)) .build(); when(mMetaMasterClient.getMasterInfo(any())).thenReturn(mMasterInfo); - } - - @After - public void after() { - mPrintStream.close(); - } - - @Test - public void ZkHaSummary() throws IOException { - prepareZKHADependencies(); - SummaryCommand summaryCommand = new SummaryCommand(mMetaMasterClient, - mBlockMasterClient, sConf.getString(PropertyKey.USER_DATE_FORMAT_PATTERN), mPrintStream); - summaryCommand.run(); - checkIfOutputValid(sConf.getString(PropertyKey.USER_DATE_FORMAT_PATTERN), "zk"); - } - - @Test - public void RaftHaSummary() throws IOException { - prepareRaftHaDependencies(); SummaryCommand summaryCommand = new SummaryCommand(mMetaMasterClient, mBlockMasterClient, sConf.getString(PropertyKey.USER_DATE_FORMAT_PATTERN), mPrintStream); summaryCommand.run(); From 9d303ee907f9ace7ad6a664f02aed245482012f2 Mon Sep 17 00:00:00 2001 From: twalluxio Date: Thu, 14 Sep 2023 10:42:23 +0800 Subject: [PATCH 20/21] fix on reviews --- dora/shell/pom.xml | 2 +- .../report/JobServiceMetricsCommand.java | 13 +++++++++++-- .../cli/fsadmin/report/JobServiceOutput.java | 17 +++++++++-------- .../cli/fsadmin/report/MetricsOutput.java | 3 ++- .../cli/fsadmin/report/SummaryCommand.java | 2 +- .../cli/fsadmin/report/MetricsCommandTest.java | 4 ++-- 6 files changed, 26 insertions(+), 15 deletions(-) diff --git a/dora/shell/pom.xml b/dora/shell/pom.xml index ce5144fa7824..e60c22d97472 100644 --- a/dora/shell/pom.xml +++ b/dora/shell/pom.xml @@ -42,7 +42,7 @@ com.google.protobuf protobuf-java-util - 3.18.1 + ${protobuf.version} commons-cli diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java index a817332143bb..5a97801d044b 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceMetricsCommand.java @@ -12,6 +12,9 @@ package alluxio.cli.fsadmin.report; import alluxio.client.job.JobMasterClient; +import alluxio.grpc.JobMasterStatus; +import alluxio.job.wire.JobServiceSummary; +import alluxio.job.wire.JobWorkerHealth; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -23,6 +26,7 @@ import java.time.ZoneId; import java.time.format.DateTimeFormatter; import java.time.format.FormatStyle; +import java.util.List; import java.util.Locale; /** @@ -30,10 +34,10 @@ */ public class JobServiceMetricsCommand { + private static final Logger LOG = LoggerFactory.getLogger(JobServiceMetricsCommand.class); private final JobMasterClient mJobMasterClient; private final PrintStream mPrintStream; private final String mDateFormatPattern; - private static final Logger LOG = LoggerFactory.getLogger(JobServiceMetricsCommand.class); /** * Creates a new instance of {@link JobServiceMetricsCommand}. @@ -60,7 +64,12 @@ public JobServiceMetricsCommand(JobMasterClient JobMasterClient, PrintStream pri */ public int run() throws IOException { ObjectMapper objectMapper = new ObjectMapper(); - JobServiceOutput jobServiceInfo = new JobServiceOutput(mJobMasterClient, mDateFormatPattern); + List allMasterStatus = mJobMasterClient.getAllMasterStatus(); + List allWorkerHealth = mJobMasterClient.getAllWorkerHealth(); + JobServiceSummary jobServiceSummary = mJobMasterClient.getJobServiceSummary(); + + JobServiceOutput jobServiceInfo = new JobServiceOutput(allMasterStatus, allWorkerHealth, + jobServiceSummary, mDateFormatPattern); try { String json = objectMapper.writeValueAsString(jobServiceInfo); mPrintStream.println(json); diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceOutput.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceOutput.java index 6f3adfc16956..8b0c4cc14a7e 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceOutput.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/JobServiceOutput.java @@ -11,7 +11,6 @@ package alluxio.cli.fsadmin.report; -import alluxio.client.job.JobMasterClient; import alluxio.grpc.JobMasterStatus; import alluxio.job.wire.JobInfo; import alluxio.job.wire.JobServiceSummary; @@ -19,7 +18,6 @@ import alluxio.job.wire.StatusSummary; import alluxio.util.CommonUtils; -import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -251,26 +249,29 @@ public void setStatus(String status) { /** * Creates a new instance of {@link JobServiceOutput}. * - * @param jobMasterClient job service info to parse from + * @param allMasterStatus status of all masters + * @param allWorkerHealth health info of all workers + * @param jobServiceSummary summary of job service * @param dateFormat specify the pattern of dates */ - public JobServiceOutput(JobMasterClient jobMasterClient, String dateFormat) throws IOException { + public JobServiceOutput(List allMasterStatus, + List allWorkerHealth, + JobServiceSummary jobServiceSummary, + String dateFormat) { mMasterStatus = new ArrayList<>(); - for (JobMasterStatus masterStatus : jobMasterClient.getAllMasterStatus()) { + for (JobMasterStatus masterStatus : allMasterStatus) { mMasterStatus.add(new SerializableJobMasterStatus(masterStatus, dateFormat)); } mWorkerHealth = new ArrayList<>(); - for (JobWorkerHealth workerHealth : jobMasterClient.getAllWorkerHealth()) { + for (JobWorkerHealth workerHealth : allWorkerHealth) { mWorkerHealth.add(new SerializableWorkerHealth(workerHealth)); } - JobServiceSummary jobServiceSummary = jobMasterClient.getJobServiceSummary(); mStatusSummary = new ArrayList<>(); for (StatusSummary statusSummary : jobServiceSummary.getSummaryPerStatus()) { mStatusSummary.add(new SerializableStatusSummary(statusSummary)); } - mRecentModifiedJobs = new ArrayList<>(); mRecentFailedJobs = new ArrayList<>(); mLongestRunningJobs = new ArrayList<>(); diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java index d2a05db9fc86..b1d0a29ee53d 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java @@ -35,6 +35,7 @@ public class MetricsOutput { private List mMetricsInfo; private class SerializableMetricInfo { + private static final String PER_MINUTE = "/min"; private String mKey; private String mType; private String mValue; @@ -50,7 +51,7 @@ public SerializableMetricInfo(Map.Entry entry) { // Bytes long can be transformed to human-readable format mValue = FormatUtils.getSizeFromBytes((long) doubleValue); if (mKey.contains(THROUGHPUT_METRIC_IDENTIFIER)) { - mValue = mValue + "/MIN"; + mValue = mValue + PER_MINUTE; } } else if (DoubleMath.isMathematicalInteger(doubleValue)) { mValue = DECIMAL_FORMAT.format((long) doubleValue); diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java index 4787ab98056f..86a094ab7383 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/SummaryCommand.java @@ -33,11 +33,11 @@ * Prints Alluxio cluster summarized information. */ public class SummaryCommand { + private static final Logger LOG = LoggerFactory.getLogger(SummaryCommand.class); private final MetaMasterClient mMetaMasterClient; private final BlockMasterClient mBlockMasterClient; private final PrintStream mPrintStream; private final String mDateFormatPattern; - private static final Logger LOG = LoggerFactory.getLogger(JobServiceMetricsCommand.class); /** * Creates a new instance of {@link SummaryCommand}. diff --git a/dora/shell/src/test/java/alluxio/cli/fsadmin/report/MetricsCommandTest.java b/dora/shell/src/test/java/alluxio/cli/fsadmin/report/MetricsCommandTest.java index cc8d7fed0696..fa0ae24e115b 100644 --- a/dora/shell/src/test/java/alluxio/cli/fsadmin/report/MetricsCommandTest.java +++ b/dora/shell/src/test/java/alluxio/cli/fsadmin/report/MetricsCommandTest.java @@ -61,7 +61,7 @@ public void after() { public void metrics() throws IOException { MetricsCommand metricsCommand = new MetricsCommand(mMetricsMasterClient, mPrintStream); metricsCommand.run(); - checkIfOutputValid(); + checkOutput(); } /** @@ -119,7 +119,7 @@ private Map generateMetricsMap() { /** * Checks if the output is expected. */ - private void checkIfOutputValid() throws JsonProcessingException { + private void checkOutput() throws JsonProcessingException { String output = new String(mOutputStream.toByteArray(), StandardCharsets.UTF_8); ObjectMapper mapper = new ObjectMapper(); JsonNode metricsInfo = mapper.readTree(output).get("metricsInfo"); From f0bdeb099d011bff73faaa51fa9f0aced13ea993 Mon Sep 17 00:00:00 2001 From: twalluxio Date: Thu, 14 Sep 2023 11:10:59 +0800 Subject: [PATCH 21/21] fix bug --- .../src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java index b1d0a29ee53d..5d1adfd072ee 100644 --- a/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java +++ b/dora/shell/src/main/java/alluxio/cli/fsadmin/report/MetricsOutput.java @@ -35,7 +35,7 @@ public class MetricsOutput { private List mMetricsInfo; private class SerializableMetricInfo { - private static final String PER_MINUTE = "/min"; + private static final String PER_MINUTE = "/MIN"; private String mKey; private String mType; private String mValue;