HBASE-23994:Add WebUI to Canary

apache · Mar 16, 2020 · 2e1cf6a · 2e1cf6a
1 parent f66cbe1
commit 2e1cf6a
Show file tree

Hide file tree

Showing 7 changed files with 414 additions and 0 deletions.
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -1464,6 +1464,12 @@ public enum OperationStatusCode {
 
   public static final String HBASE_CANARY_READ_RAW_SCAN_KEY = "hbase.canary.read.raw.enabled";
 
+  public static final String HBASE_CANARY_INFO_PORT = "hbase.canary.info.port";
+
+  public static final int DEFAULT_CANARY_INFOPORT = 16050;
+
+  public static final String HBASE_CANARY_INFO_BINDADDRESS = "hbase.canary.info.bindAddress";
+
   /**
    * Configuration keys for programmatic JAAS configuration for secured ZK interaction
    */

diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/tool/CanaryStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/tool/CanaryStatusTmpl.jamon
@@ -0,0 +1,156 @@
+
+<%doc>
+
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+</%doc>
+<%args>
+RegionStdOutSink sink;
+</%args>
+<%import>
+java.util.Map;
+java.util.concurrent.atomic.LongAdder;
+org.apache.hadoop.hbase.ServerName;
+org.apache.hadoop.hbase.tool.CanaryTool.RegionStdOutSink;
+</%import>
+
+<!--[if IE]>
+<!DOCTYPE html>
+<![endif]-->
+<?xml version="1.0" encoding="UTF-8" ?>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <title>Canary</title>
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta name="description" content="">
+    <link href="/static/css/bootstrap.min.css" rel="stylesheet">
+    <link href="/static/css/bootstrap-theme.min.css" rel="stylesheet">
+    <link href="/static/css/hbase.css" rel="stylesheet">
+  </head>
+
+  <body>
+
+    <div class="navbar  navbar-fixed-top navbar-default">
+        <div class="container">
+            <div class="navbar-header">
+                <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+                    <span class="icon-bar"></span>
+                    <span class="icon-bar"></span>
+                    <span class="icon-bar"></span>
+                </button>
+                <a class="navbar-brand" href="/canary-status"><img src="/static/hbase_logo_small.png" alt="HBase Logo"/></a>
+            </div>
+        </div>
+    </div>
+
+	<div class="container">
+		<section>
+		<h2>Failed Servers</h2>
+        <%java>
+           Map<ServerName, LongAdder> perServerFailuresCount = sink.getPerServerFailuresCount();
+        </%java>
+        <table class="table table-striped">
+          <tr>
+              <th>Server</th>
+              <th>Failures Count</th>
+          </tr>
+          <%if (perServerFailuresCount != null && perServerFailuresCount.size() > 0)%>
+          <%for Map.Entry<ServerName, LongAdder> entry : perServerFailuresCount.entrySet() %>
+          <tr>
+              <td><& serverNameLink ; serverName = entry.getKey() &></td>
+              <td><% entry.getValue() %></td>
+          </tr>
+          </%for>
+          </%if>
+          <tr><td>Total Failed Servers: <% (perServerFailuresCount != null) ? perServerFailuresCount.size() : 0 %></td></tr>
+        </table>
+		</section>
+		<section>
+			<h2>Failed Tables</h2>
+      			<%java>
+      			   Map<String, LongAdder> perTableFailuresCount = sink.getPerTableFailuresCount();
+      			</%java>
+      			<table class="table table-striped">
+      				<tr>
+      				    <th>Table</th>
+      				    <th>Failures Count</th>
+      				</tr>
+      				<%if (perTableFailuresCount != null && perTableFailuresCount.size() > 0)%>
+      				<%for Map.Entry<String, LongAdder> entry : perTableFailuresCount.entrySet()%>
+      				<tr>
+      				    <td><% entry.getKey() %></td>
+      				    <td><% entry.getValue() %></td>
+      				</tr>
+      				</%for>
+      				</%if>
+      				<tr><td>Total Failed Tables: <% (perTableFailuresCount != null) ? perTableFailuresCount.size() : 0 %></td></tr>
+      			</table>
+		</section>
+
+        <section>
+            <h2>Software Attributes</h2>
+            <table id="attributes_table" class="table table-striped">
+                <tr>
+                    <th>Attribute Name</th>
+                    <th>Value</th>
+                    <th>Description</th>
+                </tr>
+                <tr>
+                    <td>HBase Version</td>
+                    <td><% org.apache.hadoop.hbase.util.VersionInfo.getVersion() %>, r<% org.apache.hadoop.hbase.util.VersionInfo.getRevision() %></td><td>HBase version and revision</td>
+                </tr>
+                <tr>
+                    <td>HBase Compiled</td>
+                    <td><% org.apache.hadoop.hbase.util.VersionInfo.getDate() %>, <% org.apache.hadoop.hbase.util.VersionInfo.getUser() %></td>
+                    <td>When HBase version was compiled and by whom</td>
+                </tr>
+                <tr>
+                    <td>Hadoop Version</td>
+                    <td><% org.apache.hadoop.util.VersionInfo.getVersion() %>, r<% org.apache.hadoop.util.VersionInfo.getRevision() %></td>
+                    <td>Hadoop version and revision</td>
+                </tr>
+                <tr>
+                    <td>Hadoop Compiled</td>
+                    <td><% org.apache.hadoop.util.VersionInfo.getDate() %>, <% org.apache.hadoop.util.VersionInfo.getUser() %></td>
+                    <td>When Hadoop version was compiled and by whom</td>
+                </tr>
+            </table>
+        </section>
+        </div>
+    </div> <!-- /container -->
+
+    <script src="/static/js/jquery.min.js" type="text/javascript"></script>
+    <script src="/static/js/bootstrap.min.js" type="text/javascript"></script>
+    <script src="/static/js/tab.js" type="text/javascript"></script>
+  </body>
+</html>
+
+<%def serverNameLink>
+        <%args>
+        ServerName serverName;
+        </%args>
+        <%java>
+        int infoPort = serverName.getPort() + 1;
+        String url = "//" + serverName.getHostname() + ":" + infoPort + "/";
+        </%java>
+
+        <%if (infoPort > 0) %>
+            <a href="<% url %>"><% serverName.getServerName() %></a>
+        <%else>
+            <% serverName.getServerName() %>
+        </%if>
+</%def>
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/CanaryStatusServlet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/CanaryStatusServlet.java
@@ -0,0 +1,50 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.tool;
+
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.tmpl.tool.CanaryStatusTmpl;
+import org.apache.yetus.audience.InterfaceAudience;
+
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.LongAdder;
+
+@InterfaceAudience.Private
+public class CanaryStatusServlet extends HttpServlet {
+  @Override
+  protected void doGet(HttpServletRequest req, HttpServletResponse resp)
+    throws ServletException, IOException {
+    CanaryTool.RegionStdOutSink sink =
+      (CanaryTool.RegionStdOutSink) getServletContext().getAttribute(
+        "sink");
+    assert sink != null : "No tool in context!";
+
+    resp.setContentType("text/html");
+
+    CanaryStatusTmpl tmpl = new CanaryStatusTmpl();
+    tmpl.render(resp.getWriter(), sink);
+  }
+
+}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/CanaryTool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/tool/CanaryTool.java
@@ -19,11 +19,14 @@
 
 package org.apache.hadoop.hbase.tool;
 
+import static org.apache.hadoop.hbase.HConstants.DEFAULT_CANARY_INFOPORT;
 import static org.apache.hadoop.hbase.HConstants.DEFAULT_ZOOKEEPER_ZNODE_PARENT;
+import static org.apache.hadoop.hbase.HConstants.HBASE_CANARY_INFO_BINDADDRESS;
 import static org.apache.hadoop.hbase.HConstants.ZOOKEEPER_ZNODE_PARENT;
 
 import java.io.Closeable;
 import java.io.IOException;
+import java.net.BindException;
 import java.net.InetSocketAddress;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -81,6 +84,7 @@
 import org.apache.hadoop.hbase.client.TableDescriptor;
 import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
 import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
+import org.apache.hadoop.hbase.http.InfoServer;
 import org.apache.hadoop.hbase.tool.CanaryTool.RegionTask.TaskType;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
@@ -123,6 +127,31 @@
 @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.TOOLS)
 public class CanaryTool implements Tool, Canary {
 
+
+  private void putUpWebUI() throws IOException {
+    if (zookeeperMode) {
+      LOG.info("WebUI is not supported in Zookeeper mode");
+    } else if (regionServerMode) {
+      LOG.info("WebUI is not supported in RegionServer mode");
+    } else {
+      Configuration conf = new Configuration();
+      int port = conf.getInt(HConstants.HBASE_CANARY_INFO_PORT, DEFAULT_CANARY_INFOPORT);
+      // -1 is for disabling info server
+      if (port < 0) return;
+      String addr = conf.get(HBASE_CANARY_INFO_BINDADDRESS, "0.0.0.0");
+      try {
+        InfoServer infoServer = new InfoServer("canary", addr, port, false, conf);
+        infoServer.addUnprivilegedServlet("canary", "/canary-status", CanaryStatusServlet.class);
+        infoServer.setAttribute("sink", this.sink);
+        infoServer.start();
+        LOG.info("Bind Canary http info server to port: " + port);
+      } catch (BindException e) {
+        e.printStackTrace();
+        LOG.info("Failed binding Canary http info server to port: " + port);
+      }
+    }
+  }
+
   @Override
   public int checkRegions(String[] targets) throws Exception {
     String configuredReadTableTimeoutsStr = conf.get(HBASE_CANARY_REGION_READ_TABLE_TIMEOUT);
@@ -274,16 +303,50 @@ public static class RegionStdOutSink extends StdOutSink {
     private Map<String, LongAdder> perTableReadLatency = new HashMap<>();
     private LongAdder writeLatency = new LongAdder();
     private final Map<String, List<RegionTaskResult>> regionMap = new ConcurrentHashMap<>();
+    private Map<ServerName, LongAdder> perServerFailuresCount = new ConcurrentHashMap<>();
+    private Map<String, LongAdder> perTableFailuresCount = new ConcurrentHashMap<>();
+
+    public Map<ServerName, LongAdder> getPerServerFailuresCount() {
+      return perServerFailuresCount;
+    }
+
+    public Map<String, LongAdder> getPerTableFailuresCount() {
+      return perTableFailuresCount;
+    }
+
+    public void resetFailuresCountDetails() {
+      perServerFailuresCount.clear();
+      perTableFailuresCount.clear();
+    }
+
+    private void incFailuresCountDetails(ServerName serverName, RegionInfo region) {
+      perServerFailuresCount.compute(serverName, (server, count) -> {
+        if (count == null) {
+          count = new LongAdder();
+        }
+        count.increment();
+        return count;
+      });
+      perTableFailuresCount.compute(region.getTable().getNameAsString(), (tableName, count) -> {
+        if (count == null) {
+          count = new LongAdder();
+        }
+        count.increment();
+        return count;
+      });
+    }
 
     public void publishReadFailure(ServerName serverName, RegionInfo region, Exception e) {
       incReadFailureCount();
+      incFailuresCountDetails(serverName, region);
       LOG.error("Read from {} on serverName={} failed",
           region.getRegionNameAsString(), serverName, e);
     }
 
     public void publishReadFailure(ServerName serverName, RegionInfo region,
         ColumnFamilyDescriptor column, Exception e) {
       incReadFailureCount();
+      incFailuresCountDetails(serverName, region);
       LOG.error("Read from {} on serverName={}, columnFamily={} failed",
           region.getRegionNameAsString(), serverName,
           column.getNameAsString(), e);
@@ -304,12 +367,14 @@ public void publishReadTiming(ServerName serverName, RegionInfo region,
 
     public void publishWriteFailure(ServerName serverName, RegionInfo region, Exception e) {
       incWriteFailureCount();
+      incFailuresCountDetails(serverName, region);
       LOG.error("Write to {} on {} failed", region.getRegionNameAsString(), serverName, e);
     }
 
     public void publishWriteFailure(ServerName serverName, RegionInfo region,
         ColumnFamilyDescriptor column, Exception e) {
       incWriteFailureCount();
+      incFailuresCountDetails(serverName, region);
       LOG.error("Write to {} on {} {} failed", region.getRegionNameAsString(), serverName,
           column.getNameAsString(), e);
     }
@@ -908,6 +973,7 @@ public int run(String[] args) throws Exception {
       System.arraycopy(args, index, monitorTargets, 0, length);
     }
 
+    putUpWebUI();
     if (zookeeperMode) {
       return checkZooKeeper();
     } else if (regionServerMode) {
@@ -1352,6 +1418,7 @@ public void run() {
         try {
           List<Future<Void>> taskFutures = new LinkedList<>();
           RegionStdOutSink regionSink = this.getSink();
+          regionSink.resetFailuresCountDetails();
           if (this.targets != null && this.targets.length > 0) {
             String[] tables = generateMonitorTables(this.targets);
             // Check to see that each table name passed in the -readTableTimeouts argument is also

diff --git a/hbase-server/src/main/resources/hbase-webapps/canary/canary.jsp b/hbase-server/src/main/resources/hbase-webapps/canary/canary.jsp
@@ -0,0 +1,20 @@
+<%--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+--%>
+<meta HTTP-EQUIV="REFRESH" content="0;url=/canary-status"/>
diff --git a/hbase-server/src/main/resources/hbase-webapps/canary/index.html b/hbase-server/src/main/resources/hbase-webapps/canary/index.html
@@ -0,0 +1,20 @@
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<meta HTTP-EQUIV="REFRESH" content="0;url=/canary-status"/>