[Feature][ThreadPool]Add Web Page to display thread's stats

be/src/env/env.h

@@ -350,4 +350,4 @@ class RandomRWFile {
     virtual const std::string& filename() const = 0;
 };
 
-}
+}   // namespace doris

be/src/env/env_util.cpp

@@ -18,6 +18,7 @@
 #include "env/env_util.h"
 
 #include "env/env.h"
+#include "util/faststring.h"
 
 using std::shared_ptr;
 using std::string;
@@ -30,21 +31,73 @@ Status open_file_for_write(Env* env, const string& path, shared_ptr<WritableFile
     return open_file_for_write(WritableFileOptions(), env, path, file);
 }
 
-Status open_file_for_write(const WritableFileOptions& opts,
-                           Env *env, const string &path,
-                           shared_ptr<WritableFile> *file) {
+Status open_file_for_write(const WritableFileOptions& opts, Env* env, const string& path,
+                           shared_ptr<WritableFile>* file) {
     unique_ptr<WritableFile> w;
     RETURN_IF_ERROR(env->new_writable_file(opts, path, &w));
     file->reset(w.release());
     return Status::OK();
 }
 
-Status open_file_for_random(Env *env, const string &path, shared_ptr<RandomAccessFile> *file) {
+Status open_file_for_random(Env* env, const string& path, shared_ptr<RandomAccessFile>* file) {
     unique_ptr<RandomAccessFile> r;
     RETURN_IF_ERROR(env->new_random_access_file(path, &r));
     file->reset(r.release());
     return Status::OK();
 }
 
+static Status do_write_string_to_file(Env* env, const Slice& data, const std::string& fname,
+                                      bool should_sync) {
+    unique_ptr<WritableFile> file;
+    Status s = env->new_writable_file(fname, &file);
+    if (!s.ok()) {
+        return s;
+    }
+    s = file->append(data);
+    if (s.ok() && should_sync) {
+        s = file->sync();
+    }
+    if (s.ok()) {
+        s = file->close();
+    }
+    file.reset(); // Will auto-close if we did not close above
+    if (!s.ok()) {
+        RETURN_NOT_OK_STATUS_WITH_WARN(env->delete_file(fname),
+                                       "Failed to delete partially-written file " + fname);
+    }
+    return s;
+}
+
+Status write_string_to_file(Env* env, const Slice& data, const std::string& fname) {
+    return do_write_string_to_file(env, data, fname, false);
+}
+
+Status write_string_to_file_sync(Env* env, const Slice& data, const std::string& fname) {
+    return do_write_string_to_file(env, data, fname, true);
+}
+
+Status read_file_to_string(Env* env, const std::string& fname, faststring* data) {
+    data->clear();
+    unique_ptr<SequentialFile> file;
+    Status s = env->new_sequential_file(fname, &file);
+    if (!s.ok()) {
+        return s;
+    }
+    static const int kBufferSize = 8192;
+    unique_ptr<uint8_t[]> scratch(new uint8_t[kBufferSize]);
+    while (true) {
+        Slice fragment(scratch.get(), kBufferSize);
+        s = file->read(&fragment);
+        if (!s.ok()) {
+            break;
+        }
+        data->append(fragment.get_data(), fragment.get_size());
+        if (fragment.empty()) {
+            break;
+        }
+    }
+    return s;
+}
+
 } // namespace env_util
 } // namespace doris

be/src/env/env_util.h

@@ -20,6 +20,7 @@
 #include <string>
 
 #include "common/status.h"
+#include "env.h"
 
 namespace doris {
 
@@ -30,14 +31,22 @@ struct WritableFileOptions;
 
 namespace env_util {
 
-Status open_file_for_write(Env *env, const std::string& path, std::shared_ptr<WritableFile> *file);
+Status open_file_for_write(Env* env, const std::string& path, std::shared_ptr<WritableFile>* file);
 
-Status open_file_for_write(const WritableFileOptions& opts, Env *env,
-                           const std::string& path, std::shared_ptr<WritableFile> *file);
+Status open_file_for_write(const WritableFileOptions& opts, Env* env, const std::string& path,
+                           std::shared_ptr<WritableFile>* file);
 
-Status open_file_for_random(Env *env, const std::string& path,
-                            std::shared_ptr<RandomAccessFile> *file);
+Status open_file_for_random(Env* env, const std::string& path,
+                            std::shared_ptr<RandomAccessFile>* file);
+
+// A utility routine: write "data" to the named file.
+extern Status write_string_to_file(Env* env, const Slice& data, const std::string& fname);
+// Like above but also fsyncs the new file.
+
+extern Status write_string_to_file_sync(Env* env, const Slice& data, const std::string& fname);
+
+// A utility routine: read contents of named file into *data
+extern Status read_file_to_string(Env* env, const std::string& fname, faststring* data);
 
 } // namespace env_util
 } // namespace doris
-

be/src/exec/orc_scanner.cpp

@@ -245,16 +245,23 @@ Status ORCScanner::get_next(Tuple* tuple, MemPool* tuple_pool, bool* eof) {
                             } else {
                                 decimal_str = ((orc::Decimal128VectorBatch*) cvb)->values[_current_line_of_group].toString();
                             }
+
+                            int negative = decimal_str[0] == '-' ? 1 : 0;
+                            int decimal_scale_length = decimal_str.size() - negative;
 
                             std::string v;
-                            if (decimal_str.size() <= scale) {
+                            if (decimal_scale_length <= scale) {
                                 // decimal(5,2) : the integer of 0.01 is 1, so we should fill 0 befor integer
-                                v = "0.";
-                                int fill_zero = scale - decimal_str.size();
+                                v = std::string(negative ? "-0." : "0.");
+                                int fill_zero = scale - decimal_scale_length;
                                 while (fill_zero--) {
                                     v += "0";
                                 }
-                                v += decimal_str;
+                                if (negative) {
+                                    v += decimal_str.substr(1, decimal_str.length());
+                                } else {
+                                    v += decimal_str;
+                                }
                             } else {
                                 //Orc api will fill in 0 at the end, so size must greater than scale
                                 v = decimal_str.substr(0, decimal_str.size() - scale) + "." + decimal_str.substr(decimal_str.size() - scale);

be/src/http/default_path_handlers.cpp

@@ -18,20 +18,17 @@
 #include "http/default_path_handlers.h"
 
 #include <gperftools/malloc_extension.h>
-#include <sys/stat.h>
 
 #include <boost/algorithm/string.hpp>
 #include <boost/bind.hpp>
-#include <fstream>
 #include <sstream>
 
 #include "common/configbase.h"
-#include "common/logging.h"
 #include "http/web_page_handler.h"
 #include "runtime/mem_tracker.h"
 #include "util/debug_util.h"
-#include "util/logging.h"
 #include "util/pretty_printer.h"
+#include "util/thread.h"
 
 namespace doris {
 
@@ -104,10 +101,12 @@ void mem_usage_handler(MemTracker* mem_tracker, const WebPageHandler::ArgumentMa
 }
 
 void add_default_path_handlers(WebPageHandler* web_page_handler, MemTracker* process_mem_tracker) {
-    web_page_handler->register_page("/logs", "Logs", logs_handler, true /* is_on_nav_bar */);
+    // TODO(yingchun): logs_handler is not implemented yet, so not show it on navigate bar
+    web_page_handler->register_page("/logs", "Logs", logs_handler, false /* is_on_nav_bar */);
     web_page_handler->register_page("/varz", "Configs", config_handler, true /* is_on_nav_bar */);
     web_page_handler->register_page("/memz", "Memory",
         boost::bind<void>(&mem_usage_handler, process_mem_tracker, _1, _2), true /* is_on_nav_bar */);
+    start_thread_instrumentation(web_page_handler);
 }
 
 } // namespace doris

be/src/http/web_page_handler.cpp

@@ -36,6 +36,7 @@
 #include "util/debug_util.h"
 #include "util/disk_info.h"
 #include "util/mem_info.h"
+#include "util/mustache/mustache.h"
 
 using strings::Substitute;
 
@@ -44,18 +45,32 @@ namespace doris {
 static std::string s_html_content_type = "text/html";
 
 WebPageHandler::WebPageHandler(EvHttpServer* server) : _http_server(server) {
+    _www_path = std::string(getenv("DORIS_HOME")) + "/www/";
+
     // Make WebPageHandler to be static file handler, static files, e.g. css, png, will be handled by WebPageHandler.
     _http_server->register_static_file_handler(this);
 
-    PageHandlerCallback root_callback =
+    TemplatePageHandlerCallback root_callback =
             boost::bind<void>(boost::mem_fn(&WebPageHandler::root_handler), this, _1, _2);
-    register_page("/", "Home", root_callback, false /* is_on_nav_bar */);
+    register_template_page("/", "Home", root_callback, false /* is_on_nav_bar */);
 }
 
 WebPageHandler::~WebPageHandler() {
     STLDeleteValues(&_page_map);
 }
 
+void WebPageHandler::register_template_page(const std::string& path, const string& alias,
+                                            const TemplatePageHandlerCallback& callback, bool is_on_nav_bar) {
+    // Relative path which will be used to find .mustache file in _www_path
+    string render_path = (path == "/") ? "/home" : path;
+    auto wrapped_cb = [=](const ArgumentMap& args, std::stringstream* output) {
+        EasyJson ej;
+        callback(args, &ej);
+        render(render_path, ej, true /* is_styled */, output);
+    };
+    register_page(path, alias, wrapped_cb, is_on_nav_bar);
+}
+
 void WebPageHandler::register_page(const std::string& path, const string& alias,
                                    const PageHandlerCallback& callback, bool is_on_nav_bar) {
     boost::mutex::scoped_lock lock(_map_lock);
@@ -79,7 +94,7 @@ void WebPageHandler::handle(HttpRequest* req) {
 
     if (handler == nullptr) {
         // Try to handle static file request
-        do_file_response(std::string(getenv("DORIS_HOME")) + "/www/" + req->raw_path(), req);
+        do_file_response(_www_path + req->raw_path(), req);
         // Has replied in do_file_response, so we return here.
         return;
     }
@@ -90,24 +105,22 @@ void WebPageHandler::handle(HttpRequest* req) {
     bool use_style = (params.find("raw") == params.end());
 
     std::stringstream content;
-    // Append header    
-    if (use_style) {
-        bootstrap_page_header(&content); 
-    }
-
-    // Append content
     handler->callback()(params, &content);
 
-    // Append footer
+    std::string output;
     if (use_style) {
-        bootstrap_page_footer(&content);
+        std::stringstream oss;
+        render_main_template(content.str(), &oss);
+        output = oss.str();
+    } else {
+        output = content.str();
     }
 
     req->add_output_header(HttpHeaders::CONTENT_TYPE, s_html_content_type.c_str());
-    HttpChannel::send_reply(req, HttpStatus::OK, content.str());
+    HttpChannel::send_reply(req, HttpStatus::OK, output);
 }
 
-static const std::string PAGE_HEADER = R"(
+static const std::string kMainTemplate = R"(
 <!DOCTYPE html>
 <html>
   <head>
@@ -122,9 +135,6 @@ static const std::string PAGE_HEADER = R"(
     <link href='/doris.css' rel='stylesheet' />
   </head>
   <body>
-)";
-
-static const std::string NAVIGATION_BAR_PREFIX = R"(
     <nav class="navbar navbar-default">
       <div class="container-fluid">
         <div class="navbar-header">
@@ -134,49 +144,79 @@ static const std::string NAVIGATION_BAR_PREFIX = R"(
         </div>
         <div id="navbar" class="navbar-collapse collapse">
           <ul class="nav navbar-nav">
-)";
-
-static const std::string NAVIGATION_BAR_SUFFIX = R"(
+           {{#path_handlers}}
+            <li><a class="nav-link"href="{{path}}">{{alias}}</a></li>
+           {{/path_handlers}}
           </ul>
         </div><!--/.nav-collapse -->
       </div><!--/.container-fluid -->
     </nav>
-)";
-
-static const std::string PAGE_FOOTER = R"(
+      {{^static_pages_available}}
+      <div style="color: red">
+        <strong>Static pages not available. Make sure ${DORIS_HOME}/www/ exists and contains web static files.</strong>
+      </div>
+      {{/static_pages_available}}
+      {{{content}}}
+    </div>
+    {{#footer_html}}
+    <footer class="footer"><div class="container text-muted">
+      {{{.}}}
+    </div></footer>
+    {{/footer_html}}
   </body>
 </html>
 )";
 
-void WebPageHandler::bootstrap_page_header(std::stringstream* output) {
-    boost::mutex::scoped_lock lock(_map_lock);
-    (*output) << PAGE_HEADER;
-    (*output) << NAVIGATION_BAR_PREFIX;
-    for (auto& iter : _page_map) {
-        (*output) << "<li><a href=\"" << iter.first << "\">" << iter.first << "</a></li>";
+std::string WebPageHandler::mustache_partial_tag(const std::string& path) const {
+    return Substitute("{{> $0.mustache}}", path);
+}
+
+bool WebPageHandler::static_pages_available() const {
+    bool is_dir = false;
+    return Env::Default()->is_directory(_www_path, &is_dir).ok() && is_dir;
+}
+
+bool WebPageHandler::mustache_template_available(const std::string& path) const {
+    if (!static_pages_available()) {
+        return false;
     }
-    (*output) << NAVIGATION_BAR_SUFFIX;
+    return Env::Default()->path_exists(Substitute("$0/$1.mustache", _www_path, path)).ok();
 }
 
-void WebPageHandler::bootstrap_page_footer(std::stringstream* output) {
-    (*output) << PAGE_FOOTER;
+void WebPageHandler::render_main_template(const std::string& content, std::stringstream* output) {
+    static const std::string& footer = std::string("<pre>") + get_version_string(true) + std::string("</pre>");
+
+    EasyJson ej;
+    ej["static_pages_available"] = static_pages_available();
+    ej["content"] = content;
+    ej["footer_html"] = footer;
+    EasyJson path_handlers = ej.Set("path_handlers", EasyJson::kArray);
+    for (const auto& handler : _page_map) {
+        if (handler.second->is_on_nav_bar()) {
+            EasyJson path_handler = path_handlers.PushBack(EasyJson::kObject);
+            path_handler["path"] = handler.first;
+            path_handler["alias"] = handler.second->alias();
+        }
+    }
+    mustache::RenderTemplate(kMainTemplate, _www_path, ej.value(), output);
 }
 
-void WebPageHandler::root_handler(const ArgumentMap& args, std::stringstream* output) {
-    // _path_handler_lock already held by MongooseCallback
-    (*output) << "<h2>Version</h2>";
-    (*output) << "<pre>" << get_version_string(false) << "</pre>" << std::endl;
-    (*output) << "<h2>Hardware Info</h2>";
-    (*output) << "<pre>";
-    (*output) << CpuInfo::debug_string();
-    (*output) << MemInfo::debug_string();
-    (*output) << DiskInfo::debug_string();
-    (*output) << "</pre>";
-
-    (*output) << "<h2>Status Pages</h2>";
-    for (auto& iter : _page_map) {
-        (*output) << "<a href=\"" << iter.first << "\">" << iter.first << "</a><br/>";
+void WebPageHandler::render(const string& path, const EasyJson& ej, bool use_style,
+                            std::stringstream* output) {
+    if (mustache_template_available(path)) {
+        mustache::RenderTemplate(mustache_partial_tag(path), _www_path, ej.value(), output);
+    } else if (use_style) {
+        (*output) << "<pre>" << ej.ToString() << "</pre>";
+    } else {
+        (*output) << ej.ToString();
     }
 }
 
+void WebPageHandler::root_handler(const ArgumentMap& args, EasyJson* output) {
+    (*output)["version"] = get_version_string(false);
+    (*output)["cpuinfo"] = CpuInfo::debug_string();
+    (*output)["meminfo"] = MemInfo::debug_string();
+    (*output)["diskinfo"] = DiskInfo::debug_string();
+}
+
 } // namespace doris