From a1ce32b0dc65cf1b19eb74647882ba5a950bf510 Mon Sep 17 00:00:00 2001 From: Yingchun Lai <405403881@qq.com> Date: Tue, 16 Jun 2020 17:08:21 +0800 Subject: [PATCH 1/3] [webserver] Introduce mustache to simplify BE's website render cpp-mustache is a C++ implementation of a Mustache template engine with support for RapidJSON, and in order to simplify RapidJSON object building, we introduce class EasyJson from Apache Kudu. --- be/src/http/web_page_handler.cpp | 130 +++++---- be/src/http/web_page_handler.h | 34 ++- be/src/util/CMakeLists.txt | 2 + be/src/util/easy_json.cc | 209 ++++++++++++++ be/src/util/easy_json.h | 190 +++++++++++++ be/src/util/mustache/mustache.cc | 448 +++++++++++++++++++++++++++++++ be/src/util/mustache/mustache.h | 27 ++ be/test/util/CMakeLists.txt | 1 + be/test/util/easy_json-test.cpp | 110 ++++++++ webroot/be/home.mustache | 28 ++ 10 files changed, 1130 insertions(+), 49 deletions(-) create mode 100644 be/src/util/easy_json.cc create mode 100644 be/src/util/easy_json.h create mode 100644 be/src/util/mustache/mustache.cc create mode 100644 be/src/util/mustache/mustache.h create mode 100644 be/test/util/easy_json-test.cpp create mode 100644 webroot/be/home.mustache diff --git a/be/src/http/web_page_handler.cpp b/be/src/http/web_page_handler.cpp index abec965647a2c5..436494d12e46b2 100644 --- a/be/src/http/web_page_handler.cpp +++ b/be/src/http/web_page_handler.cpp @@ -19,6 +19,7 @@ #include #include +#include #include "common/config.h" #include "env/env.h" @@ -44,18 +45,32 @@ namespace doris { static std::string s_html_content_type = "text/html"; WebPageHandler::WebPageHandler(EvHttpServer* server) : _http_server(server) { + _www_path = std::string(getenv("DORIS_HOME")) + "/www/"; + // Make WebPageHandler to be static file handler, static files, e.g. css, png, will be handled by WebPageHandler. _http_server->register_static_file_handler(this); - PageHandlerCallback root_callback = + TemplatePageHandlerCallback root_callback = boost::bind(boost::mem_fn(&WebPageHandler::root_handler), this, _1, _2); - register_page("/", "Home", root_callback, false /* is_on_nav_bar */); + register_template_page("/", "Home", root_callback, false /* is_on_nav_bar */); } WebPageHandler::~WebPageHandler() { STLDeleteValues(&_page_map); } +void WebPageHandler::register_template_page(const std::string& path, const string& alias, + const TemplatePageHandlerCallback& callback, bool is_on_nav_bar) { + // Relative path which will be used to find .mustache file in _www_path + string render_path = (path == "/") ? "/home" : path; + auto wrapped_cb = [=](const ArgumentMap& args, std::stringstream* output) { + EasyJson ej; + callback(args, &ej); + render(render_path, ej, true /* is_styled */, output); + }; + register_page(path, alias, wrapped_cb, is_on_nav_bar); +} + void WebPageHandler::register_page(const std::string& path, const string& alias, const PageHandlerCallback& callback, bool is_on_nav_bar) { boost::mutex::scoped_lock lock(_map_lock); @@ -79,7 +94,7 @@ void WebPageHandler::handle(HttpRequest* req) { if (handler == nullptr) { // Try to handle static file request - do_file_response(std::string(getenv("DORIS_HOME")) + "/www/" + req->raw_path(), req); + do_file_response(_www_path + req->raw_path(), req); // Has replied in do_file_response, so we return here. return; } @@ -90,24 +105,22 @@ void WebPageHandler::handle(HttpRequest* req) { bool use_style = (params.find("raw") == params.end()); std::stringstream content; - // Append header - if (use_style) { - bootstrap_page_header(&content); - } - - // Append content handler->callback()(params, &content); - // Append footer + std::string output; if (use_style) { - bootstrap_page_footer(&content); + std::stringstream oss; + render_main_template(content.str(), &oss); + output = oss.str(); + } else { + output = content.str(); } req->add_output_header(HttpHeaders::CONTENT_TYPE, s_html_content_type.c_str()); - HttpChannel::send_reply(req, HttpStatus::OK, content.str()); + HttpChannel::send_reply(req, HttpStatus::OK, output); } -static const std::string PAGE_HEADER = R"( +static const std::string kMainTemplate = R"( @@ -122,9 +135,6 @@ static const std::string PAGE_HEADER = R"( -)"; - -static const std::string NAVIGATION_BAR_PREFIX = R"( -)"; - -static const std::string PAGE_FOOTER = R"( + {{^static_pages_available}} +
+ Static pages not available. Make sure ${DORIS_HOME}/www/ exists and contains web static files. +
+ {{/static_pages_available}} + {{{content}}} + + {{#footer_html}} +
+ {{{.}}} +
+ {{/footer_html}} )"; -void WebPageHandler::bootstrap_page_header(std::stringstream* output) { - boost::mutex::scoped_lock lock(_map_lock); - (*output) << PAGE_HEADER; - (*output) << NAVIGATION_BAR_PREFIX; - for (auto& iter : _page_map) { - (*output) << "
  • " << iter.first << "
    • "; +std::string WebPageHandler::mustache_partial_tag(const std::string& path) const { + return Substitute("{{> $0.mustache}}", path); +} + +bool WebPageHandler::static_pages_available() const { + bool is_dir = false; + return Env::Default()->is_directory(_www_path, &is_dir).ok() && is_dir; +} + +bool WebPageHandler::mustache_template_available(const std::string& path) const { + if (!static_pages_available()) { + return false; } - (*output) << NAVIGATION_BAR_SUFFIX; + return Env::Default()->path_exists(Substitute("$0/$1.mustache", _www_path, path)).ok(); } -void WebPageHandler::bootstrap_page_footer(std::stringstream* output) { - (*output) << PAGE_FOOTER; +void WebPageHandler::render_main_template(const std::string& content, std::stringstream* output) { + static const std::string& footer = std::string("
    ") + get_version_string(true) + std::string("
    "); + + EasyJson ej; + ej["static_pages_available"] = static_pages_available(); + ej["content"] = content; + ej["footer_html"] = footer; + EasyJson path_handlers = ej.Set("path_handlers", EasyJson::kArray); + for (const auto& handler : _page_map) { + if (handler.second->is_on_nav_bar()) { + EasyJson path_handler = path_handlers.PushBack(EasyJson::kObject); + path_handler["path"] = handler.first; + path_handler["alias"] = handler.second->alias(); + } + } + mustache::RenderTemplate(kMainTemplate, _www_path, ej.value(), output); } -void WebPageHandler::root_handler(const ArgumentMap& args, std::stringstream* output) { - // _path_handler_lock already held by MongooseCallback - (*output) << "

    Version

    "; - (*output) << "
    " << get_version_string(false) << "
    " << std::endl; - (*output) << "

    Hardware Info

    "; - (*output) << "
    ";
-    (*output) << CpuInfo::debug_string();
-    (*output) << MemInfo::debug_string();
-    (*output) << DiskInfo::debug_string();
-    (*output) << "
    "; - - (*output) << "

    Status Pages

    "; - for (auto& iter : _page_map) { - (*output) << "" << iter.first << "
    "; +void WebPageHandler::render(const string& path, const EasyJson& ej, bool use_style, + std::stringstream* output) { + if (mustache_template_available(path)) { + mustache::RenderTemplate(mustache_partial_tag(path), _www_path, ej.value(), output); + } else if (use_style) { + (*output) << "
    " << ej.ToString() << "
    "; + } else { + (*output) << ej.ToString(); } } +void WebPageHandler::root_handler(const ArgumentMap& args, EasyJson* output) { + (*output)["version"] = get_version_string(false); + (*output)["cpuinfo"] = CpuInfo::debug_string(); + (*output)["meminfo"] = MemInfo::debug_string(); + (*output)["diskinfo"] = DiskInfo::debug_string(); +} + } // namespace doris diff --git a/be/src/http/web_page_handler.h b/be/src/http/web_page_handler.h index 62d2f5cfcf53fd..63e505a84fd58f 100644 --- a/be/src/http/web_page_handler.h +++ b/be/src/http/web_page_handler.h @@ -27,6 +27,7 @@ #include #include "http/http_handler.h" +#include "util/easy_json.h" namespace doris { @@ -39,22 +40,46 @@ class WebPageHandler : public HttpHandler { typedef std::map ArgumentMap; typedef boost::function PageHandlerCallback; + typedef boost::function + TemplatePageHandlerCallback; WebPageHandler(EvHttpServer* http_server); virtual ~WebPageHandler(); void handle(HttpRequest *req) override; - // Register a route 'path'. + // Register a route 'path' to be rendered via template. + // The appropriate template to use is determined by 'path'. // If 'is_on_nav_bar' is true, a link to the page will be placed on the navbar // in the header of styled pages. The link text is given by 'alias'. + void register_template_page(const std::string& path, const std::string& alias, + const TemplatePageHandlerCallback& callback, bool is_on_nav_bar); + + // Register a route 'path'. See the register_template_page for details. void register_page(const std::string& path, const std::string& alias, const PageHandlerCallback& callback, bool is_on_nav_bar); private: - void bootstrap_page_header(std::stringstream* output); - void bootstrap_page_footer(std::stringstream* output); - void root_handler(const ArgumentMap& args, std::stringstream* output); + void root_handler(const ArgumentMap& args, EasyJson* output); + + // Returns a mustache tag that renders the partial at path when + // passed to mustache::RenderTemplate. + std::string mustache_partial_tag(const std::string& path) const; + + // Returns whether or not a mustache template corresponding + // to the given path can be found. + bool mustache_template_available(const std::string& path) const; + + // Renders the main HTML template with the pre-rendered string 'content' + // in the main body of the page, into 'output'. + void render_main_template(const std::string& content, std::stringstream* output); + + // Renders the template corresponding to 'path' (if available), using + // fields in 'ej'. + void render(const std::string& path, const EasyJson& ej, bool use_style, + std::stringstream* output); + + bool static_pages_available() const; // Container class for a list of path handler callbacks for a single URL. class PathHandler { @@ -85,6 +110,7 @@ class WebPageHandler : public HttpHandler { PageHandlerCallback callback_; }; + std::string _www_path; EvHttpServer* _http_server; // Lock guarding the _path_handlers map boost::mutex _map_lock; diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt index 5d286919778fb5..e9ee89b4e05ce7 100644 --- a/be/src/util/CMakeLists.txt +++ b/be/src/util/CMakeLists.txt @@ -96,6 +96,8 @@ set(UTIL_FILES trace.cpp trace_metrics.cpp timezone_utils.cpp + easy_json.cc + mustache/mustache.cc ) if (WITH_MYSQL) diff --git a/be/src/util/easy_json.cc b/be/src/util/easy_json.cc new file mode 100644 index 00000000000000..56876b6344b715 --- /dev/null +++ b/be/src/util/easy_json.cc @@ -0,0 +1,209 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "util/easy_json.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +// IWYU pragma: no_include + +using rapidjson::SizeType; +using rapidjson::Value; +using std::string; + +namespace doris { + +EasyJson::EasyJson() : alloc_(new EasyJsonAllocator), value_(&alloc_->value()) {} + +EasyJson::EasyJson(EasyJson::ComplexTypeInitializer type) + : alloc_(new EasyJsonAllocator), value_(&alloc_->value()) { + if (type == kObject) { + value_->SetObject(); + } else if (type == kArray) { + value_->SetArray(); + } +} + +EasyJson EasyJson::Get(const string& key) { + if (!value_->IsObject()) { + value_->SetObject(); + } + if (!value_->HasMember(key.c_str())) { + Value key_val(key.c_str(), alloc_->allocator()); + value_->AddMember(key_val, Value().SetNull(), alloc_->allocator()); + } + return EasyJson(&(*value_)[key.c_str()], alloc_); +} + +EasyJson EasyJson::Get(int index) { + if (!value_->IsArray()) { + value_->SetArray(); + } + while (SizeType(index) >= value_->Size()) { + value_->PushBack(Value().SetNull(), alloc_->allocator()); + } + return EasyJson(&(*value_)[index], alloc_); +} + +EasyJson EasyJson::operator[](const string& key) { + return Get(key); +} + +EasyJson EasyJson::operator[](int index) { + return Get(index); +} + +EasyJson& EasyJson::operator=(const string& val) { + value_->SetString(val.c_str(), alloc_->allocator()); + return *this; +} +template +EasyJson& EasyJson::operator=(T val) { + *value_ = val; + return *this; +} +template EasyJson& EasyJson::operator=(bool val); +template EasyJson& EasyJson::operator=(int32_t val); +template EasyJson& EasyJson::operator=(int64_t val); +template EasyJson& EasyJson::operator=(uint32_t val); +template EasyJson& EasyJson::operator=(uint64_t val); +template EasyJson& EasyJson::operator=(double val); +template<> EasyJson& EasyJson::operator=(const char* val) { + value_->SetString(val, alloc_->allocator()); + return *this; +} +template<> EasyJson& EasyJson::operator=(EasyJson::ComplexTypeInitializer val) { + if (val == kObject) { + value_->SetObject(); + } else if (val == kArray) { + value_->SetArray(); + } + return (*this); +} + +EasyJson& EasyJson::SetObject() { + if (!value_->IsObject()) { + value_->SetObject(); + } + return *this; +} + +EasyJson& EasyJson::SetArray() { + if (!value_->IsArray()) { + value_->SetArray(); + } + return *this; +} + +EasyJson EasyJson::Set(const string& key, const string& val) { + return (Get(key) = val); +} +template +EasyJson EasyJson::Set(const string& key, T val) { + return (Get(key) = val); +} +template EasyJson EasyJson::Set(const string& key, bool val); +template EasyJson EasyJson::Set(const string& key, int32_t val); +template EasyJson EasyJson::Set(const string& key, int64_t val); +template EasyJson EasyJson::Set(const string& key, uint32_t val); +template EasyJson EasyJson::Set(const string& key, uint64_t val); +template EasyJson EasyJson::Set(const string& key, double val); +template EasyJson EasyJson::Set(const string& key, const char* val); +template EasyJson EasyJson::Set(const string& key, + EasyJson::ComplexTypeInitializer val); + +EasyJson EasyJson::Set(int index, const string& val) { + return (Get(index) = val); +} +template +EasyJson EasyJson::Set(int index, T val) { + return (Get(index) = val); +} +template EasyJson EasyJson::Set(int index, bool val); +template EasyJson EasyJson::Set(int index, int32_t val); +template EasyJson EasyJson::Set(int index, int64_t val); +template EasyJson EasyJson::Set(int index, uint32_t val); +template EasyJson EasyJson::Set(int index, uint64_t val); +template EasyJson EasyJson::Set(int index, double val); +template EasyJson EasyJson::Set(int index, const char* val); +template EasyJson EasyJson::Set(int index, + EasyJson::ComplexTypeInitializer val); + +EasyJson EasyJson::PushBack(const string& val) { + if (!value_->IsArray()) { + value_->SetArray(); + } + Value push_val(val.c_str(), alloc_->allocator()); + value_->PushBack(push_val, alloc_->allocator()); + return EasyJson(&(*value_)[value_->Size() - 1], alloc_); +} +template +EasyJson EasyJson::PushBack(T val) { + if (!value_->IsArray()) { + value_->SetArray(); + } + value_->PushBack(val, alloc_->allocator()); + return EasyJson(&(*value_)[value_->Size() - 1], alloc_); +} +template EasyJson EasyJson::PushBack(bool val); +template EasyJson EasyJson::PushBack(int32_t val); +template EasyJson EasyJson::PushBack(int64_t val); +template EasyJson EasyJson::PushBack(uint32_t val); +template EasyJson EasyJson::PushBack(uint64_t val); +template EasyJson EasyJson::PushBack(double val); +template<> EasyJson EasyJson::PushBack(const char* val) { + if (!value_->IsArray()) { + value_->SetArray(); + } + Value push_val(val, alloc_->allocator()); + value_->PushBack(push_val, alloc_->allocator()); + return EasyJson(&(*value_)[value_->Size() - 1], alloc_); +} +template<> EasyJson EasyJson::PushBack(EasyJson::ComplexTypeInitializer val) { + if (!value_->IsArray()) { + value_->SetArray(); + } + Value push_val; + if (val == kObject) { + push_val.SetObject(); + } else if (val == kArray) { + push_val.SetArray(); + } else { + LOG(FATAL) << "Unknown initializer type"; + } + value_->PushBack(push_val, alloc_->allocator()); + return EasyJson(&(*value_)[value_->Size() - 1], alloc_); +} + +string EasyJson::ToString() const { + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + value_->Accept(writer); + return buffer.GetString(); +} + +EasyJson::EasyJson(Value* value, scoped_refptr alloc) + : alloc_(std::move(alloc)), value_(value) {} + +} // namespace doris diff --git a/be/src/util/easy_json.h b/be/src/util/easy_json.h new file mode 100644 index 00000000000000..d2eb0702d8738d --- /dev/null +++ b/be/src/util/easy_json.h @@ -0,0 +1,190 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +#pragma once + +#include + +#include + +#include "gutil/ref_counted.h" + +namespace doris { + +// A wrapper around rapidjson Value objects, to simplify usage. +// Intended solely for building json objects, not writing/parsing. +// +// Simplifies code like this: +// +// rapidjson::Document d; +// rapidjson::Value v; +// v.SetObject(); +// rapidjson::Value list; +// list.SetArray(); +// v.AddMember("list", list, d.GetAllocator()); +// v["list"].PushBack(rapidjson::Value().SetString("element"), d.GetAllocator()); +// +// To this: +// +// EasyJson ej; +// ej["list"][0] = "element"; +// +// Client code should build objects as demonstrated above, +// then call EasyJson::value() to obtain a reference to the +// built rapidjson Value. +class EasyJson { + public: + // Used for initializing EasyJson's with complex types. + // For example: + // + // EasyJson array; + // EasyJson nested = array.PushBack(EasyJson::kObject); + // nested["attr"] = "val"; + // // array = [ { "attr": "val" } ] + enum ComplexTypeInitializer { + kObject, + kArray + }; + + EasyJson(); + // Initializes the EasyJson object with the given type. + explicit EasyJson(ComplexTypeInitializer type); + ~EasyJson() = default; + + // Returns the child EasyJson associated with key. + // + // Note: this method can mutate the EasyJson object + // as follows: + // + // If this EasyJson's underlying Value is not an object + // (i.e. !this->value().IsObject()), then its Value is + // coerced to an object, overwriting the old Value. + // If the given key does not exist, a Null-valued + // EasyJson associated with key is created. + EasyJson Get(const std::string& key); + + // Returns the child EasyJson at index. + // + // Note: this method can mutate the EasyJson object + // as follows: + // + // If this EasyJson's underlying Value is not an array + // (i.e. !this->value().IsArray()), then its Value is + // coerced to an array, overwriting the old Value. + // If index >= this->value().Size(), then the underlying + // array's size is increased to index + 1 (new indices + // are filled with Null values). + EasyJson Get(int index); + + // Same as Get(key). + EasyJson operator[](const std::string& key); + // Same as Get(index). + EasyJson operator[](int index); + + // Sets the underlying Value equal to val. + // Returns a reference to the object itself. + // + // 'val' can be a bool, int32_t, int64_t, double, + // char*, string, or ComplexTypeInitializer. + EasyJson& operator=(const std::string& val); + template + EasyJson& operator=(T val); + + // Sets the underlying Value to an object. + // Returns a reference to the object itself. + // + // i.e. after calling SetObject(), + // value().IsObject() == true + EasyJson& SetObject(); + // Sets the underlying Value to an array. + // Returns a reference to the object itself. + // + // i.e. after calling SetArray(), + // value().IsArray() == true + EasyJson& SetArray(); + + // Associates val with key. + // Returns the child object. + // + // If this EasyJson's underlying Value is not an object + // (i.e. !this->value().IsObject()), then its Value is + // coerced to an object, overwriting the old Value. + // If the given key does not exist, a new child entry + // is created with the given value. + EasyJson Set(const std::string& key, const std::string& val); + template + EasyJson Set(const std::string& key, T val); + + // Stores val at index. + // Returns the child object. + // + // If this EasyJson's underlying Value is not an array + // (i.e. !this->value().IsArray()), then its Value is + // coerced to an array, overwriting the old Value. + // If index >= this->value().Size(), then the underlying + // array's size is increased to index + 1 (new indices + // are filled with Null values). + EasyJson Set(int index, const std::string& val); + template + EasyJson Set(int index, T val); + + // Appends val to the underlying array. + // Returns a reference to the new child object. + // + // If this EasyJson's underlying Value is not an array + // (i.e. !this->value().IsArray()), then its Value is + // coerced to an array, overwriting the old Value. + EasyJson PushBack(const std::string& val); + template + EasyJson PushBack(T val); + + // Returns a reference to the underlying Value. + rapidjson::Value& value() const { return *value_; } + + // Returns a string representation of the underlying json. + std::string ToString() const; + + private: + // One instance of EasyJsonAllocator is shared among a root + // EasyJson object and all of its descendants. The allocator + // owns the underlying rapidjson Value, and a rapidjson + // allocator (via a rapidjson::Document). + class EasyJsonAllocator : public RefCounted { + public: + rapidjson::Value& value() { return value_; } + rapidjson::Document::AllocatorType& allocator() { return value_.GetAllocator(); } + private: + friend class RefCounted; + ~EasyJsonAllocator() = default; + + // The underlying rapidjson::Value object (Document is + // a subclass of Value that has its own allocator). + rapidjson::Document value_; + }; + + // Used to instantiate descendant objects. + EasyJson(rapidjson::Value* value, scoped_refptr alloc); + + // One allocator is shared among an EasyJson object and + // all of its descendants. + scoped_refptr alloc_; + + // A pointer to the underlying Value in the object + // tree owned by alloc_. + rapidjson::Value* value_; +}; + +} // namespace doris diff --git a/be/src/util/mustache/mustache.cc b/be/src/util/mustache/mustache.cc new file mode 100644 index 00000000000000..e976677cf4554d --- /dev/null +++ b/be/src/util/mustache/mustache.cc @@ -0,0 +1,448 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "mustache.h" + +#include "rapidjson/stringbuffer.h" +#include +#include "rapidjson/writer.h" + +#include +#include +#include +#include + +#include + +using namespace rapidjson; +using namespace std; +using namespace boost::algorithm; + +namespace mustache { + +// TODO: +// # Handle malformed templates better +// # Better support for reading templates from files + +enum TagOperator { + SUBSTITUTION, + SECTION_START, + NEGATED_SECTION_START, + PREDICATE_SECTION_START, + SECTION_END, + PARTIAL, + COMMENT, + LENGTH, + EQUALITY, + INEQUALITY, + LITERAL, + NONE +}; + +struct OpCtx { + TagOperator op; + string tag_name; + string tag_arg; + bool escaped = false; +}; + +struct ContextStack { + const Value* value; + const ContextStack* parent; +}; + +TagOperator GetOperator(const string& tag) { + if (tag.size() == 0) return SUBSTITUTION; + switch (tag[0]) { + case '#': return SECTION_START; + case '^': return NEGATED_SECTION_START; + case '?': return PREDICATE_SECTION_START; + case '/': return SECTION_END; + case '>': return PARTIAL; + case '!': + if (tag.size() == 1 || tag[1] != '=') return COMMENT; + return INEQUALITY; + case '%': return LENGTH; + case '~': return LITERAL; + case '=': return EQUALITY; + default: return SUBSTITUTION; + } +} + +int EvaluateTag(const string& document, const string& document_root, int idx, + const ContextStack* context, const OpCtx& op_ctx, stringstream* out); + +static bool RenderTemplate(const string& document, const string& document_root, + const ContextStack* stack, stringstream* out); + +void EscapeHtml(const string& in, stringstream *out) { + for (const char& c: in) { + switch (c) { + case '&': (*out) << "&"; + break; + case '"': (*out) << """; + break; + case '\'': (*out) << "'"; + break; + case '<': (*out) << "<"; + break; + case '>': (*out) << ">"; + break; + default: (*out) << c; + break; + } + } +} + +void Dump(const rapidjson::Value& v) { + StringBuffer buffer; + Writer writer(buffer); + v.Accept(writer); + std::cout << buffer.GetString() << std::endl; +} + +// Breaks a dotted path into individual components. One wrinkle, which stops this from +// being a simple split() is that we allow path components to be quoted, e.g.: "foo".bar, +// and any '.' characters inside those quoted sections aren't considered to be +// delimiters. This is to allow Json keys that contain periods. +void FindJsonPathComponents(const string& path, vector* components) { + bool in_quote = false; + bool escape_this_char = false; + int start = 0; + for (int i = start; i < path.size(); ++i) { + if (path[i] == '"' && !escape_this_char) in_quote = !in_quote; + if (path[i] == '.' && !escape_this_char && !in_quote) { + // Current char == delimiter and not escaped and not in a quote pair => found a + // component + if (i - start > 0) { + if (path[start] == '"' && path[(i - 1) - start] == '"') { + if (i - start > 3) { + components->push_back(path.substr(start + 1, i - (start + 2))); + } + } else { + components->push_back(path.substr(start, i - start)); + } + start = i + 1; + } + } + + escape_this_char = (path[i] == '\\' && !escape_this_char); + } + + if (path.size() - start > 0) { + if (path[start] == '"' && path[(path.size() - 1) - start] == '"') { + if (path.size() - start > 3) { + components->push_back(path.substr(start + 1, path.size() - (start + 2))); + } + } else { + components->push_back(path.substr(start, path.size() - start)); + } + } +} + +// Looks up the json entity at 'path' in 'parent_context', and places it in 'resolved'. If +// the entity does not exist (i.e. the path is invalid), 'resolved' will be set to nullptr. +void ResolveJsonContext(const string& path, const ContextStack* stack, + const Value** resolved) { + if (path == ".") { + *resolved = stack->value; + return; + } + vector components; + FindJsonPathComponents(path, &components); + + // At each enclosing level of context, try to resolve the path. + for ( ; stack != nullptr; stack = stack->parent) { + const Value* cur = stack->value; + bool match = true; + for(const string& c: components) { + if (cur->IsObject() && cur->HasMember(c.c_str())) { + cur = &(*cur)[c.c_str()]; + } else { + match = false; + break; + } + } + if (match) { + *resolved = cur; + return; + } + } + *resolved = nullptr; +} + +int FindNextTag(const string& document, int idx, OpCtx* op, stringstream* out) { + op->op = NONE; + while (idx < document.size()) { + if (document[idx] == '{' && idx < (document.size() - 3) && document[idx + 1] == '{') { + if (document[idx + 2] == '{') { + idx += 3; + op->escaped = true; + } else { + op->escaped = false; + idx += 2; // Now at start of template expression + } + stringstream expr; + while (idx < document.size()) { + if (document[idx] != '}') { + expr << document[idx]; + ++idx; + } else { + if (!op->escaped && idx < document.size() - 1 && document[idx + 1] == '}') { + ++idx; + break; + } else if (op->escaped && idx < document.size() - 2 && document[idx + 1] == '}' + && document[idx + 2] == '}') { + idx += 2; + break; + } else { + expr << '}'; + } + } + } + + string key = expr.str(); + trim(key); + if (key != ".") trim_if(key, is_any_of(".")); + if (key.size() == 0) continue; + op->op = GetOperator(key); + if (op->op != SUBSTITUTION) { + int len = op->op == INEQUALITY ? 2 : 1; + key = key.substr(len); + trim(key); + } + if (key.size() == 0) continue; + + if (op->op == EQUALITY || op->op == INEQUALITY) { + // Find an argument + vector components; + split(components, key, is_any_of(" ")); + key = components[0]; + components.erase(components.begin()); + op->tag_arg = join(components, " "); + } + + op->tag_name = key; + return ++idx; + } else { + if (out != nullptr) (*out) << document[idx]; + } + ++idx; + } + return idx; +} + +// Evaluates a [PREDICATE_|NEGATED_]SECTION_START / SECTION_END pair by evaluating the tag +// in 'parent_context'. False or non-existant values cause the entire section to be +// skipped. True values cause the section to be evaluated as though it were a normal +// section, but with the parent context being the root context for that section. +// +// If 'is_negation' is true, the behaviour is the opposite of the above: false values +// cause the section to be normally evaluated etc. +int EvaluateSection(const string& document, const string& document_root, int idx, + const ContextStack* context_stack, const OpCtx& op_ctx, stringstream* out) { + // Precondition: idx is the immediate next character after an opening {{ #tag_name }} + const Value* context; + ResolveJsonContext(op_ctx.tag_name, context_stack, &context); + + // If we a) cannot resolve the context from the tag name or b) the context evaluates to + // false, we should skip the contents of the template until a closing {{/tag_name}}. + bool skip_contents = false; + + if (op_ctx.op == NEGATED_SECTION_START || op_ctx.op == PREDICATE_SECTION_START || + op_ctx.op == SECTION_START) { + skip_contents = (context == nullptr || context->IsFalse()); + + // If the tag is a negative block (i.e. {{^tag_name}}), do the opposite: if the + // context exists and is true, skip the contents, else echo them. + if (op_ctx.op == NEGATED_SECTION_START) { + context = context_stack->value; + skip_contents = !skip_contents; + } else if (op_ctx.op == PREDICATE_SECTION_START) { + context = context_stack->value; + } + } else if (op_ctx.op == INEQUALITY || op_ctx.op == EQUALITY) { + skip_contents = (context == nullptr || !context->IsString() || + strcasecmp(context->GetString(), op_ctx.tag_arg.c_str()) != 0); + if (op_ctx.op == INEQUALITY) skip_contents = !skip_contents; + context = context_stack->value; + } + + vector values; + if (!skip_contents && context != nullptr && context->IsArray()) { + for (int i = 0; i < context->Size(); ++i) { + values.push_back(&(*context)[i]); + } + } else { + values.push_back(skip_contents ? nullptr : context); + } + if (values.size() == 0) { + skip_contents = true; + values.push_back(nullptr); + } + + int start_idx = idx; + for(const Value* v: values) { + idx = start_idx; + stack section_starts; + section_starts.push(op_ctx); + while (idx < document.size()) { + OpCtx next_ctx; + idx = FindNextTag(document, idx, &next_ctx, skip_contents ? nullptr : out); + if (skip_contents && (next_ctx.op == SECTION_START || + next_ctx.op == PREDICATE_SECTION_START || + next_ctx.op == NEGATED_SECTION_START)) { + section_starts.push(next_ctx); + } else if (next_ctx.op == SECTION_END) { + if (next_ctx.tag_name != section_starts.top().tag_name) return -1; + section_starts.pop(); + } + if (section_starts.empty()) break; + + // Don't need to evaluate any templates if we're skipping the contents + if (!skip_contents) { + ContextStack new_context = { v, context_stack }; + idx = EvaluateTag(document, document_root, idx, &new_context, next_ctx, out); + } + } + } + return idx; +} + +// Evaluates a SUBSTITUTION tag, by replacing its contents with the value of the tag's +// name in 'parent_context'. +int EvaluateSubstitution(const string& document, const int idx, + const ContextStack* context_stack, const OpCtx& op_ctx, stringstream* out) { + const Value* val; + ResolveJsonContext(op_ctx.tag_name, context_stack, &val); + if (val == nullptr) return idx; + if (val->IsString()) { + if (!op_ctx.escaped) { + EscapeHtml(val->GetString(), out); + } else { + // TODO: Triple {{{ means don't escape + (*out) << val->GetString(); + } + } else if (val->IsInt64()) { + (*out) << val->GetInt64(); + } else if (val->IsInt()) { + (*out) << val->GetInt(); + } else if (val->IsDouble()) { + (*out) << val->GetDouble(); + } else if (val->IsBool()) { + (*out) << boolalpha << val->GetBool(); + } + return idx; +} + +// Evaluates a LENGTH tag by replacing its contents with the type-dependent 'size' of the +// value. +int EvaluateLength(const string& document, const int idx, const ContextStack* context_stack, + const string& tag_name, stringstream* out) { + const Value* val; + ResolveJsonContext(tag_name, context_stack, &val); + if (val == nullptr) return idx; + if (val->IsArray()) { + (*out) << val->Size(); + } else if (val->IsString()) { + (*out) << val->GetStringLength(); + }; + + return idx; +} + +int EvaluateLiteral(const string& document, const int idx, const ContextStack* context_stack, + const string& tag_name, stringstream* out) { + const Value* val; + ResolveJsonContext(tag_name, context_stack, &val); + if (val == nullptr) return idx; + if (!val->IsArray() && !val->IsObject()) return idx; + StringBuffer strbuf; + PrettyWriter writer(strbuf); + val->Accept(writer); + (*out) << strbuf.GetString(); + return idx; +} + +// Evaluates a 'partial' template by reading it fully from disk, then rendering it +// directly into the current output with the current context. +// +// TODO: This could obviously be more efficient (and there are lots of file accesses in a +// long list context). +void EvaluatePartial(const string& tag_name, const string& document_root, + const ContextStack* stack, stringstream* out) { + stringstream ss; + ss << document_root << tag_name; + ifstream tmpl(ss.str().c_str()); + if (!tmpl.is_open()) { + ss << ".mustache"; + tmpl.open(ss.str().c_str()); + if (!tmpl.is_open()) return; + } + stringstream file_ss; + file_ss << tmpl.rdbuf(); + RenderTemplate(file_ss.str(), document_root, stack, out); +} + +// Given a tag name, and its operator, evaluate the tag in the given context and write the +// output to 'out'. The heavy-lifting is delegated to specific Evaluate*() +// methods. Returns the new cursor position within 'document', or -1 on error. +int EvaluateTag(const string& document, const string& document_root, int idx, + const ContextStack* context, const OpCtx& op_ctx, stringstream* out) { + if (idx == -1) return idx; + switch (op_ctx.op) { + case SECTION_START: + case PREDICATE_SECTION_START: + case NEGATED_SECTION_START: + case EQUALITY: + case INEQUALITY: + return EvaluateSection(document, document_root, idx, context, op_ctx, out); + case SUBSTITUTION: + return EvaluateSubstitution(document, idx, context, op_ctx, out); + case COMMENT: + return idx; // Ignored + case PARTIAL: + EvaluatePartial(op_ctx.tag_name, document_root, context, out); + return idx; + case LENGTH: + return EvaluateLength(document, idx, context, op_ctx.tag_name, out); + case LITERAL: + return EvaluateLiteral(document, idx, context, op_ctx.tag_name, out); + case NONE: + return idx; // No tag was found + case SECTION_END: + return idx; + default: + cout << "Unknown tag: " << op_ctx.op << endl; + return -1; + } +} + +static bool RenderTemplate(const string& document, const string& document_root, + const ContextStack* stack, stringstream* out) { + int idx = 0; + while (idx < document.size() && idx != -1) { + OpCtx op; + idx = FindNextTag(document, idx, &op, out); + idx = EvaluateTag(document, document_root, idx, stack, op, out); + } + + return idx != -1; +} + +bool RenderTemplate(const string& document, const string& document_root, + const Value& context, stringstream* out) { + ContextStack stack = { &context, nullptr }; + return RenderTemplate(document, document_root, &stack, out); +} + +} diff --git a/be/src/util/mustache/mustache.h b/be/src/util/mustache/mustache.h new file mode 100644 index 00000000000000..9a2d9fb101f675 --- /dev/null +++ b/be/src/util/mustache/mustache.h @@ -0,0 +1,27 @@ +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "rapidjson/document.h" +#include + +// Routines for rendering Mustache (http://mustache.github.io) templates with RapidJson +// (https://code.google.com/p/rapidjson/) documents. +namespace mustache { + +// Render a template contained in 'document' with respect to the json context +// 'context'. Alternately finds a tag and then evaluates it. Returns when an error is +// signalled (TODO: probably doesn't work in all paths), and evaluates that tag. Output is +// accumulated in 'out'. +bool RenderTemplate(const std::string& document, const std::string& document_root, + const rapidjson::Value& context, std::stringstream* out); + +} diff --git a/be/test/util/CMakeLists.txt b/be/test/util/CMakeLists.txt index 882921e590e329..9d419bbb75f412 100644 --- a/be/test/util/CMakeLists.txt +++ b/be/test/util/CMakeLists.txt @@ -64,3 +64,4 @@ ADD_BE_TEST(scoped_cleanup_test) ADD_BE_TEST(thread_test) ADD_BE_TEST(threadpool_test) ADD_BE_TEST(trace_test) +ADD_BE_TEST(easy_json-test) diff --git a/be/test/util/easy_json-test.cpp b/be/test/util/easy_json-test.cpp new file mode 100644 index 00000000000000..616ffb8aadc0a2 --- /dev/null +++ b/be/test/util/easy_json-test.cpp @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include +#include + +#include "gutil/integral_types.h" +#include "util/easy_json.h" + +using rapidjson::SizeType; +using rapidjson::Value; +using std::string; + +namespace doris { + +class EasyJsonTest: public ::testing::Test {}; + +TEST_F(EasyJsonTest, TestNull) { + EasyJson ej; + ASSERT_TRUE(ej.value().IsNull()); +} + +TEST_F(EasyJsonTest, TestBasic) { + EasyJson ej; + ej.SetObject(); + ej.Set("1", true); + ej.Set("2", kint32min); + ej.Set("4", kint64min); + ej.Set("6", 1.0); + ej.Set("7", "string"); + + Value& v = ej.value(); + + ASSERT_EQ(v["1"].GetBool(), true); + ASSERT_EQ(v["2"].GetInt(), kint32min); + ASSERT_EQ(v["4"].GetInt64(), kint64min); + ASSERT_EQ(v["6"].GetDouble(), 1.0); + ASSERT_EQ(string(v["7"].GetString()), "string"); +} + +TEST_F(EasyJsonTest, TestNested) { + EasyJson ej; + ej.SetObject(); + ej.Get("nested").SetObject(); + ej.Get("nested").Set("nested_attr", true); + ASSERT_EQ(ej.value()["nested"]["nested_attr"].GetBool(), true); + + ej.Get("nested_array").SetArray(); + ej.Get("nested_array").PushBack(1); + ej.Get("nested_array").PushBack(2); + ASSERT_EQ(ej.value()["nested_array"][SizeType(0)].GetInt(), 1); + ASSERT_EQ(ej.value()["nested_array"][SizeType(1)].GetInt(), 2); +} + +TEST_F(EasyJsonTest, TestCompactSyntax) { + EasyJson ej; + ej["nested"]["nested_attr"] = true; + ASSERT_EQ(ej.value()["nested"]["nested_attr"].GetBool(), true); + + for (int i = 0; i < 2; i++) { + ej["nested_array"][i] = i + 1; + } + ASSERT_EQ(ej.value()["nested_array"][SizeType(0)].GetInt(), 1); + ASSERT_EQ(ej.value()["nested_array"][SizeType(1)].GetInt(), 2); +} + +TEST_F(EasyJsonTest, TestComplexInitializer) { + EasyJson ej; + ej = EasyJson::kObject; + ASSERT_TRUE(ej.value().IsObject()); + + EasyJson nested_arr = ej.Set("nested_arr", EasyJson::kArray); + ASSERT_TRUE(nested_arr.value().IsArray()); + + EasyJson nested_obj = nested_arr.PushBack(EasyJson::kObject); + ASSERT_TRUE(ej["nested_arr"][0].value().IsObject()); +} + +TEST_F(EasyJsonTest, TestAllocatorLifetime) { + EasyJson* root = new EasyJson; + EasyJson child = (*root)["child"]; + delete root; + + child["child_attr"] = 1; + ASSERT_EQ(child.value()["child_attr"].GetInt(), 1); +} +} // namespace doris + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + diff --git a/webroot/be/home.mustache b/webroot/be/home.mustache new file mode 100644 index 00000000000000..22995d2ef09910 --- /dev/null +++ b/webroot/be/home.mustache @@ -0,0 +1,28 @@ +{{! +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +}} + +

    Version Info

    +
    {{version}}
    +

    Hardware Info

    +

    CPU Info

    +
    {{cpuinfo}}
    +

    Memory Info

    +
    {{meminfo}}
    +

    Disk Info

    +
    {{diskinfo}}
    From 80f5283442977743fe6dc93e5379f4806e27a90e Mon Sep 17 00:00:00 2001 From: Yingchun Lai <405403881@qq.com> Date: Wed, 15 Jul 2020 09:02:47 +0000 Subject: [PATCH 2/3] fix include --- be/src/http/web_page_handler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/http/web_page_handler.cpp b/be/src/http/web_page_handler.cpp index 436494d12e46b2..788b028e8e4156 100644 --- a/be/src/http/web_page_handler.cpp +++ b/be/src/http/web_page_handler.cpp @@ -19,7 +19,6 @@ #include #include -#include #include "common/config.h" #include "env/env.h" @@ -37,6 +36,7 @@ #include "util/debug_util.h" #include "util/disk_info.h" #include "util/mem_info.h" +#include "util/mustache/mustache.h" using strings::Substitute; From 13964023bbc41b3ced346b768d508bcc8dcc1cac Mon Sep 17 00:00:00 2001 From: Yingchun Lai <405403881@qq.com> Date: Wed, 15 Jul 2020 14:33:50 +0000 Subject: [PATCH 3/3] disable to show Logs --- be/src/http/default_path_handlers.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp index 387de4b45fcf36..0ebbbc0010645a 100644 --- a/be/src/http/default_path_handlers.cpp +++ b/be/src/http/default_path_handlers.cpp @@ -104,7 +104,8 @@ void mem_usage_handler(MemTracker* mem_tracker, const WebPageHandler::ArgumentMa } void add_default_path_handlers(WebPageHandler* web_page_handler, MemTracker* process_mem_tracker) { - web_page_handler->register_page("/logs", "Logs", logs_handler, true /* is_on_nav_bar */); + // TODO(yingchun): logs_handler is not implemented yet, so not show it on navigate bar + web_page_handler->register_page("/logs", "Logs", logs_handler, false /* is_on_nav_bar */); web_page_handler->register_page("/varz", "Configs", config_handler, true /* is_on_nav_bar */); web_page_handler->register_page("/memz", "Memory", boost::bind(&mem_usage_handler, process_mem_tracker, _1, _2), true /* is_on_nav_bar */);