From 2b232147daa5ed06ddaac88cd65fd107cb253a40 Mon Sep 17 00:00:00 2001 From: Yingchun Lai <405403881@qq.com> Date: Sun, 5 Jul 2020 15:57:11 +0800 Subject: [PATCH 1/3] [webserver] Make BE webserver handle static files Make BE webserver handle static files, e.g. css, js, ico, then we can make BE website more pretty. --- be/src/common/config.h | 2 + be/src/http/default_path_handlers.cpp | 8 +-- be/src/http/download_action.cpp | 81 +--------------------- be/src/http/download_action.h | 17 +---- be/src/http/ev_http_server.cpp | 12 ++++ be/src/http/ev_http_server.h | 4 ++ be/src/http/utils.cpp | 97 ++++++++++++++++++++++++++- be/src/http/utils.h | 6 ++ be/src/http/web_page_handler.cpp | 87 +++++++++++++----------- be/src/http/web_page_handler.h | 51 +++++++++----- 10 files changed, 210 insertions(+), 155 deletions(-) diff --git a/be/src/common/config.h b/be/src/common/config.h index ddb4ede67b89ab..a29cbfee7a76c9 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -536,6 +536,8 @@ namespace config { // Whether to continue to start be when load tablet from header failed. CONF_Bool(ignore_load_tablet_failure, "false"); + // Directory where BE's website files placed. + CONF_String(www_path, "${DORIS_HOME}/www"); } // namespace config } // namespace doris diff --git a/be/src/http/default_path_handlers.cpp b/be/src/http/default_path_handlers.cpp index 77c8958607695e..387de4b45fcf36 100644 --- a/be/src/http/default_path_handlers.cpp +++ b/be/src/http/default_path_handlers.cpp @@ -104,10 +104,10 @@ void mem_usage_handler(MemTracker* mem_tracker, const WebPageHandler::ArgumentMa } void add_default_path_handlers(WebPageHandler* web_page_handler, MemTracker* process_mem_tracker) { - web_page_handler->register_page("/logs", logs_handler); - web_page_handler->register_page("/varz", config_handler); - web_page_handler->register_page( - "/memz", boost::bind(&mem_usage_handler, process_mem_tracker, _1, _2)); + web_page_handler->register_page("/logs", "Logs", logs_handler, true /* is_on_nav_bar */); + web_page_handler->register_page("/varz", "Configs", config_handler, true /* is_on_nav_bar */); + web_page_handler->register_page("/memz", "Memory", + boost::bind(&mem_usage_handler, process_mem_tracker, _1, _2), true /* is_on_nav_bar */); } } // namespace doris diff --git a/be/src/http/download_action.cpp b/be/src/http/download_action.cpp index 47a299dbe1655f..88d5ad3de780ed 100644 --- a/be/src/http/download_action.cpp +++ b/be/src/http/download_action.cpp @@ -18,8 +18,6 @@ #include "http/download_action.h" #include -#include -#include #include #include @@ -32,6 +30,7 @@ #include "http/http_request.h" #include "http/http_response.h" #include "http/http_status.h" +#include "http/utils.h" #include "runtime/exec_env.h" #include "util/file_utils.h" #include "util/filesystem_util.h" @@ -134,84 +133,6 @@ void DownloadAction::handle(HttpRequest *req) { LOG(INFO) << "deal with download requesst finished! "; } -void DownloadAction::do_dir_response( - const std::string& dir_path, HttpRequest *req) { - std::vector files; - Status status = FileUtils::list_files(Env::Default(), dir_path, &files); - if (!status.ok()) { - LOG(WARNING) << "Failed to scan dir. dir=" << dir_path; - HttpChannel::send_error(req, HttpStatus::INTERNAL_SERVER_ERROR); - } - - const std::string FILE_DELIMETER_IN_DIR_RESPONSE = "\n"; - - std::stringstream result; - for (const std::string& file_name : files) { - result << file_name << FILE_DELIMETER_IN_DIR_RESPONSE; - } - - std::string result_str = result.str(); - HttpChannel::send_reply(req, result_str); - return; -} - -void DownloadAction::do_file_response(const std::string& file_path, HttpRequest *req) { - // read file content and send response - int fd = open(file_path.c_str(), O_RDONLY); - if (fd < 0) { - LOG(WARNING) << "Failed to open file: " << file_path; - HttpChannel::send_error(req, HttpStatus::NOT_FOUND); - return; - } - struct stat st; - auto res = fstat(fd, &st); - if (res < 0) { - close(fd); - LOG(WARNING) << "Failed to open file: " << file_path; - HttpChannel::send_error(req, HttpStatus::NOT_FOUND); - return; - } - - int64_t file_size = st.st_size; - - // TODO(lingbin): process "IF_MODIFIED_SINCE" header - // TODO(lingbin): process "RANGE" header - const std::string& range_header = req->header(HttpHeaders::RANGE); - if (!range_header.empty()) { - // analyse range header - } - - req->add_output_header(HttpHeaders::CONTENT_TYPE, get_content_type(file_path).c_str()); - - if (req->method() == HttpMethod::HEAD) { - close(fd); - req->add_output_header(HttpHeaders::CONTENT_LENGTH, std::to_string(file_size).c_str()); - HttpChannel::send_reply(req); - return; - } - - HttpChannel::send_file(req, fd, 0, file_size); -} - -// Do a simple decision, only deal a few type -std::string DownloadAction::get_content_type(const std::string& file_name) { - std::string file_ext = path_util::file_extension(file_name); - LOG(INFO) << "file_name: " << file_name << "; file extension: [" << file_ext << "]"; - if (file_ext == std::string(".html") - || file_ext == std::string(".htm")) { - return std::string("text/html; charset=utf-8"); - } else if (file_ext == std::string(".js")) { - return std::string("application/javascript; charset=utf-8"); - } else if (file_ext == std::string(".css")) { - return std::string("text/css; charset=utf-8"); - } else if (file_ext == std::string(".txt")) { - return std::string("text/plain; charset=utf-8"); - } else { - return "text/plain; charset=utf-8"; - } - return ""; -} - Status DownloadAction::check_token(HttpRequest *req) { const std::string& token_str = req->param(TOKEN_PARAMETER); if (token_str.empty()) { diff --git a/be/src/http/download_action.h b/be/src/http/download_action.h index a82f19559c3484..e04b94a6dec4d1 100644 --- a/be/src/http/download_action.h +++ b/be/src/http/download_action.h @@ -54,28 +54,13 @@ class DownloadAction : public HttpHandler { Status check_log_path_is_allowed(const std::string& file_path); void handle_normal(HttpRequest *req, const std::string& file_param); - void handle_error_log( - HttpRequest *req, - const std::string& file_param); - - void do_file_response(const std::string& dir_path, HttpRequest *req); - void do_dir_response(const std::string& dir_path, HttpRequest *req); - - Status get_file_content( - FILE* fp, char* buffer, int32_t buffer_size, - int32_t* readed_size, bool* eos); - - int64_t get_file_size(FILE* fp); - - std::string get_content_type(const std::string& file_name); + void handle_error_log(HttpRequest *req, const std::string& file_param); ExecEnv* _exec_env; DOWNLOAD_TYPE _download_type; std::vector _allow_paths; std::string _error_log_root_dir; - - }; // end class DownloadAction } // end namespace doris diff --git a/be/src/http/ev_http_server.cpp b/be/src/http/ev_http_server.cpp index 48013d4997c767..ee8684da245920 100644 --- a/be/src/http/ev_http_server.cpp +++ b/be/src/http/ev_http_server.cpp @@ -206,6 +206,14 @@ bool EvHttpServer::register_handler( return result; } +void EvHttpServer::register_static_file_handler(HttpHandler* handler) { + DCHECK(handler != nullptr); + DCHECK(_static_file_handler == nullptr); + pthread_rwlock_wrlock(&_rw_lock); + _static_file_handler = handler; + pthread_rwlock_unlock(&_rw_lock); +} + int EvHttpServer::on_header(struct evhttp_request* ev_req) { std::unique_ptr request(new HttpRequest(ev_req)); auto res = request->init_from_evhttp(); @@ -247,6 +255,10 @@ HttpHandler* EvHttpServer::_find_handler(HttpRequest* req) { switch (req->method()) { case GET: _get_handlers.retrieve(path, &handler, req->params()); + // Static file handler is a fallback handler + if (handler == nullptr) { + handler = _static_file_handler; + } break; case PUT: _put_handlers.retrieve(path, &handler, req->params()); diff --git a/be/src/http/ev_http_server.h b/be/src/http/ev_http_server.h index 466104e9d37bc2..6ef632ed8be5c5 100644 --- a/be/src/http/ev_http_server.h +++ b/be/src/http/ev_http_server.h @@ -39,6 +39,9 @@ class EvHttpServer { // register handler for an a path-method pair bool register_handler( const HttpMethod& method, const std::string& path, HttpHandler* handler); + + void register_static_file_handler(HttpHandler* handler); + Status start(); void stop(); void join(); @@ -69,6 +72,7 @@ class EvHttpServer { pthread_rwlock_t _rw_lock; PathTrie _get_handlers; + HttpHandler* _static_file_handler = nullptr; PathTrie _put_handlers; PathTrie _post_handlers; PathTrie _delete_handlers; diff --git a/be/src/http/utils.cpp b/be/src/http/utils.cpp index 7953a7cda10c92..bc8bfbfcce2e57 100644 --- a/be/src/http/utils.cpp +++ b/be/src/http/utils.cpp @@ -15,13 +15,21 @@ // specific language governing permissions and limitations // under the License. -#include +#include "http/utils.h" + +#include +#include #include "common/logging.h" +#include "common/status.h" #include "common/utils.h" +#include "env/env.h" +#include "util/file_utils.h" #include "http/http_common.h" +#include "http/http_channel.h" #include "http/http_headers.h" #include "http/http_request.h" +#include "util/path_util.h" #include "util/url_coding.h" namespace doris { @@ -78,4 +86,91 @@ bool parse_basic_auth(const HttpRequest& req, AuthInfo* auth) { return true; } +// Do a simple decision, only deal a few type +std::string get_content_type(const std::string& file_name) { + std::string file_ext = path_util::file_extension(file_name); + LOG(INFO) << "file_name: " << file_name << "; file extension: [" << file_ext << "]"; + if (file_ext == std::string(".html") + || file_ext == std::string(".htm")) { + return std::string("text/html; charset=utf-8"); + } else if (file_ext == std::string(".js")) { + return std::string("application/javascript; charset=utf-8"); + } else if (file_ext == std::string(".css")) { + return std::string("text/css; charset=utf-8"); + } else if (file_ext == std::string(".txt")) { + return std::string("text/plain; charset=utf-8"); + } else if (file_ext == std::string(".png")) { + return std::string("image/png"); + } else if (file_ext == std::string(".ico")) { + return std::string("image/x-icon"); + } else { + return "text/plain; charset=utf-8"; + } + return ""; +} + +void do_file_response(const std::string& file_path, HttpRequest *req) { + if (file_path.find("..") != std::string::npos) { + LOG(WARNING) << "Not allowed to read relative path: " << file_path; + HttpChannel::send_error(req, HttpStatus::FORBIDDEN); + return; + } + + // read file content and send response + int fd = open(file_path.c_str(), O_RDONLY); + if (fd < 0) { + LOG(WARNING) << "Failed to open file: " << file_path; + HttpChannel::send_error(req, HttpStatus::NOT_FOUND); + return; + } + struct stat st; + auto res = fstat(fd, &st); + if (res < 0) { + close(fd); + LOG(WARNING) << "Failed to open file: " << file_path; + HttpChannel::send_error(req, HttpStatus::NOT_FOUND); + return; + } + + int64_t file_size = st.st_size; + + // TODO(lingbin): process "IF_MODIFIED_SINCE" header + // TODO(lingbin): process "RANGE" header + const std::string& range_header = req->header(HttpHeaders::RANGE); + if (!range_header.empty()) { + // analyse range header + } + + req->add_output_header(HttpHeaders::CONTENT_TYPE, get_content_type(file_path).c_str()); + + if (req->method() == HttpMethod::HEAD) { + close(fd); + req->add_output_header(HttpHeaders::CONTENT_LENGTH, std::to_string(file_size).c_str()); + HttpChannel::send_reply(req); + return; + } + + HttpChannel::send_file(req, fd, 0, file_size); +} + +void do_dir_response(const std::string& dir_path, HttpRequest *req) { + std::vector files; + Status status = FileUtils::list_files(Env::Default(), dir_path, &files); + if (!status.ok()) { + LOG(WARNING) << "Failed to scan dir. dir=" << dir_path; + HttpChannel::send_error(req, HttpStatus::INTERNAL_SERVER_ERROR); + } + + const std::string FILE_DELIMETER_IN_DIR_RESPONSE = "\n"; + + std::stringstream result; + for (const std::string& file_name : files) { + result << file_name << FILE_DELIMETER_IN_DIR_RESPONSE; + } + + std::string result_str = result.str(); + HttpChannel::send_reply(req, result_str); + return; +} + } diff --git a/be/src/http/utils.h b/be/src/http/utils.h index 8e82d7bed58aa9..0a1a50b0d5d9f7 100644 --- a/be/src/http/utils.h +++ b/be/src/http/utils.h @@ -21,6 +21,7 @@ #include "common/utils.h" #include "http/http_common.h" +#include "http/http_request.h" namespace doris { @@ -34,4 +35,9 @@ bool parse_basic_auth(const HttpRequest& req, std::string* user, std::string* pa bool parse_basic_auth(const HttpRequest& req, AuthInfo* auth); +void do_file_response(const std::string& dir_path, HttpRequest *req); + +void do_dir_response(const std::string& dir_path, HttpRequest *req); + +std::string get_content_type(const std::string& file_name); } diff --git a/be/src/http/web_page_handler.cpp b/be/src/http/web_page_handler.cpp index 2140b2749d391a..455ec272939bdd 100644 --- a/be/src/http/web_page_handler.cpp +++ b/be/src/http/web_page_handler.cpp @@ -20,80 +20,91 @@ #include #include +#include "common/config.h" +#include "env/env.h" +#include "gutil/stl_util.h" +#include "gutil/strings/substitute.h" #include "http/ev_http_server.h" #include "http/http_channel.h" #include "http/http_headers.h" #include "http/http_request.h" #include "http/http_response.h" #include "http/http_status.h" +#include "http/utils.h" +#include "olap/file_helper.h" #include "util/cpu_info.h" #include "util/debug_util.h" #include "util/disk_info.h" #include "util/mem_info.h" +using strings::Substitute; + namespace doris { static std::string s_html_content_type = "text/html"; WebPageHandler::WebPageHandler(EvHttpServer* server) : _http_server(server) { - PageHandlerCallback default_callback = + // Make WebPageHandler to be static file handler, static files, e.g. css, png, will be handled by WebPageHandler. + _http_server->register_static_file_handler(this); + + PageHandlerCallback root_callback = boost::bind(boost::mem_fn(&WebPageHandler::root_handler), this, _1, _2); - register_page("/", default_callback); + register_page("/", "Home", root_callback, false /* is_on_nav_bar */); +} + +WebPageHandler::~WebPageHandler() { + STLDeleteValues(&_page_map); } -void WebPageHandler::register_page(const std::string& path, const PageHandlerCallback& callback) { - // Put this handler to to s_handler_by_name - // because handler does't often new - // So we insert it to this set when everytime +void WebPageHandler::register_page(const std::string& path, const string& alias, + const PageHandlerCallback& callback, bool is_on_nav_bar) { boost::mutex::scoped_lock lock(_map_lock); - auto map_iter = _page_map.find(path); - if (map_iter == _page_map.end()) { - // first time, register this to web server - _http_server->register_handler(HttpMethod::GET, path, this); - } - _page_map[path].add_callback(callback); + CHECK(_page_map.find(path) == _page_map.end()); + // first time, register this to web server + _http_server->register_handler(HttpMethod::GET, path, this); + _page_map[path] = new PathHandler(true /* is_styled */, is_on_nav_bar, alias, callback); } void WebPageHandler::handle(HttpRequest* req) { - // Should we render with css styles? - bool use_style = true; - auto& params = *req->params(); - if (params.find("raw") != params.end()) { - use_style = false; - } - - std::stringstream output; - - // Append header - if (use_style) { - bootstrap_page_header(&output); - } - - // Append content - // push_content(&output); LOG(INFO) << req->debug_string(); + + PathHandler* handler = nullptr; { boost::mutex::scoped_lock lock(_map_lock); auto iter = _page_map.find(req->raw_path()); if (iter != _page_map.end()) { - for (auto& callback : iter->second.callbacks()) { - callback(*req->params(), &output); - } + handler = iter->second; } } + if (handler == nullptr) { + // Try to handle static file request + do_file_response(config::www_path + req->raw_path(), req); + // Has replied in do_file_response, so we return here. + return; + } + + const auto& params = *req->params(); + + // Should we render with css styles? + bool use_style = (params.find("raw") == params.end()); + + std::stringstream content; + // Append header + if (use_style) { + bootstrap_page_header(&content); + } + + // Append content + handler->callback()(params, &content); + // Append footer if (use_style) { - bootstrap_page_footer(&output); + bootstrap_page_footer(&content); } - std::string str = output.str(); req->add_output_header(HttpHeaders::CONTENT_TYPE, s_html_content_type.c_str()); - HttpChannel::send_reply(req, HttpStatus::OK, str); -#if 0 - HttpResponse response(HttpStatus::OK, s_html_content_type, &str); - channel->send_response(response); -#endif + HttpChannel::send_reply(req, HttpStatus::OK, content.str()); } static const std::string PAGE_HEADER = diff --git a/be/src/http/web_page_handler.h b/be/src/http/web_page_handler.h index 2c5394793d5998..2f16192e7b16e0 100644 --- a/be/src/http/web_page_handler.h +++ b/be/src/http/web_page_handler.h @@ -37,34 +37,53 @@ class EvHttpServer; class WebPageHandler : public HttpHandler { public: typedef std::map ArgumentMap; - typedef boost::function + typedef boost::function PageHandlerCallback; - WebPageHandler(EvHttpServer* http_server); - virtual ~WebPageHandler() { - } + WebPageHandler(EvHttpServer* http_server); + virtual ~WebPageHandler(); void handle(HttpRequest *req) override; - // Just use old code - void register_page(const std::string& path, const PageHandlerCallback& callback); + + // Register a route 'path'. + // If 'is_on_nav_bar' is true, a link to the page will be placed on the navbar + // in the header of styled pages. The link text is given by 'alias'. + void register_page(const std::string& path, const std::string& alias, + const PageHandlerCallback& callback, bool is_on_nav_bar); private: void bootstrap_page_header(std::stringstream* output); void bootstrap_page_footer(std::stringstream* output); void root_handler(const ArgumentMap& args, std::stringstream* output); - // all - class PageHandlers { + // Container class for a list of path handler callbacks for a single URL. + class PathHandler { public: - void add_callback(const PageHandlerCallback& callback) { - _callbacks.push_back(callback); - } - const std::vector& callbacks() const { - return _callbacks; - } + PathHandler(bool is_styled, bool is_on_nav_bar, std::string alias, + PageHandlerCallback callback) + : is_styled_(is_styled), + is_on_nav_bar_(is_on_nav_bar), + alias_(std::move(alias)), + callback_(std::move(callback)) {} + + bool is_styled() const { return is_styled_; } + bool is_on_nav_bar() const { return is_on_nav_bar_; } + const std::string& alias() const { return alias_; } + const PageHandlerCallback& callback() const { return callback_; } + private: - std::vector _callbacks; + // If true, the page appears is rendered styled. + bool is_styled_; + + // If true, the page appears in the navigation bar. + bool is_on_nav_bar_; + + // Alias used when displaying this link on the nav bar. + std::string alias_; + + // Callback to render output for this page. + PageHandlerCallback callback_; }; EvHttpServer* _http_server; @@ -73,7 +92,7 @@ class WebPageHandler : public HttpHandler { // Map of path to a PathHandler containing a list of handlers for that // path. More than one handler may register itself with a path so that many // components may contribute to a single page. - typedef std::map PageHandlersMap; + typedef std::map PageHandlersMap; PageHandlersMap _page_map; }; From d5d5306b770796ee77077754a4b5ef2e120f01ec Mon Sep 17 00:00:00 2001 From: Yingchun Lai <405403881@qq.com> Date: Sun, 5 Jul 2020 08:49:45 +0000 Subject: [PATCH 2/3] fix --- be/src/http/web_page_handler.h | 1 - 1 file changed, 1 deletion(-) diff --git a/be/src/http/web_page_handler.h b/be/src/http/web_page_handler.h index 2f16192e7b16e0..62d2f5cfcf53fd 100644 --- a/be/src/http/web_page_handler.h +++ b/be/src/http/web_page_handler.h @@ -45,7 +45,6 @@ class WebPageHandler : public HttpHandler { void handle(HttpRequest *req) override; - // Register a route 'path'. // If 'is_on_nav_bar' is true, a link to the page will be placed on the navbar // in the header of styled pages. The link text is given by 'alias'. From bb4e6ce366f6902fef6cf6dc263d44c30c4c598b Mon Sep 17 00:00:00 2001 From: Yingchun Lai <405403881@qq.com> Date: Sun, 5 Jul 2020 23:11:17 +0800 Subject: [PATCH 3/3] remove www conf --- be/src/common/config.h | 3 --- be/src/http/web_page_handler.cpp | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/be/src/common/config.h b/be/src/common/config.h index a29cbfee7a76c9..92bd75f0e6b29d 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -535,9 +535,6 @@ namespace config { // Whether to continue to start be when load tablet from header failed. CONF_Bool(ignore_load_tablet_failure, "false"); - - // Directory where BE's website files placed. - CONF_String(www_path, "${DORIS_HOME}/www"); } // namespace config } // namespace doris diff --git a/be/src/http/web_page_handler.cpp b/be/src/http/web_page_handler.cpp index 455ec272939bdd..cd318532c0e426 100644 --- a/be/src/http/web_page_handler.cpp +++ b/be/src/http/web_page_handler.cpp @@ -79,7 +79,7 @@ void WebPageHandler::handle(HttpRequest* req) { if (handler == nullptr) { // Try to handle static file request - do_file_response(config::www_path + req->raw_path(), req); + do_file_response(std::string(getenv("DORIS_HOME")) + "/www/" + req->raw_path(), req); // Has replied in do_file_response, so we return here. return; }