Support get_config & Optimize handling http request (#3212)

dbms/src/Debug/ClusterManage.cpp

@@ -9,25 +9,47 @@
 #include <Storages/Transaction/Region.h>
 #include <Storages/Transaction/RegionTable.h>
 #include <Storages/Transaction/TMTContext.h>
+#include <fmt/core.h>
 
 namespace DB
 {
-
 namespace ErrorCodes
 {
 extern const int BAD_ARGUMENTS;
 extern const int UNKNOWN_TABLE;
 } // namespace ErrorCodes
 
+HttpRequestRes HandleHttpRequestTestShow(
+    EngineStoreServerWrap *,
+    std::string_view path,
+    const std::string & api_name,
+    std::string_view query,
+    std::string_view body)
+{
+    auto * res = RawCppString::New(fmt::format(
+        "api_name: {}\npath: {}\nquery: {}\nbody: {}",
+        api_name,
+        path,
+        query,
+        body));
+    return HttpRequestRes{
+        .status = HttpRequestStatus::Ok,
+        .res = CppStrWithView{
+            .inner = GenRawCppPtr(res, RawCppPtrTypeImpl::String),
+            .view = BaseBuffView{res->data(), res->size()}}};
+}
 
-HttpRequestRes HandleHttpRequestSyncStatus(EngineStoreServerWrap * server, BaseBuffView path_, const std::string & method_name)
+HttpRequestRes HandleHttpRequestSyncStatus(
+    EngineStoreServerWrap * server,
+    std::string_view path,
+    const std::string & api_name,
+    std::string_view,
+    std::string_view)
 {
     HttpRequestStatus status = HttpRequestStatus::Ok;
     TableID table_id = 0;
     {
-        std::string_view path(path_.data, path_.len);
-
-        std::string table_id_str(path.substr(method_name.size()));
+        std::string table_id_str(path.substr(api_name.size()));
         try
         {
             table_id = std::stoll(table_id_str);
@@ -62,26 +84,38 @@ HttpRequestRes HandleHttpRequestSyncStatus(EngineStoreServerWrap * server, BaseB
         ss << region_id << ' ';
     ss << std::endl;
 
-    auto s = RawCppString::New(ss.str());
-    return HttpRequestRes{.status = status,
+    auto * s = RawCppString::New(ss.str());
+    return HttpRequestRes{
+        .status = status,
         .res = CppStrWithView{.inner = GenRawCppPtr(s, RawCppPtrTypeImpl::String), .view = BaseBuffView{s->data(), s->size()}}};
 }
 
-HttpRequestRes HandleHttpRequestStoreStatus(EngineStoreServerWrap * server, BaseBuffView, const std::string &)
+HttpRequestRes HandleHttpRequestStoreStatus(
+    EngineStoreServerWrap * server,
+    std::string_view,
+    const std::string &,
+    std::string_view,
+    std::string_view)
 {
-    auto name = RawCppString::New(IntoStoreStatusName(server->tmt->getStoreStatus(std::memory_order_relaxed)));
-    return HttpRequestRes{.status = HttpRequestStatus::Ok,
-        .res = CppStrWithView{.inner = GenRawCppPtr(name, RawCppPtrTypeImpl::String), .view = BaseBuffView{name->data(), name->size()}}};
+    auto * name = RawCppString::New(IntoStoreStatusName(server->tmt->getStoreStatus(std::memory_order_relaxed)));
+    return HttpRequestRes{
+        .status = HttpRequestStatus::Ok,
+        .res = CppStrWithView{
+            .inner = GenRawCppPtr(name, RawCppPtrTypeImpl::String),
+            .view = BaseBuffView{name->data(), name->size()}}};
 }
 
-typedef HttpRequestRes (*HANDLE_HTTP_URI_METHOD)(EngineStoreServerWrap *, BaseBuffView, const std::string &);
-static const std::map<std::string, HANDLE_HTTP_URI_METHOD> AVAILABLE_HTTP_URI
-    = {{"/tiflash/sync-status/", HandleHttpRequestSyncStatus}, {"/tiflash/store-status", HandleHttpRequestStoreStatus}};
+using HANDLE_HTTP_URI_METHOD = HttpRequestRes (*)(EngineStoreServerWrap *, std::string_view, const std::string &, std::string_view, std::string_view);
+
+static const std::map<std::string, HANDLE_HTTP_URI_METHOD> AVAILABLE_HTTP_URI = {
+    {"/tiflash/sync-status/", HandleHttpRequestSyncStatus},
+    {"/tiflash/store-status", HandleHttpRequestStoreStatus},
+    {"/tiflash/test-show", HandleHttpRequestTestShow}};
 
 uint8_t CheckHttpUriAvailable(BaseBuffView path_)
 {
     std::string_view path(path_.data, path_.len);
-    for (auto & [str, method] : AVAILABLE_HTTP_URI)
+    for (const auto & [str, method] : AVAILABLE_HTTP_URI)
     {
         std::ignore = method;
         if (path.size() >= str.size() && path.substr(0, str.size()) == str)
@@ -90,14 +124,14 @@ uint8_t CheckHttpUriAvailable(BaseBuffView path_)
     return false;
 }
 
-HttpRequestRes HandleHttpRequest(EngineStoreServerWrap * server, BaseBuffView path_)
+HttpRequestRes HandleHttpRequest(EngineStoreServerWrap * server, BaseBuffView path_, BaseBuffView query, BaseBuffView body)
 {
     std::string_view path(path_.data, path_.len);
-    for (auto & [str, method] : AVAILABLE_HTTP_URI)
+    for (const auto & [str, method] : AVAILABLE_HTTP_URI)
     {
         if (path.size() >= str.size() && path.substr(0, str.size()) == str)
         {
-            return method(server, path_, str);
+            return method(server, path, str, std::string_view(query.data, query.len), std::string_view(body.data, body.len));
         }
     }
     return HttpRequestRes{.status = HttpRequestStatus::ErrorParam, .res = CppStrWithView{.inner = GenRawCppPtr(), .view = BaseBuffView{}}};
@@ -124,7 +158,10 @@ inline std::string ToPdKey(const char * key, const size_t len)
     return res;
 }
 
-inline std::string ToPdKey(const std::string & key) { return ToPdKey(key.data(), key.size()); }
+inline std::string ToPdKey(const std::string & key)
+{
+    return ToPdKey(key.data(), key.size());
+}
 
 inline std::string FromPdKey(const char * key, const size_t len)
 {
@@ -171,7 +208,7 @@ void ClusterManage::findRegionByRange(Context & context, const ASTs & args, Prin
     auto end = FromPdKey(end_key.data(), end_key.size());
     RegionMap regions;
     kvstore->handleRegionsByRangeOverlap(RegionRangeKeys::makeComparableKeys(std::move(start), std::move(end)),
-        [&regions](RegionMap regions_, const KVStoreTaskLock &) { regions = std::move(regions_); });
+                                         [&regions](RegionMap regions_, const KVStoreTaskLock &) { regions = std::move(regions_); });
 
     output(toString(regions.size()));
     if (mode == ID_LIST)
@@ -191,7 +228,7 @@ void ClusterManage::checkTableOptimize(DB::Context & context, const DB::ASTs & a
         throw Exception("Args not matched, should be: table-id, threshold", ErrorCodes::BAD_ARGUMENTS);
 
     auto & tmt = context.getTMTContext();
-    TableID table_id = (TableID)safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[0]).value);
+    TableID table_id = static_cast<TableID>(safeGet<UInt64>(typeid_cast<const ASTLiteral &>(*args[0]).value));
     auto a = typeid_cast<const ASTLiteral &>(*args[1]).value.safeGet<DecimalField<Decimal32>>();
     tmt.getRegionTable().checkTableOptimize(table_id, a.getValue().toFloat<Float32>(a.getScale()));
 }

dbms/src/Server/DTTool/DTTool.h

@@ -3,8 +3,6 @@
 #include <Encryption/DataKeyManager.h>
 #include <Encryption/MockKeyManager.h>
 #include <Poco/File.h>
-#include <RaftStoreProxyFFI/ProxyFFI.h>
-#include <RaftStoreProxyFFI/VersionCheck.h>
 #include <Server/IServer.h>
 #include <Storages/Transaction/ProxyFFI.h>
 #include <daemon/BaseDaemon.h>
@@ -205,9 +203,7 @@ int CLIService<Func, Args>::main(const std::vector<std::string> &)
     using namespace DB;
     TiFlashProxyConfig proxy_conf(config());
     EngineStoreServerWrap tiflash_instance_wrap{};
-    auto helper = getEngineStoreServerHelper(
-        RAFT_STORE_PROXY_MAGIC_NUMBER,
-        RAFT_STORE_PROXY_VERSION,
+    auto helper = GetEngineStoreServerHelper(
         &tiflash_instance_wrap);
 
     typename RaftStoreProxyRunner::RunRaftStoreProxyParms parms{&helper, proxy_conf};

dbms/src/Server/Server.cpp

@@ -37,7 +37,6 @@
 #include <Poco/Net/NetException.h>
 #include <Poco/StringTokenizer.h>
 #include <Poco/Timestamp.h>
-#include <RaftStoreProxyFFI/VersionCheck.h>
 #include <Server/RaftConfigParser.h>
 #include <Server/StorageConfigParser.h>
 #include <Server/UserConfigParser.h>
@@ -866,9 +865,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
 
     TiFlashProxyConfig proxy_conf(config());
     EngineStoreServerWrap tiflash_instance_wrap{};
-    auto helper = getEngineStoreServerHelper(
-        RAFT_STORE_PROXY_MAGIC_NUMBER,
-        RAFT_STORE_PROXY_VERSION,
+    auto helper = GetEngineStoreServerHelper(
         &tiflash_instance_wrap);
 
     RaftStoreProxyRunner proxy_runner(RaftStoreProxyRunner::RunRaftStoreProxyParms{&helper, proxy_conf}, log);

dbms/src/Storages/Transaction/ProxyFFI.cpp

@@ -236,7 +236,6 @@ BatchReadIndexRes TiFlashRaftProxyHelper::batchReadIndex(const std::vector<kvrpc
         req_strs.emplace_back(r.SerializeAsString());
     }
     CppStrVec data(std::move(req_strs));
-    assert(req_strs.empty());
     auto outer_view = data.intoOuterView();
     BatchReadIndexRes res;
     res.reserve(req.size());
@@ -413,4 +412,30 @@ void SetServerInfoResp(BaseBuffView view, RawVoidPtr ptr)
     reinterpret_cast<ServerInfoResponse *>(ptr)->ParseFromArray(view.data, view.len);
 }
 
+CppStrWithView GetConfig(EngineStoreServerWrap * server, [[maybe_unused]] uint8_t full)
+{
+    std::string config_file_path;
+    try
+    {
+        config_file_path = server->tmt->getContext().getConfigRef().getString("config-file");
+        std::ifstream stream(config_file_path);
+        if (!stream)
+            return CppStrWithView{.inner = GenRawCppPtr(), .view = BaseBuffView{}};
+        auto * s = RawCppString::New((std::istreambuf_iterator<char>(stream)),
+                                     std::istreambuf_iterator<char>());
+        stream.close();
+        /** the returned str must be formated as TOML, proxy will parse and show in form of JASON.
+         *  curl `http://{status-addr}/config`, got:
+         *  {"raftstore-proxy":xxxx,"engine-store":xxx}
+         *
+         *  if proxy can NOT parse it, return 500 Internal Server Error.
+         * */
+        return CppStrWithView{.inner = GenRawCppPtr(s, RawCppPtrTypeImpl::String), .view = BaseBuffView{s->data(), s->size()}};
+    }
+    catch (...)
+    {
+        return CppStrWithView{.inner = GenRawCppPtr(), .view = BaseBuffView{}};
+    }
+}
+
 } // namespace DB

dbms/src/Storages/Transaction/ProxyFFI.h

@@ -2,6 +2,7 @@
 
 #include <RaftStoreProxyFFI/EncryptionFFI.h>
 #include <RaftStoreProxyFFI/ProxyFFI.h>
+#include <RaftStoreProxyFFI/VersionCheck.h>
 #include <Storages/Transaction/ColumnFamily.h>
 
 #include <atomic>
@@ -77,23 +78,22 @@ RawCppPtr PreHandleSnapshot(
     uint64_t index,
     uint64_t term);
 void ApplyPreHandledSnapshot(EngineStoreServerWrap * server, void * res, RawCppPtrType type);
-HttpRequestRes HandleHttpRequest(EngineStoreServerWrap *, BaseBuffView);
+HttpRequestRes HandleHttpRequest(EngineStoreServerWrap *, BaseBuffView path, BaseBuffView query, BaseBuffView body);
 uint8_t CheckHttpUriAvailable(BaseBuffView);
 void GcRawCppPtr(void * ptr, RawCppPtrType type);
 void InsertBatchReadIndexResp(RawVoidPtr, BaseBuffView, uint64_t);
 void SetServerInfoResp(BaseBuffView, RawVoidPtr);
 BaseBuffView strIntoView(const std::string * str_ptr);
+CppStrWithView GetConfig(EngineStoreServerWrap *, uint8_t full);
 }
 
-inline EngineStoreServerHelper getEngineStoreServerHelper(
-    uint32_t magic_number,
-    uint64_t version,
+inline EngineStoreServerHelper GetEngineStoreServerHelper(
     EngineStoreServerWrap * tiflash_instance_wrap)
 {
     return EngineStoreServerHelper{
         // a special number, also defined in proxy
-        .magic_number = magic_number,
-        .version = version,
+        .magic_number = RAFT_STORE_PROXY_MAGIC_NUMBER,
+        .version = RAFT_STORE_PROXY_VERSION,
         .inner = tiflash_instance_wrap,
         .fn_gen_cpp_string = GenCppRawString,
         .fn_handle_write_raft_cmd = HandleWriteRaftCmd,
@@ -110,6 +110,7 @@ inline EngineStoreServerHelper getEngineStoreServerHelper(
         .fn_gc_raw_cpp_ptr = GcRawCppPtr,
         .fn_insert_batch_read_index_resp = InsertBatchReadIndexResp,
         .fn_set_server_info_resp = SetServerInfoResp,
+        .fn_get_config = GetConfig,
     };
 }
 } // namespace DB