Refactor ES access with new es client (#4900)

* refact with new es client * add FTIndexUtilsTest * remove useless code * enbale limit and timeout * fix ut * address some comments
vesoft-inc · Dec 8, 2022 · b8e6101 · b8e6101
1 parent 55212e2
commit b8e6101
Show file tree

Hide file tree

Showing 53 changed files with 886 additions and 1,816 deletions.
diff --git a/src/common/expression/TextSearchExpression.h b/src/common/expression/TextSearchExpression.h
@@ -85,8 +85,8 @@ class TextSearchArgument final {
  std::string val_;
  std::string op_;
  int32_t fuzziness_{-2};
- int32_t limit_{100};
- int32_t timeout_{200};
+ int32_t limit_{-1};
+ int32_t timeout_{-1};
 };
 
 class TextSearchExpression : public Expression {

diff --git a/src/common/expression/test/CMakeLists.txt b/src/common/expression/test/CMakeLists.txt
@@ -33,7 +33,7 @@ set(expression_test_common_libs
  $<TARGET_OBJECTS:file_based_cluster_id_man_obj>
  $<TARGET_OBJECTS:process_obj>
  $<TARGET_OBJECTS:time_utils_obj>
- $<TARGET_OBJECTS:ft_es_graph_adapter_obj>
+ $<TARGET_OBJECTS:es_adapter_obj>
  $<TARGET_OBJECTS:ws_common_obj>
  $<TARGET_OBJECTS:version_obj>
  $<TARGET_OBJECTS:graph_session_obj>
@@ -47,6 +47,7 @@ set(expression_test_common_libs
  $<TARGET_OBJECTS:ssl_obj>
  $<TARGET_OBJECTS:memory_obj>
  $<TARGET_OBJECTS:gc_obj>
+ $<TARGET_OBJECTS:http_client_obj>
 )
 
 
@@ -95,6 +96,7 @@ nebula_add_test(
  gtest
  ${THRIFT_LIBRARIES}
  ${PROXYGEN_LIBRARIES}
+ curl
 )
 
 nebula_add_executable(
@@ -110,6 +112,7 @@ nebula_add_executable(
  boost_regex
  ${THRIFT_LIBRARIES}
  ${PROXYGEN_LIBRARIES}
+ curl
 )
 
 nebula_add_executable(

diff --git a/src/common/http/HttpClient.cpp b/src/common/http/HttpClient.cpp
@@ -31,29 +31,61 @@ HttpResponse HttpClient::get(const std::string& url) {
 }
 
 HttpResponse HttpClient::get(const std::string& url, const std::vector<std::string>& header) {
- return sendRequest(url, "GET", header, "");
+ return sendRequest(url, "GET", header, "", "", "");
+}
+
+HttpResponse HttpClient::get(const std::string& url,
+ const std::vector<std::string>& header,
+ const std::string& username,
+ const std::string& password) {
+ return sendRequest(url, "GET", header, "", username, password);
 }
 
 HttpResponse HttpClient::post(const std::string& url,
  const std::vector<std::string>& header,
  const std::string& body) {
- return sendRequest(url, "POST", header, body);
+ return sendRequest(url, "POST", header, body, "", "");
+}
+
+HttpResponse HttpClient::post(const std::string& url,
+ const std::vector<std::string>& header,
+ const std::string& body,
+ const std::string& username,
+ const std::string& password) {
+ return sendRequest(url, "POST", header, body, username, password);
 }
 
 HttpResponse HttpClient::delete_(const std::string& url, const std::vector<std::string>& header) {
- return sendRequest(url, "DELETE", header, "");
+ return sendRequest(url, "DELETE", header, "", "", "");
+}
+
+HttpResponse HttpClient::delete_(const std::string& url,
+ const std::vector<std::string>& header,
+ const std::string& username,
+ const std::string& password) {
+ return sendRequest(url, "DELETE", header, "", username, password);
 }
 
 HttpResponse HttpClient::put(const std::string& url,
  const std::vector<std::string>& header,
  const std::string& body) {
- return sendRequest(url, "PUT", header, body);
+ return sendRequest(url, "PUT", header, body, "", "");
+}
+
+HttpResponse HttpClient::put(const std::string& url,
+ const std::vector<std::string>& header,
+ const std::string& body,
+ const std::string& username,
+ const std::string& password) {
+ return sendRequest(url, "PUT", header, body, username, password);
 }
 
 HttpResponse HttpClient::sendRequest(const std::string& url,
  const std::string& method,
  const std::vector<std::string>& header,
- const std::string& body) {
+ const std::string& body,
+ const std::string& username,
+ const std::string& password) {
  CurlHandle::instance();
  HttpResponse resp;
  CURL* curl = curl_easy_init();
@@ -67,6 +99,7 @@ HttpResponse HttpClient::sendRequest(const std::string& url,
  setRespHeader(curl, resp.header);
  setRespBody(curl, resp.body);
  setTimeout(curl);
+ setAuth(curl, username, password);
  resp.curlCode = curl_easy_perform(curl);
  if (resp.curlCode != 0) {
  resp.curlMessage = std::string(curl_easy_strerror(resp.curlCode));
@@ -112,6 +145,15 @@ void HttpClient::setTimeout(CURL* curl) {
  curl_easy_setopt(curl, CURLOPT_TIMEOUT, 3);
 }
 
+void HttpClient::setAuth(CURL* curl, const std::string& username, const std::string& password) {
+ if (!username.empty()) {
+ curl_easy_setopt(curl, CURLOPT_USERNAME, username.c_str());
+ if (!password.empty()) {
+ curl_easy_setopt(curl, CURLOPT_PASSWORD, password.c_str());
+ }
+ }
+}
+
 size_t HttpClient::onWriteData(void* ptr, size_t size, size_t nmemb, void* stream) {
  if (ptr == nullptr || size == 0) {
  return 0;

diff --git a/src/common/http/HttpClient.h b/src/common/http/HttpClient.h
@@ -33,33 +33,63 @@ struct HttpResponse {
 class HttpClient {
  public:
  static HttpClient& instance();
+
  virtual ~HttpClient() = default;
+
  virtual HttpResponse get(const std::string& url);
+
  virtual HttpResponse get(const std::string& url, const std::vector<std::string>& headers);
 
+ virtual HttpResponse get(const std::string& url,
+ const std::vector<std::string>& headers,
+ const std::string& username,
+ const std::string& password);
+
  virtual HttpResponse post(const std::string& url,
  const std::vector<std::string>& headers,
  const std::string& body);
+
+ virtual HttpResponse post(const std::string& url,
+ const std::vector<std::string>& headers,
+ const std::string& body,
+ const std::string& username,
+ const std::string& password);
+
  virtual HttpResponse delete_(const std::string& url, const std::vector<std::string>& headers);
+
+ virtual HttpResponse delete_(const std::string& url,
+ const std::vector<std::string>& headers,
+ const std::string& username,
+ const std::string& password);
+
  virtual HttpResponse put(const std::string& url,
  const std::vector<std::string>& headers,
  const std::string& body);
 
+ virtual HttpResponse put(const std::string& url,
+ const std::vector<std::string>& headers,
+ const std::string& body,
+ const std::string& username,
+ const std::string& password);
+
  protected:
  HttpClient() = default;
 
  private:
  static HttpResponse sendRequest(const std::string& url,
  const std::string& method,
  const std::vector<std::string>& headers,
- const std::string& body);
+ const std::string& body,
+ const std::string& username,
+ const std::string& password);
  static void setUrl(CURL* curl, const std::string& url);
  static void setMethod(CURL* curl, const std::string& method);
  static curl_slist* setHeaders(CURL* curl, const std::vector<std::string>& headers);
 
  static void setRespBody(CURL* curl, const std::string& body);
  static void setRespHeader(CURL* curl, const std::string& header);
  static void setTimeout(CURL* curl);
+ static void setAuth(CURL* curl, const std::string& username, const std::string& password);
  static size_t onWriteData(void* ptr, size_t size, size_t nmemb, void* stream);
 };
 

diff --git a/src/common/plugin/fulltext/CMakeLists.txt b/src/common/plugin/fulltext/CMakeLists.txt
@@ -2,16 +2,6 @@
 #
 # This source code is licensed under Apache 2.0 License.
 
-nebula_add_library(
- ft_es_graph_adapter_obj OBJECT
- elasticsearch/ESGraphAdapter.cpp
-)
-
-nebula_add_library(
- ft_es_storage_adapter_obj OBJECT
- elasticsearch/ESStorageAdapter.cpp
-)
-
 nebula_add_library(
  es_adapter_obj OBJECT
  elasticsearch/ESAdapter.cpp

diff --git a/src/common/plugin/fulltext/FTGraphAdapter.h b/src/common/plugin/fulltext/FTGraphAdapter.h
diff --git a/src/common/plugin/fulltext/FTStorageAdapter.h b/src/common/plugin/fulltext/FTStorageAdapter.h