Cocurrency otlp http session (open-telemetry#1317)

owent · May 11, 2022 · 08a12b5 · 08a12b5
1 parent c614258
commit 08a12b5
Show file tree

Hide file tree

Showing 17 changed files with 1,805 additions and 547 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -17,6 +17,8 @@ Increment the:
 
 * [SDK] Async Batch Span/Log processor with max async support ([#1306](https://github.com/open-telemetry/opentelemetry-cpp/pull/1306))
 * [EXPORTER] OTLP http exporter allow concurrency session ([#1209](https://github.com/open-telemetry/opentelemetry-cpp/pull/1209))
+* [EXT] `curl::HttpClient` use `curl_multi_handle` instead of creating a thread
+ for every request and it's able to reuse connections now. ([#1317](https://github.com/open-telemetry/opentelemetry-cpp/pull/1317))
 
 ## [1.3.0] 2022-04-11
 

diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_client.h
@@ -79,7 +79,10 @@ struct OtlpHttpClientOptions
  OtlpHeaders http_headers = GetOtlpDefaultHeaders();
 
  // Concurrent requests
- std::size_t max_concurrent_requests = 8;
+ std::size_t max_concurrent_requests = 64;
+
+ // Concurrent requests
+ std::size_t max_requests_per_connection = 8;
 
  inline OtlpHttpClientOptions(nostd::string_view input_url,
  HttpRequestContentType input_content_type,
@@ -88,15 +91,17 @@ struct OtlpHttpClientOptions
  bool input_console_debug,
  std::chrono::system_clock::duration input_timeout,
  const OtlpHeaders &input_http_headers,
- std::size_t input_concurrent_sessions = 8)
+ std::size_t input_concurrent_sessions = 64,
+ std::size_t input_max_requests_per_connection = 8)
  : url(input_url),
  content_type(input_content_type),
  json_bytes_mapping(input_json_bytes_mapping),
  use_json_name(input_use_json_name),
  console_debug(input_console_debug),
  timeout(input_timeout),
  http_headers(input_http_headers),
- max_concurrent_requests(input_concurrent_sessions)
+ max_concurrent_requests(input_concurrent_sessions),
+ max_requests_per_connection(input_max_requests_per_connection)
  {}
 };
 

diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_exporter.h
@@ -55,7 +55,10 @@ struct OtlpHttpExporterOptions
 #ifdef ENABLE_ASYNC_EXPORT
  // Concurrent requests
  // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#otlpgrpc-concurrent-requests
- std::size_t max_concurrent_requests = 8;
+ std::size_t max_concurrent_requests = 64;
+
+ // Concurrent requests
+ std::size_t max_requests_per_connection = 8;
 #endif
 };
 

diff --git a/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_exporter.h b/exporters/otlp/include/opentelemetry/exporters/otlp/otlp_http_log_exporter.h
@@ -55,7 +55,10 @@ struct OtlpHttpLogExporterOptions
 # ifdef ENABLE_ASYNC_EXPORT
  // Concurrent requests
  // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/protocol/otlp.md#otlpgrpc-concurrent-requests
- std::size_t max_concurrent_requests = 8;
+ std::size_t max_concurrent_requests = 64;
+
+ // Concurrent requests
+ std::size_t max_requests_per_connection = 8;
 # endif
 };
 

diff --git a/exporters/otlp/src/otlp_http_client.cc b/exporters/otlp/src/otlp_http_client.cc
@@ -649,7 +649,9 @@ void ConvertListFieldToJson(nlohmann::json &value,
 
 OtlpHttpClient::OtlpHttpClient(OtlpHttpClientOptions &&options)
  : is_shutdown_(false), options_(options), http_client_(http_client::HttpClientFactory::Create())
-{}
+{
+ http_client_->SetMaxSessionsPerConnection(options_.max_requests_per_connection);
+}
 
 OtlpHttpClient::~OtlpHttpClient()
 {
@@ -682,7 +684,9 @@ OtlpHttpClient::~OtlpHttpClient()
 OtlpHttpClient::OtlpHttpClient(OtlpHttpClientOptions &&options,
  std::shared_ptr<ext::http::client::HttpClient> http_client)
  : is_shutdown_(false), options_(options), http_client_(http_client)
-{}
+{
+ http_client_->SetMaxSessionsPerConnection(options_.max_requests_per_connection);
+}
 
 // ----------------------------- HTTP Client methods ------------------------------
 opentelemetry::sdk::common::ExportResult OtlpHttpClient::Export(

diff --git a/exporters/otlp/src/otlp_http_exporter.cc b/exporters/otlp/src/otlp_http_exporter.cc
@@ -34,7 +34,8 @@ OtlpHttpExporter::OtlpHttpExporter(const OtlpHttpExporterOptions &options)
  options.http_headers
 #ifdef ENABLE_ASYNC_EXPORT
  ,
- options.max_concurrent_requests
+ options.max_concurrent_requests,
+ options.max_requests_per_connection
 #endif
  )))
 {}

diff --git a/exporters/otlp/src/otlp_http_log_exporter.cc b/exporters/otlp/src/otlp_http_log_exporter.cc
@@ -36,7 +36,8 @@ OtlpHttpLogExporter::OtlpHttpLogExporter(const OtlpHttpLogExporterOptions &optio
  options.http_headers
 # ifdef ENABLE_ASYNC_EXPORT
  ,
- options.max_concurrent_requests
+ options.max_concurrent_requests,
+ options.max_requests_per_connection
 # endif
  )))
 {}