From a42793e3f25a1ea1a6a4be753814b06746f8d5f7 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 29 Oct 2024 23:03:05 +0100 Subject: [PATCH 01/21] http_client: initial commit of the new http client component Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_http_client.h | 215 +++++ include/fluent-bit/flb_http_client_http1.h | 46 + include/fluent-bit/flb_http_client_http2.h | 46 + src/flb_http_client.c | 1003 +++++++++++++++++++- src/flb_http_client_http1.c | 727 ++++++++++++++ src/flb_http_client_http2.c | 707 ++++++++++++++ 6 files changed, 2740 insertions(+), 4 deletions(-) create mode 100644 include/fluent-bit/flb_http_client_http1.h create mode 100644 include/fluent-bit/flb_http_client_http2.h create mode 100644 src/flb_http_client_http1.c create mode 100644 src/flb_http_client_http2.c diff --git a/include/fluent-bit/flb_http_client.h b/include/fluent-bit/flb_http_client.h index 8d7058622e0..243ed253da8 100644 --- a/include/fluent-bit/flb_http_client.h +++ b/include/fluent-bit/flb_http_client.h @@ -21,8 +21,20 @@ #define FLB_HTTP_CLIENT_H #include +#include #include #include +#include +#include +#include + +#define HTTP_CLIENT_SUCCESS 0 +#define HTTP_CLIENT_PROVIDER_ERROR -1 + +#define FLB_HTTP_CLIENT_FLAG_KEEPALIVE (((uint64_t) 1) << 0) +#define FLB_HTTP_CLIENT_FLAG_AUTO_DEFLATE (((uint64_t) 1) << 1) +#define FLB_HTTP_CLIENT_FLAG_AUTO_INFLATE (((uint64_t) 1) << 2) +#define FLB_HTTP_CLIENT_FLAG_STREAM_BODY (((uint64_t) 1) << 3) /* Buffer size */ #define FLB_HTTP_BUF_SIZE 2048 @@ -63,6 +75,39 @@ #define FLB_HTTP_HEADER_CONNECTION "Connection" #define FLB_HTTP_HEADER_KA "keep-alive" +#define FLB_HTTP_CLIENT_HEADER_ARRAY 0 +#define FLB_HTTP_CLIENT_HEADER_LIST 1 +#define FLB_HTTP_CLIENT_HEADER_CONFIG_MAP_LIST 2 + +#define FLB_HTTP_CLIENT_ARGUMENT_TYPE_TERMINATOR 0 +#define FLB_HTTP_CLIENT_ARGUMENT_TYPE_METHOD 1 +#define FLB_HTTP_CLIENT_ARGUMENT_TYPE_HOST 2 +#define FLB_HTTP_CLIENT_ARGUMENT_TYPE_URI 3 +#define FLB_HTTP_CLIENT_ARGUMENT_TYPE_URL 4 +#define FLB_HTTP_CLIENT_ARGUMENT_TYPE_USER_AGENT 5 +#define FLB_HTTP_CLIENT_ARGUMENT_TYPE_CONTENT_TYPE 6 +#define FLB_HTTP_CLIENT_ARGUMENT_TYPE_BODY 7 +#define FLB_HTTP_CLIENT_ARGUMENT_TYPE_HEADERS 8 +#define FLB_HTTP_CLIENT_ARGUMENT_TYPE_AUTH_BASIC 9 +#define FLB_HTTP_CLIENT_ARGUMENT_TYPE_AUTH_BEARER_TOKEN 10 +#define FLB_HTTP_CLIENT_ARGUMENT_TYPE_AUTH_SIGNV4 11 + +#ifdef FLB_SYSTEM_WINDOWS +#ifdef _WIN64 +typedef size_t * flb_http_client_size_t; +typedef size_t * flb_http_client_int64_t; +typedef size_t * flb_http_client_uint64_t; +#else +typedef size_t * flb_http_client_size_t; +typedef int64_t flb_http_client_int64_t; +typedef uint64_t flb_http_client_uint64_t; +#endif +#else +typedef size_t flb_http_client_size_t; +typedef int64_t flb_http_client_int64_t; +typedef uint64_t flb_http_client_uint64_t; +#endif + struct flb_http_client_response { int status; /* HTTP response status */ int content_length; /* Content length set by headers */ @@ -137,6 +182,94 @@ struct flb_http_client { void *cb_ctx; }; +struct flb_http_client_ng { + struct cfl_list sessions; + + uint16_t port; + uint64_t flags; + int protocol_version; + + int releasable; + void *user_data; + + struct flb_upstream *upstream; + struct flb_upstream_ha *upstream_ha; + + flb_lock_t lock; +}; + +struct flb_http_client_session { + struct flb_http1_client_session http1; + struct flb_http2_client_session http2; + struct cfl_list streams; + + int protocol_version; + + cfl_sds_t incoming_data; + cfl_sds_t outgoing_data; + + int releasable; + + struct cfl_list response_queue; + + int stream_sequence_number; + + struct flb_upstream_node *upstream_node; + struct flb_connection *connection; + struct flb_http_client_ng *parent; + + struct cfl_list _head; +}; + +struct flb_aws_provider; + +int flb_http_client_ng_init(struct flb_http_client_ng *client, + struct flb_upstream_ha *upstream_ha, + struct flb_upstream *upstream, + int protocol_version, + uint64_t flags); + +struct flb_http_client_ng *flb_http_client_ng_create( + struct flb_upstream_ha *upstream_ha, + struct flb_upstream *upstream, + int protocol_version, + uint64_t flags); + +void flb_http_client_ng_destroy(struct flb_http_client_ng *client); + +int flb_http_client_session_init(struct flb_http_client_session *session, + struct flb_http_client_ng *client, + int protocol_version, + struct flb_connection *connection); + +struct flb_http_client_session *flb_http_client_session_create( + struct flb_http_client_ng *client, + int protocol_version, + struct flb_connection *connection); + +struct flb_http_client_session *flb_http_client_session_begin( + struct flb_http_client_ng *client); + +void flb_http_client_session_destroy(struct flb_http_client_session *session); + +struct flb_http_request *flb_http_client_request_begin( + struct flb_http_client_session *session); + +struct flb_http_response *flb_http_client_request_execute( + struct flb_http_request *request); + +struct flb_http_response *flb_http_client_request_execute_step( + struct flb_http_request *request); + +void flb_http_client_request_destroy(struct flb_http_request *request, + int destroy_session); + +int flb_http_client_session_ingest(struct flb_http_client_session *session, + unsigned char *buffer, + size_t length); + + + void flb_http_client_debug(struct flb_http_client *c, struct flb_callback *cb_ctx); @@ -164,6 +297,7 @@ int flb_http_set_callback_context(struct flb_http_client *c, int flb_http_get_response_data(struct flb_http_client *c, size_t bytes_consumed); int flb_http_do_request(struct flb_http_client *c, size_t *bytes); + int flb_http_do(struct flb_http_client *c, size_t *bytes); int flb_http_client_proxy_connect(struct flb_connection *u_conn); void flb_http_client_destroy(struct flb_http_client *c); @@ -175,4 +309,85 @@ int flb_http_strip_port_from_host(struct flb_http_client *c); int flb_http_allow_duplicated_headers(struct flb_http_client *c, int allow); int flb_http_client_debug_property_is_valid(char *key, char *val); + +#define FLB_HTTP_CLIENT_ARGUMENT(argument_type, ...) \ + (flb_http_client_size_t) argument_type, __VA_ARGS__ + +#define FLB_HTTP_CLIENT_ARGUMENT_TERMINATOR() \ + (flb_http_client_size_t) FLB_HTTP_CLIENT_ARGUMENT_TYPE_TERMINATOR + +#define FLB_HTTP_CLIENT_ARGUMENT_METHOD(method) \ + FLB_HTTP_CLIENT_ARGUMENT(FLB_HTTP_CLIENT_ARGUMENT_TYPE_METHOD, \ + (flb_http_client_size_t) method) + +#define FLB_HTTP_CLIENT_ARGUMENT_HOST(host) \ + FLB_HTTP_CLIENT_ARGUMENT(FLB_HTTP_CLIENT_ARGUMENT_TYPE_HOST, \ + host) + +#define FLB_HTTP_CLIENT_ARGUMENT_URL(url) \ + FLB_HTTP_CLIENT_ARGUMENT(FLB_HTTP_CLIENT_ARGUMENT_TYPE_URL, \ + url) + +#define FLB_HTTP_CLIENT_ARGUMENT_URI(uri) \ + FLB_HTTP_CLIENT_ARGUMENT(FLB_HTTP_CLIENT_ARGUMENT_TYPE_URI, \ + uri) + +#define FLB_HTTP_CLIENT_ARGUMENT_USER_AGENT(user_agent) \ + FLB_HTTP_CLIENT_ARGUMENT(FLB_HTTP_CLIENT_ARGUMENT_TYPE_USER_AGENT, \ + user_agent) + +#define FLB_HTTP_CLIENT_ARGUMENT_CONTENT_TYPE(content_type) \ + FLB_HTTP_CLIENT_ARGUMENT(FLB_HTTP_CLIENT_ARGUMENT_TYPE_CONTENT_TYPE, \ + content_type) + +#define FLB_HTTP_CLIENT_ARGUMENT_BODY(buffer, length, compression_algorithm) \ + FLB_HTTP_CLIENT_ARGUMENT(FLB_HTTP_CLIENT_ARGUMENT_TYPE_BODY, \ + buffer, \ + (flb_http_client_size_t) length, \ + compression_algorithm) + +#define FLB_HTTP_CLIENT_ARGUMENT_HEADERS(data_type, headers) \ + FLB_HTTP_CLIENT_ARGUMENT(FLB_HTTP_CLIENT_ARGUMENT_TYPE_HEADERS, \ + (flb_http_client_size_t) data_type, \ + headers) + +#define FLB_HTTP_CLIENT_ARGUMENT_BASIC_AUTHORIZATION(username, password) \ + FLB_HTTP_CLIENT_ARGUMENT(FLB_HTTP_CLIENT_ARGUMENT_TYPE_AUTH_BASIC, \ + username, \ + password) + +#define FLB_HTTP_CLIENT_ARGUMENT_BEARER_TOKEN(token) \ + FLB_HTTP_CLIENT_ARGUMENT(FLB_HTTP_CLIENT_ARGUMENT_TYPE_AUTH_BEARER_TOKEN, \ + token) + +#define FLB_HTTP_CLIENT_ARGUMENT_SIGNV4(aws_region, aws_service, aws_provider) \ + FLB_HTTP_CLIENT_ARGUMENT(FLB_HTTP_CLIENT_ARGUMENT_TYPE_AUTH_SIGNV4, \ + aws_region, \ + aws_service, \ + aws_provider) + +int flb_http_request_set_parameters_internal( + struct flb_http_request *request, + va_list arguments); + +int flb_http_request_set_parameters_unsafe( + struct flb_http_request *request, + ...); + +struct flb_http_request *flb_http_client_request_builder_unsafe( + struct flb_http_client_ng *client, + ...); + +#define flb_http_client_request_builder(client, ...) \ + flb_http_client_request_builder_unsafe( \ + client, \ + __VA_ARGS__, \ + FLB_HTTP_CLIENT_ARGUMENT_TERMINATOR()); + +#define flb_http_request_set_parameters(request, ...) \ + flb_http_request_set_parameters_unsafe( \ + request, \ + __VA_ARGS__, \ + FLB_HTTP_CLIENT_ARGUMENT_TERMINATOR()); + #endif diff --git a/include/fluent-bit/flb_http_client_http1.h b/include/fluent-bit/flb_http_client_http1.h new file mode 100644 index 00000000000..bf1d60fcd85 --- /dev/null +++ b/include/fluent-bit/flb_http_client_http1.h @@ -0,0 +1,46 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_HTTP_CLIENT_HTTP1_H +#define FLB_HTTP_CLIENT_HTTP1_H + +#include + +struct flb_http_client; +struct flb_http_client_session; + +struct flb_http1_client_session { + struct flb_http_client *inner_session; + int initialized; + struct flb_http_client_session *parent; +}; + +int flb_http1_client_session_init(struct flb_http1_client_session *session); + +void flb_http1_client_session_destroy(struct flb_http1_client_session *session); + +int flb_http1_client_session_ingest(struct flb_http1_client_session *session, + unsigned char *buffer, + size_t length); + +int flb_http1_request_begin(struct flb_http_request *request); + +int flb_http1_request_commit(struct flb_http_request *request); + +#endif diff --git a/include/fluent-bit/flb_http_client_http2.h b/include/fluent-bit/flb_http_client_http2.h new file mode 100644 index 00000000000..dbf3b85a22b --- /dev/null +++ b/include/fluent-bit/flb_http_client_http2.h @@ -0,0 +1,46 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_HTTP_CLIENT_HTTP2_H +#define FLB_HTTP_CLIENT_HTTP2_H + +#include +#include + +struct flb_http_client_session; + +struct flb_http2_client_session { + nghttp2_session *inner_session; + int initialized; + struct flb_http_client_session *parent; +}; + +int flb_http2_client_session_init(struct flb_http2_client_session *session); + +void flb_http2_client_session_destroy(struct flb_http2_client_session *session); + +int flb_http2_client_session_ingest(struct flb_http2_client_session *session, + unsigned char *buffer, + size_t length); + +int flb_http2_request_begin(struct flb_http_request *request); + +int flb_http2_request_commit(struct flb_http_request *request); + +#endif diff --git a/src/flb_http_client.c b/src/flb_http_client.c index 72a1fadbbd1..80a643678ee 100644 --- a/src/flb_http_client.c +++ b/src/flb_http_client.c @@ -37,12 +37,14 @@ #include #include #include +#include +#include #include #include #include #include - - +#include +#include void flb_http_client_debug(struct flb_http_client *c, struct flb_callback *cb_ctx) @@ -373,6 +375,7 @@ static int process_chunked_data(struct flb_http_client *c) /* Is this the last chunk ? */ if ((val - 2 == 0)) { + /* Update payload size */ return FLB_HTTP_OK; } @@ -385,6 +388,7 @@ static int process_chunked_data(struct flb_http_client *c) if (found_full_chunk == FLB_TRUE) { return FLB_HTTP_CHUNK_AVAILABLE; } + return FLB_HTTP_MORE; } @@ -1281,8 +1285,9 @@ int flb_http_get_response_data(struct flb_http_client *c, size_t bytes_consumed) ssize_t available; size_t out_size; - // if the caller has consumed some of the payload (via bytes_consumed) - // we consume those bytes off the payload + /* If the caller has consumed some of the payload (via bytes_consumed) + * we consume those bytes off the payload + */ if( bytes_consumed > 0 ) { if(bytes_consumed > c->resp.payload_size) { flb_error("[http_client] attempting to consume more bytes than " @@ -1467,3 +1472,993 @@ void flb_http_client_destroy(struct flb_http_client *c) flb_free((void *)c->proxy.host); flb_free(c); } + + + + + + + + + + + + + + + + + + +static int flb_http_client_session_read(struct flb_http_client_session *session); +static int flb_http_client_session_write(struct flb_http_client_session *session); + + + + + + + + + + +int flb_http_client_ng_init(struct flb_http_client_ng *client, + struct flb_upstream_ha *upstream_ha, + struct flb_upstream *upstream, + int protocol_version, + uint64_t flags) +{ + memset(client, 0, sizeof(struct flb_http_client_ng)); + + client->protocol_version = protocol_version; + client->upstream_ha = upstream_ha; + client->upstream = upstream; + client->flags = flags; + + cfl_list_init(&client->sessions); + + if (protocol_version == HTTP_PROTOCOL_VERSION_AUTODETECT) { + if (upstream->base.tls_context != NULL) { + flb_tls_set_alpn(upstream->base.tls_context, "h2,http/1.1,http/1.0"); + } + } + else if (protocol_version == HTTP_PROTOCOL_VERSION_20) { + if (upstream->base.tls_context != NULL) { + flb_tls_set_alpn(upstream->base.tls_context, "h2"); + } + } + else if (protocol_version == HTTP_PROTOCOL_VERSION_11) { + if (upstream->base.tls_context != NULL) { + flb_tls_set_alpn(upstream->base.tls_context, "http/1.1,http/1.0"); + } + } + else if (protocol_version <= HTTP_PROTOCOL_VERSION_10) { + if (upstream->base.tls_context != NULL) { + flb_tls_set_alpn(upstream->base.tls_context, "http/1.0"); + } + } + + flb_lock_init(&client->lock); + + return 0; +} + +struct flb_http_client_ng *flb_http_client_ng_create( + struct flb_upstream_ha *upstream_ha, + struct flb_upstream *upstream, + int protocol_version, + uint64_t flags) +{ + struct flb_http_client_ng *client; + int result; + + client = flb_calloc(1, sizeof(struct flb_http_client_ng)); + + if (client != NULL) { + result = flb_http_client_ng_init(client, + upstream_ha, + upstream, + protocol_version, + flags); + + client->releasable = FLB_TRUE; + + if (result != 0) { + flb_http_client_ng_destroy(client); + + client = NULL; + } + } + + return client; +} + +void flb_http_client_ng_destroy(struct flb_http_client_ng *client) +{ + struct cfl_list *iterator_backup; + struct cfl_list *iterator; + struct flb_http_client_session *session; + + flb_lock_acquire(&client->lock, + FLB_LOCK_INFINITE_RETRY_LIMIT, + FLB_LOCK_DEFAULT_RETRY_DELAY); + + cfl_list_foreach_safe(iterator, + iterator_backup, + &client->sessions) { + session = cfl_list_entry(iterator, + struct flb_http_client_session, + _head); + + flb_http_client_session_destroy(session); + } + + flb_lock_release(&client->lock, + FLB_LOCK_INFINITE_RETRY_LIMIT, + FLB_LOCK_DEFAULT_RETRY_DELAY); + + flb_lock_destroy(&client->lock); +} + +int flb_http_client_session_init(struct flb_http_client_session *session, + struct flb_http_client_ng *client, + int protocol_version, + struct flb_connection *connection) +{ + int result; + + memset(session, 0, sizeof(struct flb_http_client_session)); + + session->parent = client; + session->protocol_version = protocol_version; + session->connection = connection; + session->stream_sequence_number = 1; + + cfl_list_init(&session->streams); + cfl_list_init(&session->response_queue); + + cfl_list_entry_init(&session->_head); + + session->incoming_data = cfl_sds_create_size(1); + + if (session->incoming_data == NULL) { + return -1; + } + + session->outgoing_data = cfl_sds_create_size(1); + + if (session->outgoing_data == NULL) { + return -1; + } + + if (session->protocol_version == HTTP_PROTOCOL_VERSION_11 || + session->protocol_version == HTTP_PROTOCOL_VERSION_10) { + session->http1.parent = session; + + result = flb_http1_client_session_init(&session->http1); + + if (result != 0) { + return result; + } + } + else if (session->protocol_version == HTTP_PROTOCOL_VERSION_20) { + session->http2.parent = session; + + result = flb_http2_client_session_init(&session->http2); + + if (result != 0) { + return result; + } + } + else { + return -1; + } + + return 0; +} + +struct flb_http_client_session *flb_http_client_session_create(struct flb_http_client_ng *client, + int protocol_version, + struct flb_connection *connection) +{ + struct flb_http_client_session *session; + int result; + + session = flb_calloc(1, sizeof(struct flb_http_client_session)); + + if (session != NULL) { + if (client != NULL) { + flb_lock_acquire(&client->lock, + FLB_LOCK_INFINITE_RETRY_LIMIT, + FLB_LOCK_DEFAULT_RETRY_DELAY); + } + + result = flb_http_client_session_init(session, + client, + protocol_version, + connection); + + if (client != NULL) { + flb_lock_release(&client->lock, + FLB_LOCK_INFINITE_RETRY_LIMIT, + FLB_LOCK_DEFAULT_RETRY_DELAY); + } + + session->releasable = FLB_TRUE; + + if (result != 0) { + flb_http_client_session_destroy(session); + + session = NULL; + } + } + + return session; +} + +struct flb_http_client_session *flb_http_client_session_begin(struct flb_http_client_ng *client) +{ + int protocol_version; + struct flb_upstream_node *upstream_node; + struct flb_connection *connection; + struct flb_http_client_session *session; + const char *alpn; + + if (client->upstream_ha != NULL) { + upstream_node = flb_upstream_ha_node_get(client->upstream_ha); + + if (upstream_node == NULL) { + return NULL; + } + + connection = flb_upstream_conn_get(upstream_node->u); + } + else { + upstream_node = NULL; + + connection = flb_upstream_conn_get(client->upstream); + } + + if (connection == NULL) { + return NULL; + } + + protocol_version = client->protocol_version; + + if (protocol_version == HTTP_PROTOCOL_VERSION_AUTODETECT) { + if (connection->tls_session != NULL) { + alpn = flb_tls_session_get_alpn(connection->tls_session); + + if (alpn != NULL) { + if (strcasecmp(alpn, "h2") == 0) { + protocol_version = HTTP_PROTOCOL_VERSION_20; + } + else if (strcasecmp(alpn, "http/1.1") == 0) { + protocol_version = HTTP_PROTOCOL_VERSION_11; + } + else if (strcasecmp(alpn, "http/1.0") == 0) { + protocol_version = HTTP_PROTOCOL_VERSION_10; + } + } + } + } + + if (protocol_version == HTTP_PROTOCOL_VERSION_AUTODETECT) { + protocol_version = HTTP_PROTOCOL_VERSION_11; + } + + session = flb_http_client_session_create(client, protocol_version, connection); + + if (session == NULL) { + flb_upstream_conn_release(connection); + } + + session->upstream_node = upstream_node; + + return session; +} + +void flb_http_client_session_destroy(struct flb_http_client_session *session) +{ + struct cfl_list *iterator_backup; + struct cfl_list *iterator; + struct flb_http_stream *stream; + + if (session != NULL) { + cfl_list_foreach_safe(iterator, + iterator_backup, + &session->streams) { + stream = cfl_list_entry(iterator, struct flb_http_stream, _head); + + flb_http_stream_destroy(stream); + } + + if (session->connection != NULL) { + flb_upstream_conn_release(session->connection); + } + + if (!cfl_list_entry_is_orphan(&session->_head)) { + cfl_list_del(&session->_head); + } + + if (session->incoming_data != NULL) { + cfl_sds_destroy(session->incoming_data); + } + + if (session->outgoing_data != NULL) { + cfl_sds_destroy(session->outgoing_data); + } + + flb_http1_client_session_destroy(&session->http1); + flb_http2_client_session_destroy(&session->http2); + + if (session->releasable) { + flb_free(session); + } + } +} + +struct flb_http_request *flb_http_client_request_begin(struct flb_http_client_session *session) +{ + int stream_id; + struct flb_http_stream *stream; + int result; + + stream_id = session->stream_sequence_number; + session->stream_sequence_number += 2; + + stream = flb_http_stream_create(session, + stream_id, + HTTP_STREAM_ROLE_CLIENT, + session); + + if (stream == NULL) { + return NULL; + } + + stream->request.protocol_version = session->protocol_version; + + if (stream->request.protocol_version == HTTP_PROTOCOL_VERSION_20) { + result = flb_http2_request_begin(&stream->request); + } + else if (stream->request.protocol_version == HTTP_PROTOCOL_VERSION_11 || + stream->request.protocol_version == HTTP_PROTOCOL_VERSION_10) { + result = flb_http1_request_begin(&stream->request); + } + else { + result = -1; + } + + if (result != 0) { + flb_http_stream_destroy(stream); + + return NULL; + } + + cfl_list_add(&stream->_head, &session->streams); + + return &stream->request; +} + +struct flb_http_response *flb_http_client_request_execute_step( + struct flb_http_request *request) +{ + struct flb_http_response *response; + struct flb_http_client_session *session; + int result; + + session = (struct flb_http_client_session *) request->stream->parent; + response = &request->stream->response; + + /* We allow this to enable request mocking, there is no + * other legitimate use for it. + */ + if (session->connection == NULL) { + return response; + } + + if (session->outgoing_data != NULL && + cfl_sds_len(session->outgoing_data) > 0) + { + result = flb_http_client_session_write(session); + + if (result != 0) { + return NULL; + } + + result = flb_http_client_session_read(session); + + if (result != 0) { + return NULL; + } + } + + if (request->stream->status == HTTP_STREAM_STATUS_SENDING_HEADERS) { + result = flb_http_request_commit(request); + + if (result != 0) { + return NULL; + } + + result = flb_http_client_session_write(session); + + if (result != 0) { + return NULL; + } + + request->stream->status = HTTP_STREAM_STATUS_RECEIVING_HEADERS; + } + else if (request->stream->status == HTTP_STREAM_STATUS_RECEIVING_HEADERS || + request->stream->status == HTTP_STREAM_STATUS_RECEIVING_DATA ) { + result = flb_http_client_session_read(session); + + if (result != 0) { + return NULL; + } + + if (session->outgoing_data != NULL && + cfl_sds_len(session->outgoing_data) > 0) + { + result = flb_http_client_session_write(session); + + if (result != 0) { + return NULL; + } + } + } + + if (request->stream->status != HTTP_STREAM_STATUS_RECEIVING_HEADERS && + request->stream->status != HTTP_STREAM_STATUS_RECEIVING_DATA && + request->stream->status != HTTP_STREAM_STATUS_CLOSED && + request->stream->status != HTTP_STREAM_STATUS_READY ) { + return NULL; + } + + return response; +} + +struct flb_http_response *flb_http_client_request_execute(struct flb_http_request *request) +{ + struct flb_http_response *response; + + do { + response = flb_http_client_request_execute_step(request); + } while (response != NULL && + request->stream->status != HTTP_STREAM_STATUS_READY && + request->stream->status != HTTP_STREAM_STATUS_CLOSED); + + return response; +} + +static int flb_http_client_session_read(struct flb_http_client_session *session) +{ + unsigned char input_buffer[1024 * 65]; + ssize_t result; + + result = flb_io_net_read(session->connection, + (void *) &input_buffer, + sizeof(input_buffer)); + + if (result <= 0) { + return -1; + } + + result = (ssize_t) flb_http_client_session_ingest(session, + input_buffer, + result); + + if (result < 0) { + return -2; + } + + return 0; +} + + +void flb_http_client_request_destroy(struct flb_http_request *request, + int destroy_session) +{ + if (destroy_session == FLB_TRUE) { + flb_http_client_session_destroy((struct flb_http_client_session *) + request->stream->parent); + } + else { + flb_http_request_destroy(request); + } +} + + +static int flb_http_client_session_write(struct flb_http_client_session *session) +{ + size_t data_length; + size_t data_sent; + int result; + + if (session == NULL) { + return -1; + } + + if (session->outgoing_data == NULL) { + return 0; + } + + data_length = cfl_sds_len(session->outgoing_data); + + if (data_length > 0) { + result = flb_io_net_write(session->connection, + (void *) session->outgoing_data, + data_length, + &data_sent); + + if (result == -1) { + return -2; + } + + + if (data_sent < data_length) { + memmove(session->outgoing_data, + &session->outgoing_data[data_sent], + data_length - data_sent); + + cfl_sds_set_len(session->outgoing_data, + data_length - data_sent); + } + else { + cfl_sds_set_len(session->outgoing_data, 0); + } + } + + return 0; +} + +int flb_http_client_session_ingest(struct flb_http_client_session *session, + unsigned char *buffer, + size_t length) +{ + if (session->protocol_version == HTTP_PROTOCOL_VERSION_11 || + session->protocol_version == HTTP_PROTOCOL_VERSION_10) { + return flb_http1_client_session_ingest(&session->http1, + buffer, + length); + } + else if (session->protocol_version == HTTP_PROTOCOL_VERSION_20) { + return flb_http2_client_session_ingest(&session->http2, + buffer, + length); + } + + return -20; +} + +static int flb_http_encode_basic_auth_value(cfl_sds_t *output_buffer, + char *username, + char *password) +{ + size_t encoded_value_length; + cfl_sds_t encoded_value; + cfl_sds_t sds_result; + cfl_sds_t raw_value; + int result; + + *output_buffer = NULL; + + raw_value = cfl_sds_create_size(strlen(username) + + strlen(password) + 2); + + if (raw_value == NULL) { + return -1; + } + + sds_result = cfl_sds_printf(&raw_value, + "%s:%s", + username, + password); + + if (sds_result == NULL) { + cfl_sds_destroy(raw_value); + + return -1; + } + + encoded_value = cfl_sds_create_size(cfl_sds_len(raw_value) * 2 + 1); + + if (encoded_value == NULL) { + cfl_sds_destroy(raw_value); + + return -1; + } + + result = flb_base64_encode((unsigned char *) encoded_value, + cfl_sds_alloc(encoded_value), + &encoded_value_length, + (unsigned char *) raw_value, + cfl_sds_len(raw_value)); + + if (result == 0) { + *output_buffer = cfl_sds_create_size(cfl_sds_len(encoded_value) + 6); + + if (*output_buffer != NULL) { + sds_result = cfl_sds_printf(output_buffer, "Basic %s", encoded_value); + + if (sds_result != NULL) { + *output_buffer = sds_result; + } + else { + result = -1; + } + } + else { + result = -1; + } + } + else { + result = -1; + } + + cfl_sds_destroy(encoded_value); + cfl_sds_destroy(raw_value); + + return 0; +} + +static int flb_http_encode_bearer_auth_value(cfl_sds_t *output_buffer, + char *token) +{ + cfl_sds_t sds_result; + + *output_buffer = NULL; + + *output_buffer = cfl_sds_create_size(strlen(token) + 9); + + if (*output_buffer == NULL) { + return -1; + } + + sds_result = cfl_sds_printf(output_buffer, + "Bearer %s", + token); + + if (sds_result == NULL) { + cfl_sds_destroy(*output_buffer); + *output_buffer = NULL; + + return -1; + } + + *output_buffer = sds_result; + + return 0; +} + + +int flb_http_request_set_authorization(struct flb_http_request *request, + int type, ...) +{ + cfl_sds_t header_value; + const char *header_name; + va_list arguments; + char *username; + char *password; + int result; + char *token; + + va_start(arguments, type); + + if (type == HTTP_WWW_AUTHORIZATION_SCHEME_BASIC) { + header_name = "authorization"; + + username = va_arg(arguments, char *); + password = va_arg(arguments, char *); + + result = flb_http_encode_basic_auth_value(&header_value, + username, + password); + + if (result != 0) { + va_end(arguments); + + return -1; + } + } + else if (type == HTTP_WWW_AUTHORIZATION_SCHEME_BEARER) { + header_name = "authorization"; + + token = va_arg(arguments, char *); + + result = flb_http_encode_bearer_auth_value(&header_value, + token); + + if (result != 0) { + va_end(arguments); + + return -1; + } + } + else if (type == HTTP_PROXY_AUTHORIZATION_SCHEME_BASIC) { + header_name = "proxy-authorization"; + + username = va_arg(arguments, char *); + password = va_arg(arguments, char *); + + result = flb_http_encode_basic_auth_value(&header_value, + username, + password); + + if (result != 0) { + va_end(arguments); + + return -1; + } + } + else if (type == HTTP_PROXY_AUTHORIZATION_SCHEME_BEARER) { + header_name = "proxy-authorization"; + + token = va_arg(arguments, char *); + + result = flb_http_encode_bearer_auth_value(&header_value, + token); + + if (result != 0) { + va_end(arguments); + + return -1; + } + } + + va_end(arguments); + + result = flb_http_request_set_header(request, + (char *) header_name, 0, + (char *) header_value, 0); + + cfl_sds_destroy(header_value); + + if (result != 0) { + result = -1; + } + + return result; +} + + + + +int flb_http_request_set_parameters_internal( + struct flb_http_request *request, + va_list arguments) +{ + char *compression_algorithm; + struct flb_config_map_val *config_map_list_entry; + int failure_detected; + size_t header_data_type; + char *content_type; + char *bearer_token; + struct flb_aws_provider *aws_provider; + struct flb_slist_entry *header_value; + char **header_array; + char *aws_service; + struct mk_list *header_list; + struct flb_slist_entry *header_name; + char *aws_region; + size_t value_type; + char *user_agent; + struct mk_list *iterator; + size_t body_len; + char *username; + char *password; + int result; + size_t index; + size_t method; + char *host; + unsigned char *body; + char *uri; + char *url; + + failure_detected = FLB_FALSE; + + do { + value_type = va_arg(arguments, size_t); + + if (value_type == FLB_HTTP_CLIENT_ARGUMENT_TYPE_METHOD) { + method = va_arg(arguments, size_t); + + flb_http_request_set_method(request, (int) method); + } + else if (value_type == FLB_HTTP_CLIENT_ARGUMENT_TYPE_HOST) { + host = va_arg(arguments, char *); + + flb_http_request_set_host(request, host); + } + else if (value_type == FLB_HTTP_CLIENT_ARGUMENT_TYPE_URL) { + url = va_arg(arguments, char *); + + flb_http_request_set_url(request, url); + } + else if (value_type == FLB_HTTP_CLIENT_ARGUMENT_TYPE_URI) { + uri = va_arg(arguments, char *); + + flb_http_request_set_uri(request, uri); + } + else if (value_type == FLB_HTTP_CLIENT_ARGUMENT_TYPE_USER_AGENT) { + user_agent = va_arg(arguments, char *); + + flb_http_request_set_user_agent(request, user_agent); + } + else if (value_type == FLB_HTTP_CLIENT_ARGUMENT_TYPE_CONTENT_TYPE) { + content_type = va_arg(arguments, char *); + + result = flb_http_request_set_content_type(request, content_type); + + if (request == NULL) { + flb_debug("http request : error setting content type"); + + failure_detected = FLB_TRUE; + } + } + else if (value_type == FLB_HTTP_CLIENT_ARGUMENT_TYPE_BODY) { + body = va_arg(arguments, unsigned char *); + body_len = va_arg(arguments, size_t); + compression_algorithm = va_arg(arguments, char *); + + result = flb_http_request_set_body(request, + body, + body_len, + compression_algorithm); + + if (request == NULL) { + flb_debug("http request creation error"); + + failure_detected = FLB_TRUE; + } + } + else if (value_type == FLB_HTTP_CLIENT_ARGUMENT_TYPE_HEADERS) { + header_data_type = va_arg(arguments, size_t); + + if (header_data_type == FLB_HTTP_CLIENT_HEADER_ARRAY) { + header_array = va_arg(arguments, char **); + if (header_array != NULL) { + for (index = 0 ; + header_array[index+0] != NULL && + header_array[index+1] != NULL; + index += 2) { + result = flb_http_request_set_header(request, + header_array[index+0], 0, + header_array[index+1], 0); + + if (result != 0) { + flb_debug("http request header addition error"); + + failure_detected = FLB_TRUE; + + break; + } + } + } + } + else if (header_data_type == FLB_HTTP_CLIENT_HEADER_CONFIG_MAP_LIST) { + header_list = va_arg(arguments, struct mk_list *); + + flb_config_map_foreach(iterator, config_map_list_entry, header_list) { + header_name = mk_list_entry_first(config_map_list_entry->val.list, + struct flb_slist_entry, + _head); + + header_value = mk_list_entry_last(config_map_list_entry->val.list, + struct flb_slist_entry, + _head); + + result = flb_http_request_set_header(request, + header_name->str, 0, + header_value->str, 0); + + if (result != 0) { + flb_debug("http request header addition error"); + + failure_detected = FLB_TRUE; + + break; + } + } + } + else { + failure_detected = FLB_TRUE; + } + } + else if (value_type == FLB_HTTP_CLIENT_ARGUMENT_TYPE_AUTH_BASIC) { + username = va_arg(arguments, char *); + password = va_arg(arguments, char *); + + flb_http_request_set_authorization(request, + HTTP_WWW_AUTHORIZATION_SCHEME_BASIC, + username, + password); + } + else if (value_type == FLB_HTTP_CLIENT_ARGUMENT_TYPE_AUTH_BEARER_TOKEN) { + bearer_token = va_arg(arguments, char *); + + flb_http_request_set_authorization(request, + HTTP_WWW_AUTHORIZATION_SCHEME_BEARER, + bearer_token); + } + else if (value_type == FLB_HTTP_CLIENT_ARGUMENT_TYPE_AUTH_SIGNV4) { + aws_region = va_arg(arguments, char *); + aws_service = va_arg(arguments, char *); + aws_provider = va_arg(arguments, struct flb_aws_provider *); + + result = flb_http_request_perform_signv4_signature(request, + aws_region, + aws_service, + aws_provider); + } + } while (!failure_detected && + value_type != FLB_HTTP_CLIENT_ARGUMENT_TYPE_TERMINATOR); + + if (failure_detected) { + return -1; + } + + return 0; +} + +int flb_http_request_set_parameters_unsafe( + struct flb_http_request *request, + ...) +{ + va_list arguments; + int result; + + va_start(arguments, request); + + result = flb_http_request_set_parameters_internal(request, arguments); + + va_end(arguments); + + return result; +} + +struct flb_http_request *flb_http_client_request_builder_unsafe( + struct flb_http_client_ng *client, + ...) +{ + va_list arguments; + struct flb_http_request *request; + struct flb_http_client_session *session; + int result; + + session = flb_http_client_session_begin(client); + + if (session == NULL) { + flb_debug("http session creation error"); + + return NULL; + } + + request = flb_http_client_request_begin(session); + + if (request == NULL) { + flb_debug("http request creation error"); + + flb_http_client_session_destroy(session); + + return NULL; + } + + flb_http_request_set_port(request, client->upstream->tcp_port); + + va_start(arguments, client); + + result = flb_http_request_set_parameters_internal(request, arguments); + + va_end(arguments); + + if (result != 0) { + flb_http_client_session_destroy(session); + + /* + * The request instance is recursively disposed of + */ + request = NULL; + } + + return request; +} diff --git a/src/flb_http_client_http1.c b/src/flb_http_client_http1.c new file mode 100644 index 00000000000..e09950c70fb --- /dev/null +++ b/src/flb_http_client_http1.c @@ -0,0 +1,727 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _GNU_SOURCE +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static inline size_t http2_lower_value(size_t left_value, size_t right_value); + + +static int compose_request_line(cfl_sds_t *output_buffer, + struct flb_http_request *request); + + +static int compose_header_line(cfl_sds_t *output_buffer, + char *name, + size_t name_length, + char *value, + size_t value_length); + +static int parse_term(char **term_start, + size_t *term_length, + char **next_term, + char *current_term, + char delimiter_character, + int trim_leading_spaces, + int trim_trailing_spaces); + +static int parse_headers(struct flb_http_response *response, char *headers); + + +int flb_http1_client_session_init(struct flb_http1_client_session *session) +{ + return 0; +} + +void flb_http1_client_session_destroy(struct flb_http1_client_session *session) +{ + if (session != NULL) { + if (session->initialized) { + session->initialized = FLB_FALSE; + } + } +} + +static int cfl_sds_shift_left(cfl_sds_t *buffer, size_t positions) +{ + size_t buffer_length; + size_t remainder; + + buffer_length = cfl_sds_len(*buffer); + remainder = 0; + + if (positions < buffer_length) { + remainder = buffer_length - positions; + + memmove(*buffer, + &((*buffer)[positions]), + remainder); + } + + cfl_sds_len_set(*buffer, remainder); + + (*buffer)[remainder] = '\0'; + + return 0; +} + +static int flb_http1_client_session_process_headers(struct flb_http_client_session *session, + struct flb_http_response *response) +{ + char *header_block_begining; + size_t header_block_length; + size_t status_line_length; + char *header_block_end; + char *status_line; + int result; + + status_line = session->incoming_data; + + /* We need at least 8 characters to differentiate + * HTTP/1.x from HTTP/0.9 + */ + if (cfl_sds_len(status_line) < 9) { + return 0; + } + + if (strncasecmp(status_line, "HTTP/1.1 ", 9) == 0) { + response->protocol_version = HTTP_PROTOCOL_VERSION_11; + } + else if (strncasecmp(status_line, "HTTP/1.0 ", 9) == 0) { + response->protocol_version = HTTP_PROTOCOL_VERSION_11; + } + else { + response->protocol_version = HTTP_PROTOCOL_VERSION_09; + } + + if (response->protocol_version == HTTP_PROTOCOL_VERSION_09) { + flb_http_response_set_status(response, 200); + + flb_http_response_set_message(response, ""); + + response->stream->status = HTTP_STREAM_STATUS_RECEIVING_DATA; + } + else { + header_block_begining = strstr(status_line, "\r\n"); + + if (header_block_begining == NULL) { + return 0; + } + + status_line_length = (size_t) (((uintptr_t) header_block_begining) - + ((uintptr_t) status_line)); + + header_block_begining = &header_block_begining[2]; + + header_block_end = strstr(header_block_begining, "\r\n\r\n"); + + if (header_block_end == NULL) { + return 0; + } + + header_block_length = (size_t) ((uintptr_t) header_block_end - + (uintptr_t) header_block_begining); + + if (response->status <= 0) { + response->status = (int) strtoul(&status_line[9], NULL, 10); + + if (response->status < 100 || response->status > 599) { + response->stream->status = HTTP_STREAM_STATUS_ERROR; + + return -1; + } + + result = parse_headers(response, header_block_begining); + + if (result != 0) { + response->stream->status = HTTP_STREAM_STATUS_ERROR; + + return -1; + } + } + + cfl_sds_shift_left(&session->incoming_data, + status_line_length + 2 + + header_block_length + 4); + + response->stream->status = HTTP_STREAM_STATUS_RECEIVING_DATA; + } + + return 0; +} + +static int flb_http1_client_session_process_data(struct flb_http_client_session *session, + struct flb_http_response *response) +{ + char *body; + int body_streaming_flag; + int chunked_transfer; + char *transfer_encoding; + size_t body_length; + size_t body_remainder; + int result; + size_t chunk_length_length; + char *chunk_length_end; + char *chunk_header; + char *chunk_data; + size_t chunk_length; + size_t required_size; + + body = session->incoming_data; + + body_streaming_flag = (session->parent->flags & + FLB_HTTP_CLIENT_FLAG_STREAM_BODY) != 0; + + chunked_transfer = FLB_FALSE; + + if (response->content_length == 0) { + transfer_encoding = flb_http_response_get_header(response, "transfer-encoding"); + + if (transfer_encoding != NULL) { + if (strncasecmp(transfer_encoding, "chunked", 7) == 0) { + chunked_transfer = FLB_TRUE; + } + } + } + + body_length = cfl_sds_len(body); + + if (chunked_transfer == FLB_FALSE) { + if (response->content_length > 0) { + if ((response->body_read_offset + body_length) < + response->content_length) { + if (body_streaming_flag == FLB_FALSE) { + return 0; + } + + response->body_read_offset += body_length; + + result = flb_http_response_append_to_body( + response, + (unsigned char *) body, + body_length); + } + else { + result = flb_http_response_append_to_body( + response, + (unsigned char *) body, + body_length); + + response->stream->status = HTTP_STREAM_STATUS_READY; + } + + cfl_sds_shift_left(&session->incoming_data, body_length); + } + else { + response->stream->status = HTTP_STREAM_STATUS_READY; + } + } + else { + body_remainder = body_length; + + while (body_remainder > 0) { + chunk_header = session->incoming_data; + + if (strchr(chunk_header, '\r') == NULL) { + return 0; + } + + errno = 0; + + chunk_length = strtoull(chunk_header, &chunk_length_end, 16); + + if (errno != 0) { + response->stream->status = HTTP_STREAM_STATUS_ERROR; + + return -1; + } + + chunk_length_length = (size_t) ((uintptr_t) chunk_length_end - + (uintptr_t) chunk_header); + + required_size = chunk_length_length + 2 + chunk_length + 2; + + if (body_remainder < required_size) { + return 0; + } + + chunk_data = chunk_header + chunk_length_length + 2; + + if (chunk_length > 0) { + result = flb_http_response_append_to_body( + response, + (unsigned char *) chunk_data, + chunk_length); + + if (result != 0) { + response->stream->status = HTTP_STREAM_STATUS_ERROR; + + return -1; + } + } + else { + response->stream->status = HTTP_STREAM_STATUS_READY; + } + + cfl_sds_shift_left(&session->incoming_data, required_size); + + response->body_read_offset += required_size; + body_remainder -= required_size; + } + } + + return 0; +} + +int flb_http1_client_session_ingest(struct flb_http1_client_session *session, + unsigned char *buffer, + size_t length) +{ + cfl_sds_t resized_buffer; + int result; + struct flb_http_stream *stream; + + stream = cfl_list_entry_first(&session->parent->streams, + struct flb_http_stream, + _head); + + resized_buffer = cfl_sds_cat(session->parent->incoming_data, + (const char *) buffer, + length); + + if (resized_buffer == NULL) { + return -1; + } + + session->parent->incoming_data = resized_buffer; + result = 0; + + if (stream->response.stream->status == HTTP_STREAM_STATUS_RECEIVING_HEADERS) { + result = flb_http1_client_session_process_headers( + session->parent, + &stream->response); + } + + if (result == 0 && + stream->response.stream->status == HTTP_STREAM_STATUS_RECEIVING_DATA) { + result = flb_http1_client_session_process_data( + session->parent, + &stream->response); + } + + return result; +} + +int flb_http1_request_begin(struct flb_http_request *request) +{ + return 0; +} + + +int flb_http1_request_commit(struct flb_http_request *request) +{ + char content_length_string[16]; + struct mk_list *header_iterator; + cfl_sds_t request_buffer; + struct flb_http_client_session *parent_session; + struct flb_hash_table_entry *header_entry; + cfl_sds_t sds_result; + struct flb_http1_client_session *session; + struct flb_http_stream *stream; + int result; + + parent_session = (struct flb_http_client_session *) request->stream->parent; + + if (parent_session == NULL) { + return -1; + } + + session = &parent_session->http1; + + if (session == NULL) { + return -1; + } + + stream = (struct flb_http_stream *) request->stream; + + if (stream == NULL) { + return -2; + } + + request_buffer = cfl_sds_create_size(128); + + if (request_buffer == NULL) { + return -3; + } + + result = compose_request_line(&request_buffer, request); + + if (result != 0) { + cfl_sds_destroy(request_buffer); + + return -4; + } + + mk_list_foreach(header_iterator, &request->headers->entries) { + header_entry = mk_list_entry(header_iterator, + struct flb_hash_table_entry, + _head_parent); + + if (header_entry == NULL) { + cfl_sds_destroy(request_buffer); + + return -5; + } + + result = compose_header_line(&request_buffer, + header_entry->key, + header_entry->key_len, + header_entry->val, + header_entry->val_size); + + if (result != 0) { + cfl_sds_destroy(request_buffer); + + return -6; + } + } + + if (request->protocol_version == HTTP_PROTOCOL_VERSION_11) { + if(request->host != NULL) { + result = compose_header_line(&request_buffer, + "Host", 0, + request->host, 0); + + if (result != 0) { + cfl_sds_destroy(request_buffer); + + return -7; + } + } + } + + if(request->user_agent != NULL) { + result = compose_header_line(&request_buffer, + "User-agent", 0, + request->user_agent, 0); + + if (result != 0) { + cfl_sds_destroy(request_buffer); + + return -8; + } + } + + if(request->content_type != NULL) { + result = compose_header_line(&request_buffer, + "Content-type", 0, + request->content_type, 0); + + if (result != 0) { + cfl_sds_destroy(request_buffer); + + return -9; + } + } + + if (request->method == HTTP_METHOD_POST || + request->method == HTTP_METHOD_PUT) { + snprintf(content_length_string, + sizeof(content_length_string) - 1, + "%zu", + request->content_length); + + content_length_string[sizeof(content_length_string) - 1] = '\0'; + + result = compose_header_line(&request_buffer, + "Content-length", 0, + content_length_string, 0); + + if (result != 0) { + cfl_sds_destroy(request_buffer); + + return -7; + } + } + + sds_result = cfl_sds_cat(request_buffer, "\r\n", 2); + + if (sds_result == NULL) { + cfl_sds_destroy(request_buffer); + + return -7; + } + + if (request->body != NULL) { + sds_result = cfl_sds_cat(request_buffer, + request->body, + cfl_sds_len(request->body)); + + if (sds_result == NULL) { + cfl_sds_destroy(request_buffer); + + return -8; + } + + request_buffer = sds_result; + } + + sds_result = cfl_sds_cat(session->parent->outgoing_data, + request_buffer, + cfl_sds_len(request_buffer)); + + cfl_sds_destroy(request_buffer); + + if (sds_result == NULL) { + return -9; + } + + session->parent->outgoing_data = sds_result; + + return 0; +} + +static int compose_request_line(cfl_sds_t *output_buffer, + struct flb_http_request *request) +{ + const char *protocol_version_string; + const char *method_name; + cfl_sds_t sds_result; + + sds_result = NULL; + + if (request->protocol_version == HTTP_PROTOCOL_VERSION_11) { + protocol_version_string = " HTTP/1.1"; + } + else if (request->protocol_version == HTTP_PROTOCOL_VERSION_10) { + protocol_version_string = " HTTP/1.0"; + } + else if (request->protocol_version == HTTP_PROTOCOL_VERSION_09) { + protocol_version_string = ""; + } + + method_name = flb_http_get_method_string_from_id(request->method); + + if (method_name == NULL) { + return -1; + } + + if (request->method == HTTP_METHOD_CONNECT) { + sds_result = cfl_sds_printf(output_buffer, + "CONNECT %s:%u%s\r\n", + request->host, + request->port, + protocol_version_string); + } + else { + if (request->query_string != NULL) { + sds_result = cfl_sds_printf(output_buffer, + "%s %s?%s%s\r\n", + method_name, + request->path, + request->query_string, + protocol_version_string); + } + else { + sds_result = cfl_sds_printf(output_buffer, + "%s %s%s\r\n", + method_name, + request->path, + protocol_version_string); + } + } + + if (sds_result == NULL) { + return -1; + } + + *output_buffer = sds_result; + + return 0; +} + +static int compose_header_line(cfl_sds_t *output_buffer, + char *name, + size_t name_length, + char *value, + size_t value_length) +{ + cfl_sds_t sds_result; + + if (name_length == 0) { + name_length = strlen(name); + } + + if (value_length == 0) { + value_length = strlen(value); + } + + sds_result = cfl_sds_printf(output_buffer, + "%.*s: %.*s\r\n", + (int) name_length, + name, + (int) value_length, + value); + + if (sds_result == NULL) { + return -1; + } + + return 0; +} + + + + +static int parse_term(char **term_start, + size_t *term_length, + char **next_term, + char *current_term, + char delimiter_character, + int trim_leading_spaces, + int trim_trailing_spaces) +{ + char *term_delimiter; + + if (trim_leading_spaces) { + while (current_term[0] == ' ') { + current_term++; + } + } + + if (current_term[0] == '\0') { + return -1; + } + + term_delimiter = strchr(current_term, delimiter_character); + + if (term_delimiter == NULL) { + return -1; + } + + *term_start = current_term; + *term_length = (size_t) ((uintptr_t) term_delimiter - + (uintptr_t) current_term); + + *next_term = &term_delimiter[1]; + + if (trim_trailing_spaces) { + while (*term_length > 0 && current_term[*term_length - 1] == ' ') { + (*term_length)--; + } + } + + return 0; +} + +static int parse_headers(struct flb_http_response *response, char *headers) +{ + char temporary_buffer[21]; + char *current_term; + size_t value_length; + size_t name_length; + int result; + char *value; + char *name; + + current_term = headers; + + while (current_term != NULL && current_term[0] != '\r') { + result = parse_term(&name, + &name_length, + ¤t_term, + current_term, + ':', + FLB_TRUE, + FLB_TRUE); + + if (result != 0) { + return -1; + } + + result = parse_term(&value, + &value_length, + ¤t_term, + current_term, + '\r', + FLB_TRUE, + FLB_TRUE); + + if (result != 0) { + return -1; + } + + if (flb_http_server_strncasecmp((uint8_t *) name, + name_length, + "content-type", 0) == 0) { + response->content_type = \ + cfl_sds_create_len((const char *) value, + value_length); + + if (response->content_type == NULL) { + return -1; + } + } + else if (flb_http_server_strncasecmp((uint8_t *) name, + name_length, + "content-length", 0) == 0) { + strncpy(temporary_buffer, + (const char *) value, + http2_lower_value(sizeof(temporary_buffer), value_length + 1)); + + temporary_buffer[sizeof(temporary_buffer) - 1] = '\0'; + + response->content_length = strtoull(temporary_buffer, NULL, 10); + } + + result = flb_http_response_set_header(response, + name, + name_length, + (void *) value, + value_length); + + if (result != 0) { + return -1; + } + + current_term = ¤t_term[1]; + } + + return 0; +} + +static inline size_t http2_lower_value(size_t left_value, size_t right_value) +{ + if (left_value < right_value) { + return left_value; + } + + return right_value; +} diff --git a/src/flb_http_client_http2.c b/src/flb_http_client_http2.c new file mode 100644 index 00000000000..d2d074cc23e --- /dev/null +++ b/src/flb_http_client_http2.c @@ -0,0 +1,707 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#define _GNU_SOURCE +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +static ssize_t http2_send_callback(nghttp2_session *inner_session, + const uint8_t *data, + size_t length, + int flags, + void *user_data); + +static int http2_header_callback(nghttp2_session *inner_session, + const nghttp2_frame *frame, + const uint8_t *name, + size_t namelen, + const uint8_t *value, + size_t valuelen, + uint8_t flags, + void *user_data); + +static int http2_frame_recv_callback(nghttp2_session *inner_session, + const nghttp2_frame *frame, + void *user_data); + +static int http2_stream_close_callback(nghttp2_session *session, + int32_t stream_id, + uint32_t error_code, + void *user_data); + +static int http2_begin_headers_callback(nghttp2_session *inner_session, + const nghttp2_frame *frame, + void *user_data); + +static int http2_data_chunk_recv_callback(nghttp2_session *inner_session, + uint8_t flags, + int32_t stream_id, + const uint8_t *data, + size_t len, + void *user_data); + +static ssize_t http2_data_source_read_callback(nghttp2_session *session, + int32_t stream_id, + uint8_t *buf, + size_t length, + uint32_t *data_flags, + nghttp2_data_source *source, + void *user_data); + +static inline size_t http2_lower_value(size_t left_value, size_t right_value) +{ + if (left_value < right_value) { + return left_value; + } + + return right_value; +} + +/* PRIVATE */ + +static ssize_t http2_send_callback(nghttp2_session *inner_session, + const uint8_t *data, + size_t length, + int flags, + void *user_data) +{ + cfl_sds_t resized_buffer; + struct flb_http2_client_session *session; + + session = (struct flb_http2_client_session *) user_data; + + resized_buffer = cfl_sds_cat(session->parent->outgoing_data, + (const char *) data, + length); + + if (resized_buffer == NULL) { + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + + session->parent->outgoing_data = resized_buffer; + + return length; +} + +static int http2_header_callback(nghttp2_session *inner_session, + const nghttp2_frame *frame, + const uint8_t *name, + size_t name_length, + const uint8_t *value, + size_t value_length, + uint8_t flags, + void *user_data) +{ + char temporary_buffer[16]; + struct flb_http_stream *stream; + int result; + + stream = nghttp2_session_get_stream_user_data(inner_session, + frame->hd.stream_id); + + if (stream == NULL) { + return 0; + } + + if (flb_http_server_strncasecmp( + name, name_length, ":status", 0) == 0) { + strncpy(temporary_buffer, + (const char *) value, + http2_lower_value(sizeof(temporary_buffer), value_length + 1)); + + temporary_buffer[sizeof(temporary_buffer) - 1] = '\0'; + + stream->response.status = strtoull(temporary_buffer, NULL, 10); + } + else if (flb_http_server_strncasecmp( + name, name_length, "content-type", 0) == 0) { + + stream->response.content_type = cfl_sds_create_len((const char *) value, value_length); + + if (stream->response.content_type == NULL) { + return -1; + } + } + else if (flb_http_server_strncasecmp( + name, name_length, "content-length", 0) == 0) { + strncpy(temporary_buffer, + (const char *) value, + http2_lower_value(sizeof(temporary_buffer), value_length + 1)); + + temporary_buffer[sizeof(temporary_buffer) - 1] = '\0'; + + stream->response.content_length = strtoull(temporary_buffer, NULL, 10); + } + + result = flb_http_response_set_header(&stream->response, + (char *) name, + name_length, + (void *) value, + value_length); + + if (result != 0) { + return -1; + } + + return 0; +} + +static int http2_frame_recv_callback(nghttp2_session *inner_session, + const nghttp2_frame *frame, + void *user_data) +{ + struct flb_http_client_session *parent_session; + struct flb_http_stream *stream; + + stream = nghttp2_session_get_stream_user_data(inner_session, + frame->hd.stream_id); + + if (stream == NULL) { + return 0; + } + + switch (frame->hd.type) { + case NGHTTP2_CONTINUATION: + case NGHTTP2_HEADERS: + if ((frame->hd.flags & NGHTTP2_FLAG_END_HEADERS) != 0) { + if (stream->status == HTTP_STREAM_STATUS_RECEIVING_HEADERS) { + stream->status = HTTP_STREAM_STATUS_RECEIVING_DATA; + } + } + else { + if (stream->status == HTTP_STREAM_STATUS_RECEIVING_DATA) { + stream->status = HTTP_STREAM_STATUS_RECEIVING_TRAILER; + } + //else if + //stream->status = HTTP_STREAM_STATUS_RECEIVING_HEADERS; + } + + break; + default: + break; + } + + if ((frame->hd.flags & NGHTTP2_FLAG_END_STREAM) != 0) { + stream->status = HTTP_STREAM_STATUS_READY; + + if (!cfl_list_entry_is_orphan(&stream->request._head)) { + cfl_list_del(&stream->request._head); + } + + parent_session = (struct flb_http_client_session *) stream->parent; + + if (parent_session == NULL) { + return -1; + } + + cfl_list_add(&stream->response._head, + &parent_session->response_queue); + } + + return 0; +} + +static int http2_stream_close_callback(nghttp2_session *session, + int32_t stream_id, + uint32_t error_code, + void *user_data) +{ + struct flb_http_stream *stream; + + stream = nghttp2_session_get_stream_user_data(session, stream_id); + + if (stream == NULL) { + return 0; + } + + stream->status = HTTP_STREAM_STATUS_CLOSED; + + return 0; +} + +static int http2_begin_headers_callback(nghttp2_session *inner_session, + const nghttp2_frame *frame, + void *inner_user_data) +{ + return 0; +} + +static int http2_data_chunk_recv_callback(nghttp2_session *inner_session, + uint8_t flags, + int32_t stream_id, + const uint8_t *data, + size_t len, + void *user_data) +{ + struct flb_http_client_session *parent_session; + cfl_sds_t resized_buffer; + struct flb_http_stream *stream; + + stream = nghttp2_session_get_stream_user_data(inner_session, stream_id); + + if (stream == NULL) { + return 0; + } + + if (stream->status != HTTP_STREAM_STATUS_RECEIVING_DATA) { + stream->status = HTTP_STREAM_STATUS_ERROR; + + return -1; + } + + if (stream->response.body == NULL) { + stream->response.body = cfl_sds_create_size(len); + + if (stream->response.body == NULL) { + stream->status = HTTP_STREAM_STATUS_ERROR; + + return -1; + } + + memcpy(stream->response.body, data, len); + + cfl_sds_set_len(stream->response.body, len); + + stream->response.body_read_offset = len; + } + else { + resized_buffer = cfl_sds_cat(stream->response.body, + (const char *) data, + len); + + if (resized_buffer == NULL) { + stream->status = HTTP_STREAM_STATUS_ERROR; + + return -1; + } + + stream->response.body = resized_buffer; + stream->response.body_read_offset += len; + } + + if (stream->status == HTTP_STREAM_STATUS_RECEIVING_DATA) { + if (stream->response.content_length >= + stream->response.body_read_offset) { + stream->status = HTTP_STREAM_STATUS_READY; + + if (!cfl_list_entry_is_orphan(&stream->response._head)) { + cfl_list_del(&stream->response._head); + } + + parent_session = (struct flb_http_client_session *) stream->parent; + + if (parent_session == NULL) { + return -1; + } + + cfl_list_add(&stream->response._head, + &parent_session->response_queue); + } + } + + return 0; +} + +static ssize_t http2_data_source_read_callback(nghttp2_session *session, + int32_t stream_id, + uint8_t *buf, + size_t length, + uint32_t *data_flags, + nghttp2_data_source *source, + void *user_data) +{ + size_t content_length; + size_t body_offset; + struct flb_http_stream *stream; + ssize_t result; + + stream = nghttp2_session_get_stream_user_data(session, + stream_id); + + if (stream == NULL) { + return NGHTTP2_ERR_CALLBACK_FAILURE; + } + + if (stream->request.body != NULL) { + body_offset = stream->request.body_read_offset; + content_length = cfl_sds_len(stream->request.body) - body_offset; + } + else { + body_offset = 0; + content_length = 0; + } + + if (content_length > length) { + memcpy(buf, + &stream->request.body[body_offset], length); + + result = length; + + stream->request.body_read_offset += length; + } + else { + if (content_length > 0) { + memcpy(buf, stream->request.body, content_length); + + stream->request.body_read_offset += content_length; + } + + result = content_length; + + *data_flags |= NGHTTP2_DATA_FLAG_EOF; + + if (mk_list_is_empty(&stream->request.trailer_headers->entries) != 0) { + *data_flags |= NGHTTP2_DATA_FLAG_NO_END_STREAM; + } + } + + return result; +} + +int flb_http2_client_session_init(struct flb_http2_client_session *session) +{ + nghttp2_settings_entry session_settings[1]; + nghttp2_session_callbacks *callbacks; + int result; + + result = nghttp2_session_callbacks_new(&callbacks); + + if (result != 0) { + return -1; + } + + nghttp2_session_callbacks_set_send_callback(callbacks, http2_send_callback); + + nghttp2_session_callbacks_set_on_frame_recv_callback(callbacks, http2_frame_recv_callback); + + nghttp2_session_callbacks_set_on_stream_close_callback(callbacks, http2_stream_close_callback); + + nghttp2_session_callbacks_set_on_begin_headers_callback(callbacks, http2_begin_headers_callback); + + nghttp2_session_callbacks_set_on_data_chunk_recv_callback(callbacks, http2_data_chunk_recv_callback); + + nghttp2_session_callbacks_set_on_header_callback(callbacks, http2_header_callback); + + result = nghttp2_session_client_new(&session->inner_session, callbacks, session); + + nghttp2_session_callbacks_del(callbacks); + + if (result != 0) { + return -2; + } + + session->initialized = FLB_TRUE; + + session_settings[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS; + session_settings[0].value = 1; + + result = nghttp2_submit_settings(session->inner_session, + NGHTTP2_FLAG_NONE, + session_settings, + 1); + + if (result != 0) { + return -3; + } + + result = nghttp2_session_send(session->inner_session); + + if (result != 0) { + return -4; + } + + return 0; +} + +void flb_http2_client_session_destroy(struct flb_http2_client_session *session) +{ + if (session != NULL) { + if (session->initialized) { + nghttp2_session_del(session->inner_session); + + session->initialized = FLB_FALSE; + } + } +} + +int flb_http2_client_session_ingest(struct flb_http2_client_session *session, + unsigned char *buffer, + size_t length) +{ + ssize_t result; + + result = nghttp2_session_mem_recv(session->inner_session, buffer, length); + + if (result < 0) { + return HTTP_CLIENT_PROVIDER_ERROR; + } + + result = nghttp2_session_send(session->inner_session); + + if (result < 0) { + return HTTP_CLIENT_PROVIDER_ERROR; + } + + return HTTP_CLIENT_SUCCESS; +} + +int flb_http2_request_begin(struct flb_http_request *request) +{ + return 0; +} + +int flb_http2_request_commit(struct flb_http_request *request) +{ + struct flb_http_client_session *parent_session; + struct flb_http2_client_session *session; + struct flb_http_stream *stream; + int result; + + char content_length_string[21]; + struct mk_list *header_iterator; + const char *scheme_as_text; + const char *method_as_text; + nghttp2_data_provider data_provider; + size_t header_count; + size_t header_index; + struct flb_hash_table_entry *header_entry; + nghttp2_nv *headers; + + parent_session = (struct flb_http_client_session *) request->stream->parent; + + if (parent_session == NULL) { + return -1; + } + + session = &parent_session->http2; + + if (session == NULL) { + return -1; + } + + stream = (struct flb_http_stream *) request->stream; + + if (stream == NULL) { + return -2; + } + + if (request->host == NULL) { + return -1; + } + + if (parent_session->connection->tls_session != NULL) { + scheme_as_text = "HTTPS"; + } + else { + scheme_as_text = "HTTP"; + } + + switch (request->method) { + case HTTP_METHOD_GET: + method_as_text = "GET"; + break; + case HTTP_METHOD_POST: + method_as_text = "POST"; + break; + case HTTP_METHOD_HEAD: + method_as_text = "HEAD"; + break; + case HTTP_METHOD_PUT: + method_as_text = "PUT"; + break; + case HTTP_METHOD_DELETE: + method_as_text = "DELETE"; + break; + case HTTP_METHOD_OPTIONS: + method_as_text = "OPTIONS"; + break; + case HTTP_METHOD_CONNECT: + method_as_text = "CONNECT"; + break; + default: + method_as_text = NULL; + break; + } + + if (method_as_text == NULL) { + return -1; + } + + header_count = request->headers->total_count + 7; + + headers = flb_calloc(header_count, sizeof(nghttp2_nv)); + + if (headers == NULL) { + return -3; + } + + header_index = 0; + + headers[header_index].name = (uint8_t *) ":method"; + headers[header_index].namelen = strlen(":method"); + headers[header_index].value = (uint8_t *) method_as_text; + headers[header_index].valuelen = strlen(method_as_text); + + header_index++; + + headers[header_index].name = (uint8_t *) ":scheme"; + headers[header_index].namelen = strlen(":scheme"); + headers[header_index].value = (uint8_t *) scheme_as_text; + headers[header_index].valuelen = strlen(scheme_as_text); + + header_index++; + + headers[header_index].name = (uint8_t *) ":authority"; + headers[header_index].namelen = strlen(":authority"); + headers[header_index].value = (uint8_t *) request->host; + headers[header_index].valuelen = strlen(request->host); + + header_index++; + + if (request->method == HTTP_METHOD_OPTIONS && + request->path == NULL) { + headers[header_index].name = (uint8_t *) ":path"; + headers[header_index].namelen = strlen(":path"); + headers[header_index].value = (uint8_t *) "*"; + headers[header_index].valuelen = strlen("*"); + + header_index++; + } + else if (request->method != HTTP_METHOD_CONNECT) { + if (request->path == NULL) { + flb_free(headers); + + return -1; + } + + headers[header_index].name = (uint8_t *) ":path"; + headers[header_index].namelen = strlen(":path"); + headers[header_index].value = (uint8_t *) request->path; + headers[header_index].valuelen = strlen(request->path); + + header_index++; + } + + if(request->user_agent != NULL) { + headers[header_index].name = (uint8_t *) "User-agent"; + headers[header_index].namelen = strlen("User-agent"); + headers[header_index].value = (uint8_t *) request->user_agent; + headers[header_index].valuelen = strlen(request->user_agent); + + header_index++; + } + + if(request->content_type != NULL) { + headers[header_index].name = (uint8_t *) "Content-type"; + headers[header_index].namelen = strlen("Content-type"); + headers[header_index].value = (uint8_t *) request->content_type; + headers[header_index].valuelen = strlen(request->content_type); + + header_index++; + } + + if (request->method == HTTP_METHOD_POST || + request->method == HTTP_METHOD_PUT ) { + snprintf(content_length_string, + sizeof(content_length_string) - 1, + "%zu", + request->content_length); + + content_length_string[sizeof(content_length_string) - 1] = '\0'; + + headers[header_index].name = (uint8_t *) "Content-length"; + headers[header_index].namelen = strlen("Content-length"); + headers[header_index].value = (uint8_t *) content_length_string; + headers[header_index].valuelen = strlen(content_length_string); + + header_index++; + } + + header_count = request->headers->total_count + header_index; + + mk_list_foreach(header_iterator, &request->headers->entries) { + header_entry = mk_list_entry(header_iterator, + struct flb_hash_table_entry, + _head_parent); + + if (header_entry == NULL) { + flb_free(headers); + + return -4; + } + + headers[header_index].name = (uint8_t *) header_entry->key; + headers[header_index].namelen = header_entry->key_len; + headers[header_index].value = (uint8_t *) header_entry->val; + headers[header_index].valuelen = header_entry->val_size; + + if (headers[header_index].value[0] == '\0') { + headers[header_index].valuelen = 0; + } + + header_index++; + } + + data_provider.source.fd = 0; + data_provider.read_callback = http2_data_source_read_callback; + + stream->status = HTTP_STREAM_STATUS_PROCESSING; + + result = nghttp2_submit_request(session->inner_session, + NULL, + headers, + header_count, + &data_provider, + stream); + + if (result < 0) { + stream->status = HTTP_STREAM_STATUS_ERROR; + + flb_free(headers); + + return -5; + } + + stream->id = result; + + result = nghttp2_session_send(session->inner_session); + + flb_free(headers); + + if (result != 0) { + stream->status = HTTP_STREAM_STATUS_ERROR; + + return -8; + } + + stream->status = HTTP_STREAM_STATUS_RECEIVING_HEADERS; + + return 0; +} From bdc40db6b51456c8d8a1563ee715da5176c0261b Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 29 Oct 2024 23:03:59 +0100 Subject: [PATCH 02/21] tls: client side alpn support addition Signed-off-by: Leonardo Alminana --- include/fluent-bit/tls/flb_tls.h | 13 ++++--- src/tls/flb_tls.c | 9 +++++ src/tls/openssl.c | 62 +++++++++++++++++++++++++++++--- 3 files changed, 75 insertions(+), 9 deletions(-) diff --git a/include/fluent-bit/tls/flb_tls.h b/include/fluent-bit/tls/flb_tls.h index 02165d461a5..6a2caa232da 100644 --- a/include/fluent-bit/tls/flb_tls.h +++ b/include/fluent-bit/tls/flb_tls.h @@ -27,6 +27,8 @@ #include #include +#define FLB_TLS_ALPN_MAX_LENGTH 16 + #define FLB_TLS_CLIENT "Fluent Bit" /* TLS backend return status on read/write */ @@ -78,6 +80,7 @@ struct flb_tls_backend { /* Session management */ void *(*session_create) (struct flb_tls *, int); int (*session_destroy) (void *); + const char *(*session_alpn_get) (void *); /* I/O */ int (*net_read) (struct flb_tls_session *, void *, size_t); @@ -125,18 +128,20 @@ int flb_tls_session_create(struct flb_tls *tls, struct flb_connection *connection, struct flb_coro *co); -int flb_tls_net_read(struct flb_tls_session *session, - void *buf, +const char *flb_tls_session_get_alpn(struct flb_tls_session *session); + +int flb_tls_net_read(struct flb_tls_session *session, + void *buf, size_t len); -int flb_tls_net_read_async(struct flb_coro *th, +int flb_tls_net_read_async(struct flb_coro *th, struct flb_tls_session *session, void *buf, size_t len); int flb_tls_net_write(struct flb_tls_session *session, const void *data, size_t len, size_t *out_len); -int flb_tls_net_write_async(struct flb_coro *th, +int flb_tls_net_write_async(struct flb_coro *th, struct flb_tls_session *session, const void *data, size_t len, size_t *out_len); diff --git a/src/tls/flb_tls.c b/src/tls/flb_tls.c index f00e9b1ae0e..c8d5e55663f 100644 --- a/src/tls/flb_tls.c +++ b/src/tls/flb_tls.c @@ -672,6 +672,15 @@ int flb_tls_session_create(struct flb_tls *tls, return result; } +const char *flb_tls_session_get_alpn(struct flb_tls_session *session) +{ + if (session->ptr != NULL) { + return session->tls->api->session_alpn_get(session); + } + + return NULL; +} + int flb_tls_session_destroy(struct flb_tls_session *session) { int ret; diff --git a/src/tls/openssl.c b/src/tls/openssl.c index 38bba04e4f0..bc78f4572c9 100644 --- a/src/tls/openssl.c +++ b/src/tls/openssl.c @@ -52,11 +52,11 @@ struct tls_context { struct tls_session { SSL *ssl; int fd; + char alpn[FLB_TLS_ALPN_MAX_LENGTH]; int continuation_flag; struct tls_context *parent; /* parent struct tls_context ref */ }; - static int tls_init(void) { /* @@ -152,8 +152,8 @@ int tls_context_alpn_set(void *ctx_backend, const char *alpn) result = 0; if (alpn != NULL) { - wire_format_alpn = flb_calloc(strlen(alpn), - sizeof(char) + 1); + wire_format_alpn = flb_calloc(strlen(alpn) + 3, + sizeof(char)); if (wire_format_alpn == NULL) { return -1; @@ -373,6 +373,11 @@ static void *tls_context_create(int verify, tls_context_server_alpn_select_callback, ctx); } + else { + SSL_CTX_set_next_proto_select_cb(ssl_ctx, + tls_context_server_alpn_select_callback, + ctx); + } /* Verify peer: by default OpenSSL always verify peer */ if (verify == FLB_FALSE) { @@ -514,6 +519,37 @@ static int tls_session_destroy(void *session) return 0; } +static const char *tls_session_alpn_get(void *session_) +{ + const unsigned char *backend_alpn_buffer; + unsigned int backend_alpn_length; + struct tls_session *backend_session; + struct flb_tls_session *session; + + session = (struct flb_tls_session *) session_; + backend_session = (struct tls_session *) session->ptr; + + if (backend_session->alpn[0] == '\0') { + backend_alpn_buffer = NULL; + + SSL_get0_alpn_selected(backend_session->ssl, + &backend_alpn_buffer, + &backend_alpn_length); + + if (backend_alpn_buffer != NULL) { + if (backend_alpn_length >= FLB_TLS_ALPN_MAX_LENGTH) { + backend_alpn_length = FLB_TLS_ALPN_MAX_LENGTH - 1; + } + + strncpy(backend_session->alpn, + (char *) backend_alpn_buffer, + backend_alpn_length); + } + } + + return backend_session->alpn; +} + static int tls_net_read(struct flb_tls_session *session, void *buf, size_t len) { @@ -644,6 +680,7 @@ static int tls_net_write(struct flb_tls_session *session, ERR_error_string_n(err_code, err_buf, sizeof(err_buf) - 1); flb_error("[tls] error: %s", err_buf); } + ret = -1; } } @@ -691,6 +728,18 @@ static int tls_net_handshake(struct flb_tls *tls, if (!session->continuation_flag) { if (tls->mode == FLB_TLS_CLIENT_MODE) { SSL_set_connect_state(session->ssl); + + if (ctx->alpn != NULL) { + ret = SSL_set_alpn_protos(session->ssl, + (const unsigned char *) &ctx->alpn[1], + (unsigned int) ctx->alpn[0]); + + if (ret != 0) { + flb_error("[tls] error: alpn setup failed : %d", ret); + pthread_mutex_unlock(&ctx->mutex); + return -1; + } + } } else if (tls->mode == FLB_TLS_SERVER_MODE) { SSL_set_accept_state(session->ssl); @@ -738,8 +787,10 @@ static int tls_net_handshake(struct flb_tls *tls, if (ret != SSL_ERROR_WANT_READ && ret != SSL_ERROR_WANT_WRITE) { ret = SSL_get_error(session->ssl, ret); - // The SSL_ERROR_SYSCALL with errno value of 0 indicates unexpected - // EOF from the peer. This is fixed in OpenSSL 3.0. + /* The SSL_ERROR_SYSCALL with errno value of 0 indicates unexpected + * EOF from the peer. This is fixed in OpenSSL 3.0. + */ + if (ret == 0) { ssl_code = SSL_get_verify_result(session->ssl); if (ssl_code != X509_V_OK) { @@ -788,6 +839,7 @@ static struct flb_tls_backend tls_openssl = { .context_create = tls_context_create, .context_destroy = tls_context_destroy, .context_alpn_set = tls_context_alpn_set, + .session_alpn_get = tls_session_alpn_get, .session_create = tls_session_create, .session_destroy = tls_session_destroy, .net_read = tls_net_read, From 563a4475baa40c37b711c1d3a87df3fc07fe0546 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 29 Oct 2024 23:04:25 +0100 Subject: [PATCH 03/21] oauth2: new http client support added Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_oauth2.h | 1 + src/flb_oauth2.c | 112 ++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/include/fluent-bit/flb_oauth2.h b/include/fluent-bit/flb_oauth2.h index 7a1e4c82ebf..ea6266e95c7 100644 --- a/include/fluent-bit/flb_oauth2.h +++ b/include/fluent-bit/flb_oauth2.h @@ -64,6 +64,7 @@ void flb_oauth2_payload_clear(struct flb_oauth2 *ctx); int flb_oauth2_payload_append(struct flb_oauth2 *ctx, const char *key_str, int key_len, const char *val_str, int val_len); +char *flb_oauth2_token_get_ng(struct flb_oauth2 *ctx); char *flb_oauth2_token_get(struct flb_oauth2 *ctx); int flb_oauth2_token_expired(struct flb_oauth2 *ctx); diff --git a/src/flb_oauth2.c b/src/flb_oauth2.c index 28b377a24a3..12090c3257f 100644 --- a/src/flb_oauth2.c +++ b/src/flb_oauth2.c @@ -326,6 +326,118 @@ void flb_oauth2_destroy(struct flb_oauth2 *ctx) flb_free(ctx); } +char *flb_oauth2_token_get_ng(struct flb_oauth2 *ctx) +{ + int ret; + time_t now; + struct flb_http_client_ng http_client; + struct flb_http_response *response; + struct flb_http_request *request; + uint64_t http_client_flags; + + now = time(NULL); + if (ctx->access_token) { + /* validate unexpired token */ + if (ctx->expires > now && flb_sds_len(ctx->access_token) > 0) { + return ctx->access_token; + } + } + + http_client_flags = FLB_HTTP_CLIENT_FLAG_AUTO_DEFLATE | + FLB_HTTP_CLIENT_FLAG_AUTO_INFLATE; + + ret = flb_http_client_ng_init(&http_client, + NULL, + ctx->u, + HTTP_PROTOCOL_VERSION_11, + http_client_flags); + + if (ret != 0) { + flb_debug("[oauth2] http client creation error"); + + return NULL; + } + + request = flb_http_client_request_builder( + &http_client, + FLB_HTTP_CLIENT_ARGUMENT_METHOD(FLB_HTTP_POST), + FLB_HTTP_CLIENT_ARGUMENT_HOST(ctx->host), + FLB_HTTP_CLIENT_ARGUMENT_URI(ctx->uri), + FLB_HTTP_CLIENT_ARGUMENT_CONTENT_TYPE( + FLB_OAUTH2_HTTP_ENCODING), + FLB_HTTP_CLIENT_ARGUMENT_BODY(ctx->payload, + cfl_sds_len(ctx->payload), + NULL)); + + if (request == NULL) { + flb_stream_enable_flags(&ctx->u->base, FLB_IO_IPV6); + + request = flb_http_client_request_builder( + &http_client, + FLB_HTTP_CLIENT_ARGUMENT_METHOD(FLB_HTTP_POST), + FLB_HTTP_CLIENT_ARGUMENT_HOST(ctx->host), + FLB_HTTP_CLIENT_ARGUMENT_URI(ctx->uri), + FLB_HTTP_CLIENT_ARGUMENT_CONTENT_TYPE( + FLB_OAUTH2_HTTP_ENCODING), + FLB_HTTP_CLIENT_ARGUMENT_BODY(ctx->payload, + cfl_sds_len(ctx->payload), + NULL)); + if (request == NULL) { + flb_error("[oauth2] could not get an upstream connection to %s:%i", + ctx->u->tcp_host, ctx->u->tcp_port); + + flb_stream_disable_flags(&ctx->u->base, FLB_IO_IPV6); + flb_http_client_request_destroy(request, FLB_TRUE); + flb_http_client_ng_destroy(&http_client); + + return NULL; + } + } + + response = flb_http_client_request_execute(request); + + if (response == NULL) { + flb_debug("[oauth2] http request execution error"); + + flb_http_client_request_destroy(request, FLB_TRUE); + flb_http_client_ng_destroy(&http_client); + + return NULL; + } + + flb_info("[oauth2] HTTP Status=%i", response->status); + if (response->body != NULL && + cfl_sds_len(response->body) > 0) { + flb_info("[oauth2] payload:\n%s", response->body); + } + + /* Extract token */ + if (response->body != NULL && + cfl_sds_len(response->body) > 0 && + response->status == 200) { + ret = flb_oauth2_parse_json_response(response->body, + cfl_sds_len(response->body), + ctx); + if (ret == 0) { + flb_info("[oauth2] access token from '%s:%s' retrieved", + ctx->host, ctx->port); + + flb_http_client_request_destroy(request, FLB_TRUE); + flb_http_client_ng_destroy(&http_client); + + ctx->issued = time(NULL); + ctx->expires = ctx->issued + ctx->expires_in; + + return ctx->access_token; + } + } + + flb_http_client_request_destroy(request, FLB_TRUE); + flb_http_client_ng_destroy(&http_client); + + return NULL; +} + char *flb_oauth2_token_get(struct flb_oauth2 *ctx) { int ret; From f4f2de10dc0e3ab32aa8425e79a41ca203a44922 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 29 Oct 2024 23:04:48 +0100 Subject: [PATCH 04/21] signv4: new http client support added Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_signv4_ng.h | 54 ++ src/flb_signv4_ng.c | 1188 ++++++++++++++++++++++++++++ 2 files changed, 1242 insertions(+) create mode 100644 include/fluent-bit/flb_signv4_ng.h create mode 100644 src/flb_signv4_ng.c diff --git a/include/fluent-bit/flb_signv4_ng.h b/include/fluent-bit/flb_signv4_ng.h new file mode 100644 index 00000000000..05b1a5d2ddb --- /dev/null +++ b/include/fluent-bit/flb_signv4_ng.h @@ -0,0 +1,54 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include + +#ifdef FLB_HAVE_SIGNV4 + +#ifndef FLB_SIGNV4_NG_H +#define FLB_SIGNV4_NG_H + +#ifndef S3_MODE_NONE +/* Request is not Amazon S3 PutObject */ +#define S3_MODE_NONE 0 +#endif + +#ifndef S3_MODE_SIGNED_PAYLOAD +/* Set the x-amz-content-sha256 header with the sha value */ +#define S3_MODE_SIGNED_PAYLOAD 1 +#endif + +#ifndef S3_MODE_UNSIGNED_PAYLOAD +/* Set the x-amz-content-sha256 header with the value UNSIGNED-PAYLOAD */ +#define S3_MODE_UNSIGNED_PAYLOAD 2 +#endif + +flb_sds_t flb_signv4_ng_do(struct flb_http_request *request, + int normalize_uri, + int amz_date, + time_t t_now, + char *region, char *service, + int s3_mode, + struct mk_list *unsigned_headers, /* flb_slist */ + struct flb_aws_provider *provider); + +#endif +#endif /* FLB_HAVE_SIGNV4 */ diff --git a/src/flb_signv4_ng.c b/src/flb_signv4_ng.c new file mode 100644 index 00000000000..f7624ecfcfe --- /dev/null +++ b/src/flb_signv4_ng.c @@ -0,0 +1,1188 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * + * AWS Signv4 documentation + * + * https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +static flb_sds_t sha256_to_hex(unsigned char *sha256) +{ + int i; + flb_sds_t hex; + flb_sds_t tmp; + + hex = flb_sds_create_size(64); + if (!hex) { + flb_error("[signv4] cannot allocate buffer to convert sha256 to hex"); + return NULL; + } + + for (i = 0; i < 32; i++) { + tmp = flb_sds_printf(&hex, "%02x", sha256[i]); + if (!tmp) { + flb_error("[signedv4] error formatting sha256 -> hex"); + flb_sds_destroy(hex); + return NULL; + } + hex = tmp; + } + + return hex; +} + +static int hmac_sha256_sign(unsigned char out[32], + unsigned char *key, size_t key_len, + unsigned char *msg, size_t msg_len) +{ + int result; + + result = flb_hmac_simple(FLB_HASH_SHA256, + key, key_len, + msg, msg_len, + out, 32); + + if (result != FLB_CRYPTO_SUCCESS) { + return -1; + } + + return 0; +} + +static int kv_key_cmp(const void *a_arg, const void *b_arg) +{ + int ret; + struct flb_kv *kv_a = *(struct flb_kv **) a_arg; + struct flb_kv *kv_b = *(struct flb_kv **) b_arg; + + ret = strcmp(kv_a->key, kv_b->key); + if (ret == 0) { + /* + * NULL pointer is allowed in kv_a->val and kv_b->val. + * Handle NULL pointer cases before passing to strcmp. + */ + if (kv_a->val == NULL && kv_b->val == NULL) { + ret = 0; + } + else if (kv_a->val == NULL) { + ret = -1; + } + else if (kv_b->val == NULL) { + ret = 1; + } + else { + ret = strcmp(kv_a->val, kv_b->val); + } + } + + return ret; +} + +static inline int to_encode(char c) +{ + if ((c >= 48 && c <= 57) || /* 0-9 */ + (c >= 65 && c <= 90) || /* A-Z */ + (c >= 97 && c <= 122) || /* a-z */ + (c == '-' || c == '_' || c == '.' || c == '~' || c == '/' || + c == '=')) { + return FLB_FALSE; + } + + return FLB_TRUE; +} + +static inline int to_encode_path(char c) +{ + if ((c >= 48 && c <= 57) || /* 0-9 */ + (c >= 65 && c <= 90) || /* A-Z */ + (c >= 97 && c <= 122) || /* a-z */ + (c == '-' || c == '_' || c == '.' || c == '~' || c == '/')) { + return FLB_FALSE; + } + + return FLB_TRUE; +} + +static flb_sds_t uri_encode(const char *uri, size_t len) +{ + int i; + flb_sds_t buf = NULL; + flb_sds_t tmp = NULL; + int is_query_string = FLB_FALSE; + int do_encode = FLB_FALSE; + + buf = flb_sds_create_size(len * 2); + if (!buf) { + flb_error("[signv4] cannot allocate buffer for URI encoding"); + return NULL; + } + + for (i = 0; i < len; i++) { + if (uri[i] == '?') { + is_query_string = FLB_TRUE; + } + do_encode = FLB_FALSE; + + if (is_query_string == FLB_FALSE && to_encode_path(uri[i]) == FLB_TRUE) { + do_encode = FLB_TRUE; + } + if (is_query_string == FLB_TRUE && to_encode(uri[i]) == FLB_TRUE) { + do_encode = FLB_TRUE; + } + if (do_encode == FLB_TRUE) { + tmp = flb_sds_printf(&buf, "%%%02X", (unsigned char) *(uri + i)); + if (!tmp) { + flb_error("[signv4] error formatting special character"); + flb_sds_destroy(buf); + return NULL; + } + buf = tmp; + continue; + } + + /* Direct assignment, just copy the character */ + if (buf) { + tmp = flb_sds_cat(buf, uri + i, 1); + if (!tmp) { + flb_error("[signv4] error composing outgoing buffer"); + flb_sds_destroy(buf); + return NULL; + } + buf = tmp; + } + } + + return buf; +} + +/* + * Encodes URI parameters, which can not have "/" characters in them + * (This happens in an STS request, the role ARN has a slash and is + * given as a query parameter). + */ +static flb_sds_t uri_encode_params(const char *uri, size_t len) +{ + int i; + flb_sds_t buf = NULL; + flb_sds_t tmp = NULL; + + buf = flb_sds_create_size(len * 2); + if (!buf) { + flb_error("[signv4] cannot allocate buffer for URI encoding"); + return NULL; + } + + for (i = 0; i < len; i++) { + if (to_encode(uri[i]) == FLB_TRUE || uri[i] == '/') { + tmp = flb_sds_printf(&buf, "%%%02X", (unsigned char) *(uri + i)); + if (!tmp) { + flb_error("[signv4] error formatting special character"); + flb_sds_destroy(buf); + return NULL; + } + continue; + } + + /* Direct assignment, just copy the character */ + if (buf) { + tmp = flb_sds_cat(buf, uri + i, 1); + if (!tmp) { + flb_error("[signv4] error composing outgoing buffer"); + flb_sds_destroy(buf); + return NULL; + } + buf = tmp; + } + } + + return buf; +} + +/* + * Convert URL encoded params (query string or POST payload) to a sorted + * key/value linked list + */ +static flb_sds_t url_params_format(char *params) +{ + int i; + int ret; + int len; + int items; + char *p; + struct mk_list list; + struct mk_list split; + struct mk_list *h_tmp; + struct mk_list *head; + struct flb_slist_entry *e; + flb_sds_t key; + flb_sds_t val; + flb_sds_t tmp; + flb_sds_t buf = NULL; + struct flb_kv *kv; + struct flb_kv **arr; + + mk_list_init(&list); + mk_list_init(&split); + + ret = flb_slist_split_string(&split, params, '&', -1); + if (ret == -1) { + flb_error("[signv4] error processing given query string"); + flb_slist_destroy(&split); + flb_kv_release(&list); + return NULL; + } + + mk_list_foreach_safe(head, h_tmp, &split) { + e = mk_list_entry(head, struct flb_slist_entry, _head); + p = strchr(e->str, '='); + if (!p) { + continue; + } + + len = (p - e->str); + p++; + + /* URI encode every key and value */ + key = uri_encode_params(e->str, len); + len++; + val = uri_encode_params(p, flb_sds_len(e->str) - len); + if (!key || !val) { + flb_error("[signv4] error encoding uri for query string"); + if (key) { + flb_sds_destroy(key); + } + if (val) { + flb_sds_destroy(val); + } + flb_slist_destroy(&split); + flb_kv_release(&list); + return NULL; + } + + /* + * If key length is 0 then a problem occurs because val + * will not be set flb_kv_item_create_len, which eventually + * results in issues since kv->val will be equal to NULL. + * Thus, check here whether key length is satisfied + */ + if (flb_sds_len(key) == 0) { + flb_sds_destroy(key); + flb_sds_destroy(val); + flb_slist_destroy(&split); + flb_kv_release(&list); + return NULL; + } + + kv = flb_kv_item_create_len(&list, + key, flb_sds_len(key), + val, flb_sds_len(val)); + flb_sds_destroy(key); + flb_sds_destroy(val); + + if (!kv) { + flb_error("[signv4] error processing key/value from query string"); + flb_slist_destroy(&split); + flb_kv_release(&list); + return NULL; + } + } + flb_slist_destroy(&split); + + /* Sort the kv list of parameters */ + items = mk_list_size(&list); + if (items == 0) { + flb_kv_release(&list); + return flb_sds_create(""); + } + + arr = flb_calloc(1, sizeof(struct flb_kv *) * items); + if (!arr) { + flb_errno(); + flb_kv_release(&list); + return NULL; + } + + i = 0; + mk_list_foreach(head, &list) { + kv = mk_list_entry(head, struct flb_kv, _head); + arr[i] = kv; + i++; + } + /* sort headers by key */ + qsort(arr, items, sizeof(struct flb_kv *), kv_key_cmp); + + /* Format query string parameters */ + buf = flb_sds_create_size(items * 64); + if (!buf) { + flb_kv_release(&list); + flb_free(arr); + return NULL; + } + + for (i = 0; i < items; i++) { + kv = (struct flb_kv *) arr[i]; + if (i + 1 < items) { + if (kv->val == NULL) { + tmp = flb_sds_printf(&buf, "%s=&", + kv->key); + } + else { + tmp = flb_sds_printf(&buf, "%s=%s&", + kv->key, kv->val); + } + } + else { + if (kv->val == NULL) { + tmp = flb_sds_printf(&buf, "%s=", + kv->key); + } + else { + tmp = flb_sds_printf(&buf, "%s=%s", + kv->key, kv->val); + } + } + if (!tmp) { + flb_error("[signv4] error allocating value"); + } + buf = tmp; + } + + flb_kv_release(&list); + flb_free(arr); + + return buf; +} + +/* + * Given an original list of kv headers with 'in_list' as the list headed, + * generate new entries on 'out_list' considering lower case headers key, + * sorted by keys and values and merged duplicates. + */ +static void headers_sanitize_ng(struct flb_http_request *request, struct mk_list *out_list) +{ + int x; + char *v_start; + char *v_end; + char *val; + struct mk_list *head; + struct mk_list *c_head; + struct mk_list *tmp; + struct mk_list out_tmp; + struct flb_kv *kv; + struct flb_kv *c_kv; + flb_sds_t t; + struct flb_hash_table_entry *header_entry; + + mk_list_init(&out_tmp); + + /* Create lowercase key headers in the temporal list */ + mk_list_foreach(head, &request->headers->entries) { + header_entry = mk_list_entry(head, + struct flb_hash_table_entry, + _head_parent); + + /* Sanitize value */ + v_start = header_entry->val; + v_end = header_entry->val + header_entry->val_size; + + if (header_entry->val_size > 0) { + v_end--; + } + + while (*v_start == ' ' || *v_start == '\t') { + v_start++; + } + while (*v_end == ' ' || *v_end == '\t') { + v_end--; + } + + /* + * The original headers might have upper case characters, for safety just + * make a copy of them so we can lowercase them if required. + */ + kv = flb_kv_item_create_len(&out_tmp, + header_entry->key, + header_entry->key_len, + v_start, v_end - v_start); + if (kv == NULL) { + continue; + } + for (x = 0; x < flb_sds_len(kv->key); x++) { + kv->key[x] = tolower(kv->key[x]); + } + + /* + * trim: kv->val alreay have a copy of the original value, now we need + * to look for double empty spaces in the middle of the value and do + * proper adjustments. + */ + val = kv->val; + while (v_start < v_end) { + if (*v_start == ' ') { + if (v_start < v_end && *(v_start + 1) == ' ') { + v_start++; + continue; + } + } + *val = *v_start; + v_start++; + val++; + } + *val = '\0'; + flb_sds_len_set(kv->val, val - kv->val); + } + + /* Find and merge duplicates */ + mk_list_foreach_safe(head, tmp, &out_tmp) { + kv = mk_list_entry(head, struct flb_kv, _head); + + /* Check if this kv exists in out_list */ + c_kv = NULL; + mk_list_foreach(c_head, out_list) { + c_kv = mk_list_entry(c_head, struct flb_kv, _head); + if (strcmp(kv->key, c_kv->key) == 0) { + break; + } + c_kv = NULL; + } + + /* if c_kv is set, means the key already exists in the outgoing list */ + if (c_kv) { + t = flb_sds_printf(&c_kv->val, ",%s", kv->val); + c_kv->val = t; + flb_kv_item_destroy(kv); + } + else { + mk_list_del(&kv->_head); + mk_list_add(&kv->_head, out_list); + } + } +} + +/* + * Task 1: Create a canonical request + * ================================== + * + * https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html + * + * CanonicalRequest = + * HTTPRequestMethod + '\n' + + * CanonicalURI + '\n' + + * CanonicalQueryString + '\n' + + * CanonicalHeaders + '\n' + + * SignedHeaders + '\n' + + * HexEncode(Hash(RequestPayload)) + */ +static flb_sds_t flb_signv4_ng_canonical_request(struct flb_http_request *request, + int normalize_uri, + int amz_date_header, + char *amzdate, + char *security_token, + int s3_mode, + struct mk_list *excluded_headers, + flb_sds_t *signed_headers) +{ + int i; + int len; + int items; + int all_items; + int excluded_items; + int post_params = FLB_FALSE; + int result; + size_t size; + int skip_header; + char *val; + struct flb_kv **arr; + flb_sds_t cr; + flb_sds_t uri; + flb_sds_t tmp = NULL; + flb_sds_t params = NULL; + flb_sds_t payload_hash = NULL; + struct flb_kv *kv; + struct mk_list list_tmp; + struct mk_list *head; + struct mk_list *head_2; + struct flb_slist_entry *sle; + unsigned char sha256_buf[64] = {0}; + + /* Size hint */ + size = strlen(request->path) + + (request->headers->total_count * 64) + + 256; + + cr = flb_sds_create_size(size); + if (!cr) { + flb_error("[signv4] cannot allocate buffer"); + return NULL; + } + + switch (request->method) { + case FLB_HTTP_GET: + tmp = flb_sds_cat(cr, "GET\n", 4); + break; + case FLB_HTTP_POST: + tmp = flb_sds_cat(cr, "POST\n", 5); + break; + case FLB_HTTP_PUT: + tmp = flb_sds_cat(cr, "PUT\n", 4); + break; + case FLB_HTTP_HEAD: + tmp = flb_sds_cat(cr, "HEAD\n", 5); + break; + }; + + if (!tmp) { + flb_error("[signv4] invalid processing of HTTP method"); + flb_sds_destroy(cr); + return NULL; + } + + cr = tmp; + + /* + * URI normalization is required by certain AWS service, for hence the caller + * plugin is responsible to enable/disable this flag. If set the URI in the + * canonical request will be normalized. + */ + if (normalize_uri == FLB_TRUE) { + tmp = flb_signv4_uri_normalize_path((char *) request->path, + strlen(request->path)); + if (!tmp) { + flb_error("[signv4] error normalizing path"); + flb_sds_destroy(cr); + return NULL; + } + len = flb_sds_len(tmp); + } + else { + tmp = (char *) request->path; + } + + /* Do URI encoding (rfc3986) */ + uri = uri_encode(tmp, len); + if (tmp != request->path) { + flb_sds_destroy(tmp); + } + if (!uri) { + /* error composing outgoing buffer */ + flb_sds_destroy(cr); + return NULL; + } + + tmp = flb_sds_cat(cr, uri, flb_sds_len(uri)); + if (!tmp) { + flb_error("[signv4] error concatenating encoded URI"); + flb_sds_destroy(uri); + flb_sds_destroy(cr); + return NULL; + } + cr = tmp; + flb_sds_destroy(uri); + + tmp = flb_sds_cat(cr, "\n", 1); + if (!tmp) { + flb_error("[signv4] error concatenating encoded URI break line"); + flb_sds_destroy(cr); + return NULL; + } + cr = tmp; + + /* Canonical Query String */ + tmp = NULL; + if (request->query_string) { + params = url_params_format((char *) request->query_string); + if (!params) { + flb_sds_destroy(cr); + return NULL; + } + tmp = flb_sds_cat(cr, params, flb_sds_len(params)); + if (!tmp) { + flb_error("[signv4] error concatenating query string"); + flb_sds_destroy(params); + flb_sds_destroy(cr); + return NULL; + } + flb_sds_destroy(params); + cr = tmp; + } + + /* + * If the original HTTP method is POST and we have some urlencoded parameters + * as payload, we must handle them as we did for the query string. + */ + if (request->method == HTTP_METHOD_POST && + (request->body != NULL && cfl_sds_len(request->body) > 0)) { + val = (char *) flb_http_request_get_header(request, "Content-Type"); + + if (val) { + if (strstr(val, "application/x-www-form-urlencoded")) { + params = url_params_format((char *) request->body); + if (!params) { + flb_error("[signv4] error processing POST payload params"); + flb_sds_destroy(cr); + return NULL; + } + tmp = flb_sds_cat(cr, params, flb_sds_len(params)); + if (!tmp) { + flb_error("[signv4] error concatenating POST payload params"); + flb_sds_destroy(params); + flb_sds_destroy(cr); + return NULL; + } + cr = tmp; + flb_sds_destroy(params); + post_params = FLB_TRUE; + } + } + } + + /* query string / POST separator */ + tmp = flb_sds_cat(cr, "\n", 1); + if (!tmp) { + flb_error("[signv4] error adding params breakline separator"); + flb_sds_destroy(cr); + return NULL; + } + cr = tmp; + + /* + * Calculate payload hash. + * This is added at the end of all canonical requests, unless + * S3_MODE_UNSIGNED_PAYLOAD is set. + * If we're using S3_MODE_SIGNED_PAYLOAD, then the hash is added to the + * canonical headers. + */ + if (s3_mode == S3_MODE_UNSIGNED_PAYLOAD) { + payload_hash = flb_sds_create("UNSIGNED-PAYLOAD"); + } else { + if ((request->body != NULL && cfl_sds_len(request->body) > 0) && + post_params == FLB_FALSE) { + result = flb_hash_simple(FLB_HASH_SHA256, + (unsigned char *) request->body, + cfl_sds_len(request->body), + sha256_buf, + sizeof(sha256_buf)); + } + else { + result = flb_hash_simple(FLB_HASH_SHA256, + (unsigned char *) NULL, 0, + sha256_buf, sizeof(sha256_buf)); + } + + if (result != FLB_CRYPTO_SUCCESS) { + flb_error("[signv4] error hashing payload"); + flb_sds_destroy(cr); + return NULL; + } + + payload_hash = flb_sds_create_size(64); + if (!payload_hash) { + flb_error("[signv4] error formatting hashed payload"); + flb_sds_destroy(cr); + return NULL; + } + for (i = 0; i < 32; i++) { + tmp = flb_sds_printf(&payload_hash, "%02x", + (unsigned char) sha256_buf[i]); + if (!tmp) { + flb_error("[signv4] error formatting hashed payload"); + flb_sds_destroy(cr); + flb_sds_destroy(payload_hash); + return NULL; + } + payload_hash = tmp; + } + } + + /* + * Canonical Headers + * + * Add the required custom headers: + * + * - x-amz-date + * - x-amz-security-token (if set) + * - x-amz-content-sha256 (if S3_MODE_SIGNED_PAYLOAD) + */ + mk_list_init(&list_tmp); + + /* include x-amz-date header ? */ + if (amz_date_header == FLB_TRUE) { + flb_http_request_set_header(request, + "x-amz-date", 0, + amzdate, 0); + } + + /* x-amz-security_token */ + if (security_token) { + flb_http_request_set_header(request, + "x-amz-security-token", 0, + security_token, 0); + } + + if (s3_mode == S3_MODE_SIGNED_PAYLOAD) { + flb_http_request_set_header(request, + "-amz-content-sha256", 0, + payload_hash, 0); + } + + headers_sanitize_ng(request, &list_tmp); + + /* + * For every header registered, append it to the temporal array so we can sort them + * later. + */ + all_items = mk_list_size(&list_tmp); + excluded_items = 0; + size = (sizeof(struct flb_kv *) * (all_items)); + arr = flb_calloc(1, size); + if (!arr) { + flb_errno(); + flb_kv_release(&list_tmp); + flb_sds_destroy(cr); + flb_sds_destroy(payload_hash); + return NULL; + } + + /* Compose temporal array to sort headers */ + i = 0; + mk_list_foreach(head, &list_tmp) { + kv = mk_list_entry(head, struct flb_kv, _head); + + /* Skip excluded headers */ + if (excluded_headers) { + skip_header = FLB_FALSE; + mk_list_foreach(head_2, excluded_headers) { + sle = mk_list_entry(head_2, struct flb_slist_entry, _head); + if (flb_sds_casecmp(kv->key, sle->str, flb_sds_len(sle->str)) == 0) { + + /* Skip header */ + excluded_items++; + skip_header = FLB_TRUE; + break; + } + } + if (skip_header) { + continue; + } + } + + arr[i] = kv; + i++; + } + + /* Count items */ + items = all_items - excluded_items; + + /* Sort the headers from the temporal array */ + qsort(arr, items, sizeof(struct flb_kv *), kv_key_cmp); + + /* Iterate sorted headers and append them to the outgoing buffer */ + for (i = 0; i < items; i++) { + kv = (struct flb_kv *) arr[i]; + tmp = flb_sds_printf(&cr, "%s:%s\n", kv->key, kv->val); + if (!tmp) { + flb_error("[signv4] error composing canonical headers"); + flb_free(arr); + flb_kv_release(&list_tmp); + flb_sds_destroy(cr); + flb_sds_destroy(payload_hash); + return NULL; + } + cr = tmp; + } + + /* Add required breakline */ + tmp = flb_sds_printf(&cr, "\n"); + if (!tmp) { + flb_error("[signv4] error adding extra breakline separator"); + flb_free(arr); + flb_kv_release(&list_tmp); + flb_sds_destroy(cr); + flb_sds_destroy(payload_hash); + return NULL; + } + cr = tmp; + + /* Signed Headers for canonical request context */ + for (i = 0; i < items; i++) { + kv = (struct flb_kv *) arr[i]; + + /* Check if this is the last header, if so add breakline separator */ + if (i + 1 == items) { + tmp = flb_sds_printf(&cr, "%s\n", kv->key); + } + else { + tmp = flb_sds_printf(&cr, "%s;", kv->key); + } + if (!tmp) { + flb_error("[signv4] error composing canonical signed headers"); + flb_free(arr); + flb_kv_release(&list_tmp); + flb_sds_destroy(cr); + flb_sds_destroy(payload_hash); + return NULL; + } + cr = tmp; + } + + /* Signed Headers for authorization header (Task 4) */ + for (i = 0; i < items; i++) { + kv = (struct flb_kv *) arr[i]; + + /* Check if this is the last header, if so add breakline separator */ + if (i + 1 == items) { + tmp = flb_sds_printf(signed_headers, "%s", kv->key); + } + else { + tmp = flb_sds_printf(signed_headers, "%s;", kv->key); + } + if (!tmp) { + flb_error("[signv4] error composing auth signed headers"); + flb_free(arr); + flb_kv_release(&list_tmp); + flb_sds_destroy(cr); + flb_sds_destroy(payload_hash); + return NULL; + } + *signed_headers = tmp; + } + + flb_free(arr); + flb_kv_release(&list_tmp); + + /* Add Payload Hash */ + tmp = flb_sds_printf(&cr, "%s", payload_hash); + if (!tmp) { + flb_error("[signv4] error adding payload hash"); + flb_sds_destroy(cr); + flb_sds_destroy(payload_hash); + return NULL; + } + cr = tmp; + flb_sds_destroy(payload_hash); + + return cr; +} + +/* + * Task 2 + * ====== + * https://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html + */ +static flb_sds_t flb_signv4_string_to_sign(struct flb_http_request *request, + flb_sds_t cr, char *amzdate, + char *datestamp, char *service, + char *region) +{ + int i; + int result; + flb_sds_t tmp; + flb_sds_t sign; + unsigned char sha256_buf[64] = {0}; + + sign = flb_sds_create_size(256); + if (!sign) { + flb_error("[signv4] cannot create buffer for signature"); + return NULL; + } + + /* Hashing Algorithm */ + tmp = flb_sds_cat(sign, "AWS4-HMAC-SHA256\n", 17); + if (!tmp) { + flb_error("[signv4] cannot add algorithm to signature"); + flb_sds_destroy(sign); + return NULL; + } + sign = tmp; + + /* Amazon date */ + tmp = flb_sds_printf(&sign, "%s\n", amzdate); + if (!tmp) { + flb_error("[signv4] cannot add amz-date to signature"); + flb_sds_destroy(sign); + return NULL; + } + sign = tmp; + + /* Credentials Scope */ + tmp = flb_sds_printf(&sign, "%s/%s/%s/aws4_request\n", + datestamp, region, service); + if (!tmp) { + flb_error("[signv4] cannot add credentials scope to signature"); + flb_sds_destroy(sign); + return NULL; + } + + /* Hash of Canonical Request */ + result = flb_hash_simple(FLB_HASH_SHA256, + (unsigned char *) cr, flb_sds_len(cr), + sha256_buf, sizeof(sha256_buf)); + + if (result != FLB_CRYPTO_SUCCESS) { + flb_error("[signv4] error hashing canonical request"); + flb_sds_destroy(sign); + return NULL; + } + + for (i = 0; i < 32; i++) { + tmp = flb_sds_printf(&sign, "%02x", (unsigned char) sha256_buf[i]); + if (!tmp) { + flb_error("[signv4] error formatting hashed canonical request"); + flb_sds_destroy(sign); + return NULL; + } + sign = tmp; + } + + return sign; +} + +/* + * Task 3 + * ====== + * + * https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html + * + * TODO: The signing key could be cached to improve performance + */ +static flb_sds_t flb_signv4_calculate_signature(flb_sds_t string_to_sign, + char *datestamp, char *service, + char *region, char *secret_key) +{ + int len; + int klen = 32; + flb_sds_t tmp; + flb_sds_t key; + unsigned char key_date[32]; + unsigned char key_region[32]; + unsigned char key_service[32]; + unsigned char key_signing[32]; + unsigned char signature[32]; + + /* Compose initial key */ + key = flb_sds_create_size(256); + if (!key) { + flb_error("[signv4] cannot create buffer for signature calculation"); + return NULL; + } + + tmp = flb_sds_printf(&key, "AWS4%s", secret_key); + if (!tmp) { + flb_error("[signv4] error formatting initial key"); + flb_sds_destroy(key); + return NULL; + } + key = tmp; + + /* key_date */ + len = strlen(datestamp); + hmac_sha256_sign(key_date, (unsigned char *) key, flb_sds_len(key), + (unsigned char *) datestamp, len); + flb_sds_destroy(key); + + /* key_region */ + len = strlen(region); + hmac_sha256_sign(key_region, key_date, klen, (unsigned char *) region, len); + + /* key_service */ + len = strlen(service); + hmac_sha256_sign(key_service, key_region, klen, (unsigned char *) service, len); + + /* key_signing */ + hmac_sha256_sign(key_signing, key_service, klen, + (unsigned char *) "aws4_request", 12); + + /* Signature */ + hmac_sha256_sign(signature, key_signing, klen, + (unsigned char *) string_to_sign, flb_sds_len(string_to_sign)); + + return sha256_to_hex(signature); +} + +/* + * Task 4 + * ====== + * + * https://docs.aws.amazon.com/general/latest/gr/sigv4-add-signature-to-request.html + */ +static flb_sds_t flb_signv4_add_authorization(struct flb_http_request *request, + char *access_key, + char *datestamp, + char *region, char *service, + flb_sds_t signed_headers, + flb_sds_t signature) +{ + int ret; + flb_sds_t tmp; + flb_sds_t header_value; + + header_value = flb_sds_create_size(512); + if (!header_value) { + flb_error("[signv4] cannot allocate buffer for authorization header"); + return NULL; + } + + tmp = flb_sds_printf(&header_value, + "AWS4-HMAC-SHA256 Credential=%s/%s/%s/%s/aws4_request, " + "SignedHeaders=%s, Signature=%s", + access_key, datestamp, region, service, + signed_headers, signature); + if (!tmp) { + flb_error("[signv4] error composing authorization header"); + flb_sds_destroy(header_value); + return NULL; + } + header_value = tmp; + + ret = flb_http_request_set_header(request, + "Authorization", 0, + header_value, 0); + + if (ret != 0) { + flb_error("[signv4] could not add authorization header"); + flb_sds_destroy(header_value); + return NULL; + + } + + /* Return the composed final header for testing if required */ + return header_value; +} + +flb_sds_t flb_signv4_ng_do(struct flb_http_request *request, + int normalize_uri, + int amz_date_header, + time_t t_now, + char *region, char *service, + int s3_mode, + struct mk_list *unsigned_headers, + struct flb_aws_provider *provider) +{ + char amzdate[32]; + char datestamp[32]; + struct tm *gmt; + flb_sds_t cr; + flb_sds_t string_to_sign; + flb_sds_t signature; + flb_sds_t signed_headers; + flb_sds_t auth_header; + struct flb_aws_credentials *creds; + + creds = provider->provider_vtable->get_credentials(provider); + if (!creds) { + flb_error("[signv4] Provider returned no credentials, service=%s", + service); + return NULL; + } + + gmt = flb_calloc(1, sizeof(struct tm)); + if (!gmt) { + flb_errno(); + flb_aws_credentials_destroy(creds); + return NULL; + } + + if (!gmtime_r(&t_now, gmt)) { + flb_error("[signv4] error converting given unix timestamp"); + flb_free(gmt); + flb_aws_credentials_destroy(creds); + return NULL; + } + + strftime(amzdate, sizeof(amzdate) - 1, "%Y%m%dT%H%M%SZ", gmt); + strftime(datestamp, sizeof(datestamp) - 1, "%Y%m%d", gmt); + flb_free(gmt); + + /* Task 1: canonical request */ + signed_headers = flb_sds_create_size(256); + if (!signed_headers) { + flb_error("[signedv4] cannot allocate buffer for auth signed headers"); + flb_aws_credentials_destroy(creds); + return NULL; + } + + cr = flb_signv4_ng_canonical_request(request, normalize_uri, + amz_date_header, amzdate, + creds->session_token, s3_mode, + unsigned_headers, + &signed_headers); + if (!cr) { + flb_error("[signv4] failed canonical request"); + flb_sds_destroy(signed_headers); + flb_aws_credentials_destroy(creds); + return NULL; + } + + /* Task 2: string to sign */ + string_to_sign = flb_signv4_string_to_sign(request, cr, amzdate, + datestamp, service, region); + if (!string_to_sign) { + flb_error("[signv4] failed string to sign"); + flb_sds_destroy(cr); + flb_sds_destroy(signed_headers); + flb_aws_credentials_destroy(creds); + return NULL; + } + flb_sds_destroy(cr); + + /* Task 3: calculate the signature */ + signature = flb_signv4_calculate_signature(string_to_sign, datestamp, + service, region, + creds->secret_access_key); + if (!signature) { + flb_error("[signv4] failed calculate_string"); + flb_sds_destroy(signed_headers); + flb_sds_destroy(string_to_sign); + flb_aws_credentials_destroy(creds); + return NULL; + } + flb_sds_destroy(string_to_sign); + + /* Task 4: add signature to HTTP request */ + auth_header = flb_signv4_add_authorization(request, + creds->access_key_id, + datestamp, region, service, + signed_headers, signature); + flb_sds_destroy(signed_headers); + flb_sds_destroy(signature); + flb_aws_credentials_destroy(creds); + + if (!auth_header) { + flb_error("[signv4] error creating authorization header"); + return NULL; + } + + return auth_header; +} From 3e9e4f493d86fdbac8a9c50857a864dc7052fef4 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 29 Oct 2024 23:05:48 +0100 Subject: [PATCH 05/21] lock: locking layer previously used by the processor stack generalized Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_lock.h | 51 ++++++++++++++++++ src/flb_lock.c | 99 +++++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 include/fluent-bit/flb_lock.h create mode 100644 src/flb_lock.c diff --git a/include/fluent-bit/flb_lock.h b/include/fluent-bit/flb_lock.h new file mode 100644 index 00000000000..2f6f24f78a4 --- /dev/null +++ b/include/fluent-bit/flb_lock.h @@ -0,0 +1,51 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2024 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FLB_LOCK_H +#define FLB_LOCK_H + +#include +#include + +/* The current values mean the system will + * wait for 100 seconds at most in 50 millisecond increments. + * + * This is the worst case scenario and in reality there will + * be no wait in 99.9% of the cases. + */ + +#define FLB_LOCK_INFINITE_RETRY_LIMIT 0 +#define FLB_LOCK_DEFAULT_RETRY_LIMIT 100 +#define FLB_LOCK_DEFAULT_RETRY_DELAY 50000 + +typedef pthread_mutex_t flb_lock_t; + +int flb_lock_init(flb_lock_t *lock); + +int flb_lock_destroy(flb_lock_t *lock); + +int flb_lock_acquire(flb_lock_t *lock, + size_t retry_limit, + size_t retry_delay); + +int flb_lock_release(flb_lock_t *lock, + size_t retry_limit, + size_t retry_delay); + +#endif diff --git a/src/flb_lock.c b/src/flb_lock.c new file mode 100644 index 00000000000..212bb19d4ea --- /dev/null +++ b/src/flb_lock.c @@ -0,0 +1,99 @@ +#include +#include +#include + +int flb_lock_init(flb_lock_t *lock) +{ + int result; + + result = pthread_mutex_init(lock, NULL); + + if (result != 0) { + result = -1; + } + + return result; +} + +int flb_lock_destroy(flb_lock_t *lock) +{ + int result; + + result = pthread_mutex_destroy(lock); + + if (result != 0) { + result = -1; + } + + return result; +} + +int flb_lock_acquire(flb_lock_t *lock, + size_t retry_limit, + size_t retry_delay) +{ + size_t retry_count; + int result; + + retry_count = 0; + + do { + result = pthread_mutex_trylock(lock); + + if (result != 0) { + if (result == EAGAIN) { + if (retry_limit != FLB_LOCK_INFINITE_RETRY_LIMIT) { + retry_count++; + } + + usleep(retry_delay); + } + else { + break; + } + } + } + while (result != 0 && + retry_count < retry_limit); + + if (result != 0) { + result = -1; + } + + return result; +} + +int flb_lock_release(pthread_mutex_t *lock, + size_t retry_limit, + size_t retry_delay) +{ + size_t retry_count; + int result; + + retry_count = 0; + + do { + result = pthread_mutex_unlock(lock); + + if (result != 0) { + if (result == EAGAIN) { + if (retry_limit != FLB_LOCK_INFINITE_RETRY_LIMIT) { + retry_count++; + } + + usleep(retry_delay); + } + else { + break; + } + } + } + while (result != 0 && + retry_count < retry_limit); + + if (result != 0) { + result = -1; + } + + return result; +} From 494b84f8fd7b5af00a9b4f364a58102f3fff8f88 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 29 Oct 2024 23:06:37 +0100 Subject: [PATCH 06/21] http_common: client side specific request and response additions Signed-off-by: Leonardo Alminana --- include/fluent-bit/flb_http_common.h | 153 +++- src/flb_http_common.c | 1180 +++++++++++++++++++++++++- 2 files changed, 1284 insertions(+), 49 deletions(-) diff --git a/include/fluent-bit/flb_http_common.h b/include/fluent-bit/flb_http_common.h index e87cfe68db6..9d4c593dcdd 100644 --- a/include/fluent-bit/flb_http_common.h +++ b/include/fluent-bit/flb_http_common.h @@ -22,12 +22,15 @@ #include +#include +#include + #include #include -/* These definitions are temporary and should be moved - * to monkey. - * This fallback has been added to be able to merge this +/* These definitions are temporary and should be moved + * to monkey. + * This fallback has been added to be able to merge this * feature with the current monkey version. */ @@ -39,11 +42,11 @@ #define MK_HTTP_PROTOCOL_20_STR "HTTP/2" #endif -#define HTTP_PROTOCOL_AUTODETECT -1 -#define HTTP_PROTOCOL_HTTP0 0 -#define HTTP_PROTOCOL_HTTP1 1 -#define HTTP_PROTOCOL_HTTP2 2 +#ifndef MK_METHOD_CONNECT +#define MK_METHOD_CONNECT (MK_METHOD_UNKNOWN + 1) +#endif +#define HTTP_PROTOCOL_VERSION_AUTODETECT -1 #define HTTP_PROTOCOL_VERSION_09 MK_HTTP_PROTOCOL_09 #define HTTP_PROTOCOL_VERSION_10 MK_HTTP_PROTOCOL_10 #define HTTP_PROTOCOL_VERSION_11 MK_HTTP_PROTOCOL_11 @@ -55,17 +58,30 @@ #define HTTP_METHOD_PUT MK_METHOD_PUT #define HTTP_METHOD_DELETE MK_METHOD_DELETE #define HTTP_METHOD_OPTIONS MK_METHOD_OPTIONS +#define HTTP_METHOD_CONNECT MK_METHOD_CONNECT #define HTTP_METHOD_UNKNOWN MK_METHOD_UNKNOWN #define HTTP_STREAM_ROLE_SERVER 0 #define HTTP_STREAM_ROLE_CLIENT 1 -#define HTTP_STREAM_STATUS_RECEIVING_HEADERS 0 -#define HTTP_STREAM_STATUS_RECEIVING_DATA 1 -#define HTTP_STREAM_STATUS_READY 2 -#define HTTP_STREAM_STATUS_PROCESSING 3 -#define HTTP_STREAM_STATUS_CLOSED 4 -#define HTTP_STREAM_STATUS_ERROR 5 +#define HTTP_STREAM_STATUS_SENDING_HEADERS 0 +#define HTTP_STREAM_STATUS_SENDING_DATA 1 +#define HTTP_STREAM_STATUS_SENDING_TRAILER 2 +#define HTTP_STREAM_STATUS_RECEIVING_HEADERS 3 +#define HTTP_STREAM_STATUS_RECEIVING_DATA 4 +#define HTTP_STREAM_STATUS_RECEIVING_TRAILER 5 +#define HTTP_STREAM_STATUS_READY 6 +#define HTTP_STREAM_STATUS_PROCESSING 7 +#define HTTP_STREAM_STATUS_CLOSED 8 +#define HTTP_STREAM_STATUS_ERROR 9 + +#define HTTP_WWW_AUTHORIZATION_SCHEME_NONE 0 +#define HTTP_WWW_AUTHORIZATION_SCHEME_BASIC (((uint64_t) 1) << 0) +#define HTTP_WWW_AUTHORIZATION_SCHEME_BEARER (((uint64_t) 1) << 1) +#define HTTP_WWW_AUTHORIZATION_SCHEME_SIGNV4 (((uint64_t) 1) << 2) + +#define HTTP_PROXY_AUTHORIZATION_SCHEME_BASIC 10 +#define HTTP_PROXY_AUTHORIZATION_SCHEME_BEARER 11 struct flb_http_stream; struct flb_http_server_session; @@ -75,27 +91,37 @@ struct flb_http_request { int method; cfl_sds_t path; cfl_sds_t host; + uint16_t port; cfl_sds_t query_string; struct flb_hash_table *headers; + struct flb_hash_table *trailer_headers; + cfl_sds_t user_agent; size_t content_length; char *content_type; cfl_sds_t body; + size_t body_read_offset; struct flb_http_stream *stream; + int releasable; struct cfl_list _head; }; struct flb_http_response { + int protocol_version; int status; cfl_sds_t message; struct flb_hash_table *headers; struct flb_hash_table *trailer_headers; size_t content_length; + char *content_type; cfl_sds_t body; size_t body_read_offset; struct flb_http_stream *stream; + + int releasable; + struct cfl_list _head; }; struct flb_http_stream { @@ -113,12 +139,18 @@ struct flb_http_stream { struct cfl_list _head; }; +struct flb_aws_provider; + /* HTTP REQUEST */ int flb_http_request_init(struct flb_http_request *request); +struct flb_http_request *flb_http_request_create(); + void flb_http_request_destroy(struct flb_http_request *request); +int flb_http_request_commit(struct flb_http_request *request); + char *flb_http_request_get_header(struct flb_http_request *request, char *name); @@ -129,48 +161,127 @@ int flb_http_request_set_header(struct flb_http_request *request, int flb_http_request_unset_header(struct flb_http_request *request, char *name); +int flb_http_request_set_method(struct flb_http_request *request, + int method); + +int flb_http_request_set_host(struct flb_http_request *request, + char *host); + +int flb_http_request_set_port(struct flb_http_request *request, + uint16_t port); + +int flb_http_request_set_url(struct flb_http_request *request, + char *url); + +int flb_http_request_set_uri(struct flb_http_request *request, + char *uri); + +int flb_http_request_set_query_string(struct flb_http_request *request, + char *query_string); + +int flb_http_request_set_content_type(struct flb_http_request *request, + char *content_type); + +int flb_http_request_set_user_agent(struct flb_http_request *request, + char *user_agent); + +int flb_http_request_set_content_length(struct flb_http_request *request, + size_t content_length); + +int flb_http_request_set_content_encoding(struct flb_http_request *request, + char *encoding); + +int flb_http_request_set_body(struct flb_http_request *request, + unsigned char *body, size_t body_length, + char *compression_algorithm); + +int flb_http_request_set_authorization(struct flb_http_request *request, + int type, ...); + +int flb_http_request_compress_body( + struct flb_http_request *request, + char *content_encoding_header_value); + +int flb_http_request_uncompress_body( + struct flb_http_request *request); + +int flb_http_request_perform_signv4_signature( + struct flb_http_request *request, + const char *aws_region, + const char *aws_service, + struct flb_aws_provider *aws_provider); + /* HTTP RESPONSE */ int flb_http_response_init(struct flb_http_response *response); +struct flb_http_response *flb_http_response_create(); + void flb_http_response_destroy(struct flb_http_response *response); struct flb_http_response *flb_http_response_begin( - struct flb_http_server_session *session, + struct flb_http_server_session *session, void *stream); int flb_http_response_commit(struct flb_http_response *response); -int flb_http_response_set_header(struct flb_http_response *response, +char *flb_http_response_get_header(struct flb_http_response *response, + char *name); + +int flb_http_response_set_header(struct flb_http_response *response, char *name, size_t name_length, char *value, size_t value_length); -int flb_http_response_set_trailer_header(struct flb_http_response *response, +int flb_http_response_unset_header(struct flb_http_response *response, + char *name); + +int flb_http_response_set_trailer_header(struct flb_http_response *response, char *name, size_t name_length, char *value, size_t value_length); -int flb_http_response_set_status(struct flb_http_response *response, +int flb_http_response_set_status(struct flb_http_response *response, int status); -int flb_http_response_set_message(struct flb_http_response *response, +int flb_http_response_set_message(struct flb_http_response *response, char *message); -int flb_http_response_set_body(struct flb_http_response *response, +int flb_http_response_set_body(struct flb_http_response *response, unsigned char *body, size_t body_length); +int flb_http_response_append_to_body(struct flb_http_response *response, + unsigned char *body, size_t body_length); + +int flb_http_response_compress_body( + struct flb_http_response *response, + char *content_encoding_header_value); + +int flb_http_response_uncompress_body( + struct flb_http_response *response); + + /* HTTP STREAM */ int flb_http_stream_init(struct flb_http_stream *stream, - void *parent, + void *parent, int32_t stream_id, int role, void *user_data); -struct flb_http_stream *flb_http_stream_create(void *parent, +struct flb_http_stream *flb_http_stream_create(void *parent, int32_t stream_id, int role, void *user_data); void flb_http_stream_destroy(struct flb_http_stream *stream); + +const char *flb_http_get_method_string_from_id(int method); + +char *flb_http_server_convert_string_to_lowercase(char *input_buffer, + size_t length); + +int flb_http_server_strncasecmp(const uint8_t *first_buffer, + size_t first_length, + const char *second_buffer, + size_t second_length); #endif diff --git a/src/flb_http_common.c b/src/flb_http_common.c index 23290f5ccf3..b39391147a3 100644 --- a/src/flb_http_common.c +++ b/src/flb_http_common.c @@ -21,9 +21,88 @@ #include #include +#include +#include +#include + +/* PRIVATE */ + +static \ +int uncompress_zlib(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size); + +static \ +int uncompress_zstd(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size); + +static \ +int uncompress_deflate(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size); + +static \ +int uncompress_snappy(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size); + +static \ +int uncompress_gzip(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size); + +static \ +int compress_zlib(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size); + +static \ +int compress_zstd(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size); + +static \ +int compress_deflate(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size); + +static \ +int compress_snappy(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size); + +static \ +int compress_gzip(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size); /* HTTP REQUEST */ +static int flb_http_request_get_version(struct flb_http_request *request) +{ + int version; + + if (request->stream->role == HTTP_STREAM_ROLE_SERVER) { + version = ((struct flb_http_server_session *) request->stream->parent)->version; + } + else { + version = ((struct flb_http_client_session *) request->stream->parent)->protocol_version; + } + + return version; +} + int flb_http_request_init(struct flb_http_request *request) { flb_http_request_destroy(request); @@ -36,9 +115,39 @@ int flb_http_request_init(struct flb_http_request *request) return -1; } + request->trailer_headers = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 16, -1); + + if (request->trailer_headers == NULL) { + return -1; + } + return 0; } +struct flb_http_request *flb_http_request_create() +{ + struct flb_http_request *request; + int result; + + request = flb_calloc(1, sizeof(struct flb_http_request)); + + if (request == NULL) { + return NULL; + } + + request->releasable = FLB_TRUE; + + result = flb_http_request_init(request); + + if (result != 0) { + flb_http_request_destroy(request); + + return NULL; + } + + return request; +} + void flb_http_request_destroy(struct flb_http_request *request) { if (request->path != NULL) { @@ -53,6 +162,10 @@ void flb_http_request_destroy(struct flb_http_request *request) cfl_sds_destroy(request->content_type); } + if (request->user_agent != NULL) { + cfl_sds_destroy(request->user_agent); + } + if (request->query_string != NULL) { cfl_sds_destroy(request->query_string); } @@ -65,11 +178,32 @@ void flb_http_request_destroy(struct flb_http_request *request) flb_hash_table_destroy(request->headers); } + if (request->trailer_headers != NULL) { + flb_hash_table_destroy(request->trailer_headers); + } + if (!cfl_list_entry_is_orphan(&request->_head)) { cfl_list_del(&request->_head); } memset(request, 0, sizeof(struct flb_http_request)); + + if (request->releasable == FLB_TRUE) { + flb_free(request); + } +} + +int flb_http_request_commit(struct flb_http_request *request) +{ + int version; + + version = flb_http_request_get_version(request); + + if (version == HTTP_PROTOCOL_VERSION_20) { + return flb_http2_request_commit(request); + } + + return flb_http1_request_commit(request); } char *flb_http_request_get_header(struct flb_http_request *request, @@ -168,12 +302,501 @@ int flb_http_request_unset_header(struct flb_http_request *request, return 0; } +int flb_http_request_compress_body( + struct flb_http_request *request, + char *content_encoding_header_value) +{ + char new_content_length[21]; + cfl_sds_t inflated_body; + char *output_buffer; + size_t output_size; + int result; + + result = 0; + + if (request->body == NULL) { + return 0; + } + + if (content_encoding_header_value == NULL) { + return 0; + } + + if (strncasecmp(content_encoding_header_value, "gzip", 4) == 0) { + result = compress_gzip(&output_buffer, + &output_size, + request->body, + cfl_sds_len(request->body)); + } + else if (strncasecmp(content_encoding_header_value, "zlib", 4) == 0) { + result = compress_zlib(&output_buffer, + &output_size, + request->body, + cfl_sds_len(request->body)); + } + else if (strncasecmp(content_encoding_header_value, "zstd", 4) == 0) { + result = compress_zstd(&output_buffer, + &output_size, + request->body, + cfl_sds_len(request->body)); + } + else if (strncasecmp(content_encoding_header_value, "snappy", 6) == 0) { + result = compress_snappy(&output_buffer, + &output_size, + request->body, + cfl_sds_len(request->body)); + } + else if (strncasecmp(content_encoding_header_value, "deflate", 4) == 0) { + result = compress_deflate(&output_buffer, + &output_size, + request->body, + cfl_sds_len(request->body)); + } + + if (result == 1) { + inflated_body = cfl_sds_create_len(output_buffer, output_size); + + flb_free(output_buffer); + + if (inflated_body == NULL) { + return -1; + } + + cfl_sds_destroy(request->body); + + request->body = inflated_body; + + snprintf(new_content_length, + sizeof(new_content_length), + "%zu", + output_size); + + flb_http_request_set_header(request, + "content-encoding", 0, + content_encoding_header_value, 0); + + flb_http_request_set_header(request, + "content-length", 0, + new_content_length, 0); + + request->content_length = output_size; + } + + return 0; +} + +int flb_http_request_uncompress_body( + struct flb_http_request *request) +{ + char *content_encoding_header_value; + char new_content_length[21]; + cfl_sds_t inflated_body; + char *output_buffer; + size_t output_size; + int result; + + result = 0; + + if (request->body == NULL) { + return 0; + } + + content_encoding_header_value = flb_http_request_get_header( + request, + "content-encoding"); + + if (content_encoding_header_value == NULL) { + return 0; + } + + if (strncasecmp(content_encoding_header_value, "gzip", 4) == 0) { + result = uncompress_gzip(&output_buffer, + &output_size, + request->body, + cfl_sds_len(request->body)); + } + else if (strncasecmp(content_encoding_header_value, "zlib", 4) == 0) { + result = uncompress_zlib(&output_buffer, + &output_size, + request->body, + cfl_sds_len(request->body)); + } + else if (strncasecmp(content_encoding_header_value, "zstd", 4) == 0) { + result = uncompress_zstd(&output_buffer, + &output_size, + request->body, + cfl_sds_len(request->body)); + } + else if (strncasecmp(content_encoding_header_value, "snappy", 6) == 0) { + result = uncompress_snappy(&output_buffer, + &output_size, + request->body, + cfl_sds_len(request->body)); + } + else if (strncasecmp(content_encoding_header_value, "deflate", 4) == 0) { + result = uncompress_deflate(&output_buffer, + &output_size, + request->body, + cfl_sds_len(request->body)); + } + + if (result == 1) { + inflated_body = cfl_sds_create_len(output_buffer, output_size); + + flb_free(output_buffer); + + if (inflated_body == NULL) { + return -1; + } + + cfl_sds_destroy(request->body); + + request->body = inflated_body; + + snprintf(new_content_length, + sizeof(new_content_length), + "%zu", + output_size); + + flb_http_request_unset_header(request, "content-encoding"); + flb_http_request_set_header(request, + "content-length", 0, + new_content_length, 0); + + request->content_length = output_size; + } + + return 0; +} + + +int flb_http_request_set_method(struct flb_http_request *request, + int method) +{ + request->method = method; + + return 0; +} + +int flb_http_request_set_host(struct flb_http_request *request, + char *host) +{ + request->host = cfl_sds_create(host); + + if (request->host == NULL) { + return -1; + } + + return 0; +} + +int flb_http_request_set_port(struct flb_http_request *request, + uint16_t port) +{ + request->port = port; + + return 0; +} + +int flb_http_request_set_url(struct flb_http_request *request, + char *url) +{ + char *start_of_authorization; + char *start_of_query_string; + char *start_of_authority; + char *start_of_username; + char *start_of_password; + char *start_of_port; + char *start_of_host; + char *start_of_path; + flb_sds_t local_url; + int result; + uint16_t port; + + local_url = cfl_sds_create(url); + + if (local_url == NULL) { + return -1; + } + + start_of_authority = strstr(local_url, "://"); + + if (start_of_authority == NULL) { + cfl_sds_destroy(local_url); + + return -1; + } + + start_of_authority = &start_of_authority[3]; + + start_of_path = strstr(start_of_authority, "/"); + + if (start_of_path == NULL) { + cfl_sds_destroy(local_url); + + return -1; + } + + start_of_query_string = strstr(start_of_path, "?"); + + if (start_of_query_string != NULL) { + result = flb_http_request_set_query_string(request, &start_of_query_string[1]); + + if (result != 0) { + cfl_sds_destroy(local_url); + + return -1; + } + + start_of_query_string[0] = '\0'; + } + + if (start_of_path != NULL) { + result = flb_http_request_set_uri(request, start_of_path); + + if (result != 0) { + cfl_sds_destroy(local_url); + + return -1; + } + + start_of_path[0] = '\0'; + } + + start_of_host = strstr(start_of_authority, "@"); + + if (start_of_host == NULL) { + start_of_host = start_of_authority; + + start_of_authorization = NULL; + } + else { + start_of_authorization = start_of_authority; + } + + if (start_of_host[0] == '@') { + start_of_host[0] = '\0'; + } + + if (start_of_authorization != NULL) { + start_of_password = strstr(start_of_authorization, ":"); + + if (start_of_password != NULL) { + start_of_password[0] = '\0'; + + start_of_password = &start_of_password[1]; + } + + start_of_username = start_of_authorization; + } + + start_of_port = strstr(start_of_host, ":"); + + if (start_of_port != NULL) { + start_of_port[0] = '\0'; + + start_of_port = &start_of_port[1]; + + port = (uint16_t) strtoul(start_of_port, NULL, 10); + + result = flb_http_request_set_port( + request, + port); + + if (result != 0) { + cfl_sds_destroy(local_url); + + return -1; + } + } + + if (start_of_username != NULL && + start_of_password != NULL) { + result = flb_http_request_set_authorization( + request, + HTTP_WWW_AUTHORIZATION_SCHEME_BASIC, + start_of_username, + start_of_password); + + if (result != 0) { + cfl_sds_destroy(local_url); + + return -1; + } + } + + if (start_of_host != NULL) { + result = flb_http_request_set_host( + request, + start_of_host); + + if (result != 0) { + cfl_sds_destroy(local_url); + + return -1; + } + } + + cfl_sds_destroy(local_url); + + return 0; +} + +int flb_http_request_set_uri(struct flb_http_request *request, + char *uri) +{ + request->path = cfl_sds_create(uri); + + if (request->path == NULL) { + return -1; + } + + return 0; +} + +int flb_http_request_set_query_string(struct flb_http_request *request, + char *query_string) +{ + request->query_string = cfl_sds_create(query_string); + + if (request->query_string == NULL) { + return -1; + } + + return 0; +} + +int flb_http_request_set_content_type(struct flb_http_request *request, + char *content_type) +{ + request->content_type = cfl_sds_create(content_type); + + if (request->content_type == NULL) { + return -1; + } + + return 0; +} + +int flb_http_request_set_user_agent(struct flb_http_request *request, + char *user_agent) +{ + request->user_agent = cfl_sds_create(user_agent); + + if (request->user_agent == NULL) { + return -1; + } + + return 0; +} + +int flb_http_request_set_content_length(struct flb_http_request *request, + size_t content_length) +{ + request->content_length = content_length; + + return 0; +} + +int flb_http_request_set_content_encoding(struct flb_http_request *request, + char *encoding) +{ + return flb_http_request_set_header(request, + "content-encoding", 0, + encoding, 0); + +} + +int flb_http_request_set_body(struct flb_http_request *request, + unsigned char *body, size_t body_length, + char *compression_algorithm) +{ + int compress; + uint64_t flags; + + if (request->stream->role == HTTP_STREAM_ROLE_SERVER) { + flags = ((struct flb_http_server_session *) request->stream->parent)->parent->flags; + + compress = flags & FLB_HTTP_SERVER_FLAG_AUTO_DEFLATE; + } + else { + flags = ((struct flb_http_client_session *) request->stream->parent)->parent->flags; + + compress = flags & FLB_HTTP_CLIENT_FLAG_AUTO_DEFLATE; + } + + request->body = cfl_sds_create_len((const char *) body, body_length); + + if (request->body == NULL) { + return -1; + } + + if (compress != 0 && compression_algorithm != NULL) { + return flb_http_request_compress_body(request, compression_algorithm); + } + else { + flb_http_request_set_content_length(request, body_length); + } + + return 0; +} + +int flb_http_request_perform_signv4_signature( + struct flb_http_request *request, + const char *aws_region, + const char *aws_service, + struct flb_aws_provider *aws_provider) +{ + flb_sds_t signature; + +#ifdef FLB_HAVE_SIGNV4 +#ifdef FLB_HAVE_AWS + flb_debug("signing request with AWS Sigv4"); + + signature = flb_signv4_ng_do(request, + FLB_TRUE, /* normalize URI ? */ + FLB_TRUE, /* add x-amz-date header ? */ + time(NULL), + (char *) aws_region, + (char *) aws_service, + 0, NULL, + aws_provider); + + if (signature == NULL) { + flb_error("could not sign request with sigv4"); + + return -1; + } + + flb_sds_destroy(signature); +#endif +#endif + + return 0; +} + /* HTTP RESPONSE */ +static int flb_http_response_get_version(struct flb_http_response *response) +{ + int version; + + if (response->stream->role == HTTP_STREAM_ROLE_SERVER) { + version = ((struct flb_http_server_session *) response->stream->parent)->version; + } + else { + version = ((struct flb_http_client_session *) response->stream->parent)->protocol_version; + } + + return version; +} + int flb_http_response_init(struct flb_http_response *response) { flb_http_response_destroy(response); + cfl_list_entry_init(&response->_head); + response->headers = flb_hash_table_create(FLB_HASH_TABLE_EVICT_NONE, 16, -1); if (response->headers == NULL) { @@ -191,6 +814,30 @@ int flb_http_response_init(struct flb_http_response *response) return 0; } +struct flb_http_response *flb_http_response_create() +{ + struct flb_http_response *response; + int result; + + response = flb_calloc(1, sizeof(struct flb_http_response)); + + if (response == NULL) { + return NULL; + } + + response->releasable = FLB_TRUE; + + result = flb_http_response_init(response); + + if (result != 0) { + flb_http_response_destroy(response); + + return NULL; + } + + return response; +} + void flb_http_response_destroy(struct flb_http_response *response) { if (response->message != NULL) { @@ -201,6 +848,10 @@ void flb_http_response_destroy(struct flb_http_response *response) cfl_sds_destroy(response->body); } + if (response->content_type != NULL) { + cfl_sds_destroy(response->content_type); + } + if (response->headers != NULL) { flb_hash_table_destroy(response->headers); } @@ -209,6 +860,10 @@ void flb_http_response_destroy(struct flb_http_response *response) flb_hash_table_destroy(response->trailer_headers); } + if (!cfl_list_entry_is_orphan(&response->_head)) { + cfl_list_del(&response->_head); + } + memset(response, 0, sizeof(struct flb_http_response)); } @@ -216,7 +871,7 @@ struct flb_http_response *flb_http_response_begin( struct flb_http_server_session *session, void *stream) { - if (session->version == HTTP_PROTOCOL_HTTP2) { + if (session->version == HTTP_PROTOCOL_VERSION_20) { return flb_http2_response_begin(&session->http2, stream); } else { @@ -228,9 +883,9 @@ int flb_http_response_commit(struct flb_http_response *response) { int len; char tmp[64]; - struct flb_http_server_session *session; + int version; - session = (struct flb_http_server_session *) response->stream->parent; + version = flb_http_response_get_version(response); if (response->body == NULL) { flb_http_response_set_header(response, @@ -241,7 +896,7 @@ int flb_http_response_commit(struct flb_http_response *response) } else { /* if the session is HTTP/1.x, always set the content-length header */ - if (session->version < HTTP_PROTOCOL_HTTP2) { + if (version < HTTP_PROTOCOL_VERSION_20) { len = snprintf(tmp, sizeof(tmp) - 1, "%zu", cfl_sds_len(response->body)); flb_http_response_set_header(response, "content-length", @@ -250,18 +905,56 @@ int flb_http_response_commit(struct flb_http_response *response) } } - if (session->version == HTTP_PROTOCOL_HTTP2) { + if (version == HTTP_PROTOCOL_VERSION_20) { return flb_http2_response_commit(response); } return flb_http1_response_commit(response); } +char *flb_http_response_get_header(struct flb_http_response *response, + char *name) +{ + char *lowercase_name; + size_t value_length; + int result; + void *value; + + lowercase_name = flb_http_server_convert_string_to_lowercase( + name, strlen(name)); + + if (lowercase_name == NULL) { + return NULL; + } + + result = flb_hash_table_get(response->headers, + lowercase_name, + strlen(lowercase_name), + &value, &value_length); + + flb_free(lowercase_name); + + if (result == -1) { + return NULL; + } + + return (char *) value; +} + int flb_http_response_set_header(struct flb_http_response *response, char *name, size_t name_length, char *value, size_t value_length) { - struct flb_http_server_session *session; + char *lowercase_name; + int version; + int result; + + lowercase_name = flb_http_server_convert_string_to_lowercase( + name, name_length); + + if (lowercase_name == NULL) { + return -1; + } if (name_length == 0) { name_length = strlen(name); @@ -276,18 +969,47 @@ int flb_http_response_set_header(struct flb_http_response *response, } } - session = (struct flb_http_server_session *) response->stream->parent; + version = flb_http_response_get_version(response); - if (session->version == HTTP_PROTOCOL_HTTP2) { - return flb_http2_response_set_header(response, - name, name_length, - value, value_length); + if (version == HTTP_PROTOCOL_VERSION_20) { + result = flb_http2_response_set_header(response, + lowercase_name, name_length, + value, value_length); } else { - return flb_http1_response_set_header(response, - name, name_length, - value, value_length); + result = flb_http1_response_set_header(response, + lowercase_name, name_length, + value, value_length); + } + + flb_free(lowercase_name); + + return result; +} + +int flb_http_response_unset_header(struct flb_http_response *response, + char *name) +{ + char *lowercase_name; + int result; + + lowercase_name = flb_http_server_convert_string_to_lowercase( + name, strlen(name)); + + if (lowercase_name == NULL) { + return -1; + } + + result = flb_hash_table_del(response->headers, + (const char *) lowercase_name); + + flb_free(lowercase_name); + + if (result == -1) { + return -1; } + + return 0; } int flb_http_response_set_trailer_header(struct flb_http_response *response, @@ -335,13 +1057,13 @@ int flb_http_response_set_trailer_header(struct flb_http_response *response, int flb_http_response_set_status(struct flb_http_response *response, int status) { - struct flb_http_server_session *session; + int version; - session = (struct flb_http_server_session *) response->stream->parent; + version = flb_http_response_get_version(response); response->status = status; - if (session->version == HTTP_PROTOCOL_HTTP2) { + if (version == HTTP_PROTOCOL_VERSION_20) { return flb_http2_response_set_status(response, status); } @@ -367,21 +1089,213 @@ int flb_http_response_set_message(struct flb_http_response *response, } int flb_http_response_set_body(struct flb_http_response *response, - unsigned char *body, size_t body_length) + unsigned char *body, size_t body_length) { - struct flb_http_server_session *session; - - session = (struct flb_http_server_session *) response->stream->parent; + if (response->body != NULL) { + cfl_sds_destroy(response->body); + } response->body = cfl_sds_create_len((const char *) body, body_length); - if (session->version == HTTP_PROTOCOL_HTTP2) { - return flb_http2_response_set_body(response, body, body_length); + if (response->body == NULL) { + return -1; + } + + return 0; +} + +int flb_http_response_append_to_body(struct flb_http_response *response, + unsigned char *body, size_t body_length) +{ + cfl_sds_t resized_buffer; + + if (response->body == NULL) { + return flb_http_response_set_body(response, body, body_length); + } + else { + resized_buffer = cfl_sds_cat(response->body, + (const char *) body, + body_length); + + if (resized_buffer == NULL) { + return -1; + } + + response->body = resized_buffer; } - return flb_http1_response_set_body(response, body, body_length); + return 0; } + +int flb_http_response_compress_body( + struct flb_http_response *response, + char *content_encoding_header_value) +{ + char new_content_length[21]; + cfl_sds_t inflated_body; + char *output_buffer; + size_t output_size; + int result; + + result = 0; + + if (response->body == NULL) { + return 0; + } + + if (content_encoding_header_value == NULL) { + return 0; + } + + if (strncasecmp(content_encoding_header_value, "gzip", 4) == 0) { + result = compress_gzip(&output_buffer, + &output_size, + response->body, + cfl_sds_len(response->body)); + } + else if (strncasecmp(content_encoding_header_value, "zlib", 4) == 0) { + result = compress_zlib(&output_buffer, + &output_size, + response->body, + cfl_sds_len(response->body)); + } + else if (strncasecmp(content_encoding_header_value, "zstd", 4) == 0) { + result = compress_zstd(&output_buffer, + &output_size, + response->body, + cfl_sds_len(response->body)); + } + else if (strncasecmp(content_encoding_header_value, "snappy", 6) == 0) { + result = compress_snappy(&output_buffer, + &output_size, + response->body, + cfl_sds_len(response->body)); + } + else if (strncasecmp(content_encoding_header_value, "deflate", 4) == 0) { + result = compress_deflate(&output_buffer, + &output_size, + response->body, + cfl_sds_len(response->body)); + } + + if (result == 1) { + inflated_body = cfl_sds_create_len(output_buffer, output_size); + + flb_free(output_buffer); + + if (inflated_body == NULL) { + return -1; + } + + cfl_sds_destroy(response->body); + + response->body = inflated_body; + + snprintf(new_content_length, + sizeof(new_content_length), + "%zu", + output_size); + + flb_http_response_set_header(response, + "content-encoding", 0, + content_encoding_header_value, 0); + + flb_http_response_set_header(response, + "content-length", 0, + new_content_length, 0); + + response->content_length = output_size; + } + + return 0; +} + +int flb_http_response_uncompress_body( + struct flb_http_response *response) +{ + char *content_encoding_header_value; + char new_content_length[21]; + cfl_sds_t inflated_body; + char *output_buffer; + size_t output_size; + int result; + + result = 0; + + if (response->body == NULL) { + return 0; + } + + content_encoding_header_value = flb_http_response_get_header( + response, + "content-encoding"); + + if (content_encoding_header_value == NULL) { + return 0; + } + + if (strncasecmp(content_encoding_header_value, "gzip", 4) == 0) { + result = uncompress_gzip(&output_buffer, + &output_size, + response->body, + cfl_sds_len(response->body)); + } + else if (strncasecmp(content_encoding_header_value, "zlib", 4) == 0) { + result = uncompress_zlib(&output_buffer, + &output_size, + response->body, + cfl_sds_len(response->body)); + } + else if (strncasecmp(content_encoding_header_value, "zstd", 4) == 0) { + result = uncompress_zstd(&output_buffer, + &output_size, + response->body, + cfl_sds_len(response->body)); + } + else if (strncasecmp(content_encoding_header_value, "snappy", 6) == 0) { + result = uncompress_snappy(&output_buffer, + &output_size, + response->body, + cfl_sds_len(response->body)); + } + else if (strncasecmp(content_encoding_header_value, "deflate", 4) == 0) { + result = uncompress_deflate(&output_buffer, + &output_size, + response->body, + cfl_sds_len(response->body)); + } + + if (result == 1) { + inflated_body = cfl_sds_create_len(output_buffer, output_size); + + flb_free(output_buffer); + + if (inflated_body == NULL) { + return -1; + } + + cfl_sds_destroy(response->body); + + response->body = inflated_body; + + snprintf(new_content_length, + sizeof(new_content_length), + "%zu", + output_size); + + flb_http_response_unset_header(response, "content-encoding"); + flb_http_response_set_header(response, + "content-length", 0, + new_content_length, 0); + + response->content_length = output_size; + } + + return 0; +} + + /* HTTP STREAM */ int flb_http_stream_init(struct flb_http_stream *stream, @@ -393,7 +1307,13 @@ int flb_http_stream_init(struct flb_http_stream *stream, int result; stream->id = stream_id; - stream->status = HTTP_STREAM_STATUS_RECEIVING_HEADERS; + + if (role == HTTP_STREAM_ROLE_SERVER) { + stream->status = HTTP_STREAM_STATUS_RECEIVING_HEADERS; + } + else { + stream->status = HTTP_STREAM_STATUS_SENDING_HEADERS; + } result = flb_http_request_init(&stream->request); @@ -418,9 +1338,9 @@ int flb_http_stream_init(struct flb_http_stream *stream, } struct flb_http_stream *flb_http_stream_create(void *parent, - int32_t stream_id, - int role, - void *user_data) + int32_t stream_id, + int role, + void *user_data) { struct flb_http_stream *stream; int result; @@ -457,3 +1377,207 @@ void flb_http_stream_destroy(struct flb_http_stream *stream) } } } + + +const char *flb_http_get_method_string_from_id(int method) +{ + switch (method) { + case HTTP_METHOD_GET: + return "GET"; + case HTTP_METHOD_POST: + return "POST"; + case HTTP_METHOD_HEAD: + return "HEAD"; + case HTTP_METHOD_PUT: + return "PUT"; + case HTTP_METHOD_DELETE: + return "DELETE"; + case HTTP_METHOD_OPTIONS: + return "OPTIONS"; + case HTTP_METHOD_CONNECT: + return "CONNECT"; + } + + return NULL; +} + +char *flb_http_server_convert_string_to_lowercase(char *input_buffer, + size_t length) +{ + char *output_buffer; + size_t index; + + output_buffer = flb_calloc(1, length + 1); + + if (output_buffer != NULL) { + for (index = 0 ; index < length ; index++) { + output_buffer[index] = tolower(input_buffer[index]); + } + + } + + return output_buffer; +} + + +int flb_http_server_strncasecmp(const uint8_t *first_buffer, + size_t first_length, + const char *second_buffer, + size_t second_length) +{ + const char *first_buffer_; + const char *second_buffer_; + + first_buffer_ = (const char *) first_buffer; + second_buffer_ = (const char *) second_buffer; + + if (first_length == 0) { + first_length = strlen(first_buffer_); + } + + if (second_length == 0) { + second_length = strlen(second_buffer_); + } + + if (first_length < second_length) { + return -1; + } + else if (first_length > second_length) { + return 1; + } + + return strncasecmp(first_buffer_, second_buffer_, first_length); +} + +/* PRIVATE */ + +static \ +int uncompress_zlib(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size) +{ + return 0; +} + +static \ +int uncompress_zstd(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size) +{ + return 0; +} + +static \ +int uncompress_deflate(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size) +{ + return 0; +} + +static \ +int uncompress_snappy(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size) +{ + int ret; + + ret = flb_snappy_uncompress_framed_data(input_buffer, + input_size, + output_buffer, + output_size); + + if (ret != 0) { + flb_error("[opentelemetry] snappy decompression failed"); + + return -1; + } + + return 1; +} + +static \ +int uncompress_gzip(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size) +{ + int ret; + + ret = flb_gzip_uncompress(input_buffer, + input_size, + (void **) output_buffer, + output_size); + + if (ret == -1) { + flb_error("[opentelemetry] gzip decompression failed"); + + return -1; + } + + return 1; +} + + +static \ +int compress_zlib(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size) +{ + return 0; +} + +static \ +int compress_zstd(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size) +{ + return 0; +} + +static \ +int compress_deflate(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size) +{ + return 0; +} + +static \ +int compress_snappy(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size) +{ + return 0; +} + +static \ +int compress_gzip(char **output_buffer, + size_t *output_size, + char *input_buffer, + size_t input_size) +{ + int ret; + + ret = flb_gzip_compress((void *) input_buffer, + input_size, + (void **) output_buffer, + output_size); + + if (ret == -1) { + flb_error("http client gzip compression failed"); + + return -1; + } + + return 1; +} + From 99d1b34737d565f6582c3026f4b7e0cacb0099df Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 29 Oct 2024 23:07:40 +0100 Subject: [PATCH 07/21] http_server: deduplicated code and updated constant names Signed-off-by: Leonardo Alminana --- .../fluent-bit/http_server/flb_http_server.h | 15 +- src/http_server/flb_http_server.c | 296 ++---------------- 2 files changed, 28 insertions(+), 283 deletions(-) diff --git a/include/fluent-bit/http_server/flb_http_server.h b/include/fluent-bit/http_server/flb_http_server.h index d4fb5d17e29..df9e8264c98 100755 --- a/include/fluent-bit/http_server/flb_http_server.h +++ b/include/fluent-bit/http_server/flb_http_server.h @@ -40,7 +40,8 @@ #define HTTP_SERVER_MAXIMUM_BUFFER_SIZE (10 * (1000 * 1024)) #define FLB_HTTP_SERVER_FLAG_KEEPALIVE (((uint64_t) 1) << 0) -#define FLB_HTTP_SERVER_FLAG_AUTO_INFLATE (((uint64_t) 1) << 1) +#define FLB_HTTP_SERVER_FLAG_AUTO_DEFLATE (((uint64_t) 1) << 1) +#define FLB_HTTP_SERVER_FLAG_AUTO_INFLATE (((uint64_t) 1) << 2) #define HTTP_SERVER_SUCCESS 0 #define HTTP_SERVER_PROVIDER_ERROR -1 @@ -96,17 +97,11 @@ struct flb_http_server_session { /* COMMON */ -char *flb_http_server_convert_string_to_lowercase(char *input_buffer, - size_t length); -int flb_http_server_strncasecmp(const uint8_t *first_buffer, - size_t first_length, - const char *second_buffer, - size_t second_length); /* HTTP SERVER */ -int flb_http_server_init(struct flb_http_server *session, +int flb_http_server_init(struct flb_http_server *session, int protocol_version, uint64_t flags, flb_http_server_request_processor_callback @@ -134,8 +129,8 @@ struct flb_http_server_session *flb_http_server_session_create(int version); void flb_http_server_session_destroy(struct flb_http_server_session *session); -int flb_http_server_session_ingest(struct flb_http_server_session *session, - unsigned char *buffer, +int flb_http_server_session_ingest(struct flb_http_server_session *session, + unsigned char *buffer, size_t length); #endif diff --git a/src/http_server/flb_http_server.c b/src/http_server/flb_http_server.c index 47400e4db74..81f107f8083 100644 --- a/src/http_server/flb_http_server.c +++ b/src/http_server/flb_http_server.c @@ -24,90 +24,6 @@ #include #include -static \ -int uncompress_zlib(char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size); - -static \ -int uncompress_zstd(char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size); - -static \ -int uncompress_deflate(char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size); - -static \ -int uncompress_snappy(char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size); - -static \ -int uncompress_gzip(char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size); - -/* COMMON */ - -char *flb_http_server_convert_string_to_lowercase(char *input_buffer, - size_t length) -{ - char *output_buffer; - size_t index; - - output_buffer = flb_calloc(1, length + 1); - - if (output_buffer != NULL) { - for (index = 0 ; index < length ; index++) { - output_buffer[index] = tolower(input_buffer[index]); - } - - } - - return output_buffer; -} - - -int flb_http_server_strncasecmp(const uint8_t *first_buffer, - size_t first_length, - const char *second_buffer, - size_t second_length) -{ - const char *first_buffer_; - const char *second_buffer_; - - first_buffer_ = (const char *) first_buffer; - second_buffer_ = (const char *) second_buffer; - - if (first_length == 0) { - first_length = strlen(first_buffer_); - } - - if (second_length == 0) { - second_length = strlen(second_buffer_); - } - - if (first_length < second_length) { - return -1; - } - else if (first_length > second_length) { - return 1; - } - - return strncasecmp(first_buffer_, second_buffer_, first_length); -} - - - - - /* PRIVATE */ static int flb_http_server_session_read(struct flb_http_server_session *session) @@ -176,109 +92,16 @@ static int flb_http_server_session_write(struct flb_http_server_session *session return 0; } - - -static int flb_http_server_inflate_request_body( - struct flb_http_request *request) -{ - char *content_encoding_header_value; - char new_content_length[21]; - struct flb_http_server_session *parent_session; - cfl_sds_t inflated_body; - char *output_buffer; - size_t output_size; - struct flb_http_server *server; - int result; - - parent_session = (struct flb_http_server_session *) request->stream->parent; - - server = parent_session->parent; - result = 0; - - if (request->body == NULL) { - return 0; - } - - if ((server->flags & FLB_HTTP_SERVER_FLAG_AUTO_INFLATE) == 0) { - return 0; - } - - content_encoding_header_value = flb_http_request_get_header( - request, - "content-encoding"); - - if (content_encoding_header_value == NULL) { - return 0; - } - - if (strncasecmp(content_encoding_header_value, "gzip", 4) == 0) { - result = uncompress_gzip(&output_buffer, - &output_size, - request->body, - cfl_sds_len(request->body)); - } - else if (strncasecmp(content_encoding_header_value, "zlib", 4) == 0) { - result = uncompress_zlib(&output_buffer, - &output_size, - request->body, - cfl_sds_len(request->body)); - } - else if (strncasecmp(content_encoding_header_value, "zstd", 4) == 0) { - result = uncompress_zstd(&output_buffer, - &output_size, - request->body, - cfl_sds_len(request->body)); - } - else if (strncasecmp(content_encoding_header_value, "snappy", 6) == 0) { - result = uncompress_snappy(&output_buffer, - &output_size, - request->body, - cfl_sds_len(request->body)); - } - else if (strncasecmp(content_encoding_header_value, "deflate", 4) == 0) { - result = uncompress_deflate(&output_buffer, - &output_size, - request->body, - cfl_sds_len(request->body)); - } - - if (result == 1) { - inflated_body = cfl_sds_create_len(output_buffer, output_size); - - flb_free(output_buffer); - - if (inflated_body == NULL) { - return -1; - } - - cfl_sds_destroy(request->body); - - request->body = inflated_body; - - snprintf(new_content_length, - sizeof(new_content_length), - "%zu", - output_size); - - flb_http_request_unset_header(request, "content-encoding"); - flb_http_request_set_header(request, - "content-length", strlen("content-length"), - new_content_length, strlen(new_content_length)); - - request->content_length = output_size; - } - - return 0; -} - static int flb_http_server_should_connection_be_closed( struct flb_http_request *request) { - int keepalive = FLB_FALSE; - char *connection_header_value; + char *connection_header_value; struct flb_http_server_session *parent_session; + struct flb_downstream *downstream; + int keepalive; struct flb_http_server *server; - struct flb_downstream *downstream; + + keepalive = FLB_FALSE; parent_session = (struct flb_http_server_session *) request->stream->parent; @@ -379,12 +202,14 @@ static int flb_http_server_client_activity_event_handler(void *data) request->content_length = cfl_sds_len(request->body); } - result = flb_http_server_inflate_request_body(request); + if ((server->flags & FLB_HTTP_SERVER_FLAG_AUTO_INFLATE) != 0) { + result = flb_http_request_uncompress_body(request); - if (result != 0) { - flb_http_server_session_destroy(session); + if (result != 0) { + flb_http_server_session_destroy(session); - return -1; + return -1; + } } if (server->request_callback != NULL) { @@ -619,14 +444,14 @@ int flb_http_server_session_init(struct flb_http_server_session *session, int ve session->version = version; - if (session->version == HTTP_PROTOCOL_HTTP2) { + if (session->version == HTTP_PROTOCOL_VERSION_20) { result = flb_http2_server_session_init(&session->http2, session); if (result != 0) { return -3; } } - else if (session->version == HTTP_PROTOCOL_HTTP1) { + else if (session->version <= HTTP_PROTOCOL_VERSION_11) { result = flb_http1_server_session_init(&session->http1, session); if (result != 0) { @@ -694,8 +519,8 @@ int flb_http_server_session_ingest(struct flb_http_server_session *session, cfl_sds_t resized_buffer; int result; - if (session->version == HTTP_PROTOCOL_AUTODETECT || - session->version == HTTP_PROTOCOL_HTTP1) { + if (session->version == HTTP_PROTOCOL_VERSION_AUTODETECT || + session->version <= HTTP_PROTOCOL_VERSION_11) { resized_buffer = cfl_sds_cat(session->incoming_data, (const char *) buffer, length); @@ -707,31 +532,31 @@ int flb_http_server_session_ingest(struct flb_http_server_session *session, session->incoming_data = resized_buffer; } - if (session->version == HTTP_PROTOCOL_AUTODETECT) { + if (session->version == HTTP_PROTOCOL_VERSION_AUTODETECT) { if (cfl_sds_len(session->incoming_data) >= 24) { if (strncmp(session->incoming_data, "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n", 24) == 0) { - session->version = HTTP_PROTOCOL_HTTP2; + session->version = HTTP_PROTOCOL_VERSION_20; } else { - session->version = HTTP_PROTOCOL_HTTP1; + session->version = HTTP_PROTOCOL_VERSION_11; } } else if (cfl_sds_len(session->incoming_data) >= 4) { if (strncmp(session->incoming_data, "PRI ", 4) != 0) { - session->version = HTTP_PROTOCOL_HTTP1; + session->version = HTTP_PROTOCOL_VERSION_11; } } - if (session->version == HTTP_PROTOCOL_HTTP1) { + if (session->version <= HTTP_PROTOCOL_VERSION_11) { result = flb_http1_server_session_init(&session->http1, session); if (result != 0) { return -1; } } - else if (session->version == HTTP_PROTOCOL_HTTP2) { + else if (session->version == HTTP_PROTOCOL_VERSION_20) { result = flb_http2_server_session_init(&session->http2, session); if (result != 0) { @@ -740,12 +565,12 @@ int flb_http_server_session_ingest(struct flb_http_server_session *session, } } - if (session->version == HTTP_PROTOCOL_HTTP1) { + if (session->version <= HTTP_PROTOCOL_VERSION_11) { return flb_http1_server_session_ingest(&session->http1, buffer, length); } - else if (session->version == HTTP_PROTOCOL_HTTP2) { + else if (session->version == HTTP_PROTOCOL_VERSION_20) { return flb_http2_server_session_ingest(&session->http2, buffer, length); @@ -753,78 +578,3 @@ int flb_http_server_session_ingest(struct flb_http_server_session *session, return -1; } - - -/* PRIVATE */ - -static \ -int uncompress_zlib(char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size) -{ - return 0; -} - -static \ -int uncompress_zstd(char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size) -{ - return 0; -} - -static \ -int uncompress_deflate(char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size) -{ - return 0; -} - -static \ -int uncompress_snappy(char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size) -{ - int ret; - - ret = flb_snappy_uncompress_framed_data(input_buffer, - input_size, - output_buffer, - output_size); - - if (ret != 0) { - flb_error("[opentelemetry] snappy decompression failed"); - - return -1; - } - - return 1; -} - -static \ -int uncompress_gzip(char **output_buffer, - size_t *output_size, - char *input_buffer, - size_t input_size) -{ - int ret; - - ret = flb_gzip_uncompress(input_buffer, - input_size, - (void *) output_buffer, - output_size); - - if (ret == -1) { - flb_error("[opentelemetry] gzip decompression failed"); - - return -1; - } - - return 1; -} - From d2e613a5b9c19bad314abe0a704c84b7a0b576e9 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 29 Oct 2024 23:07:57 +0100 Subject: [PATCH 08/21] build: added lock and http client related files Signed-off-by: Leonardo Alminana --- src/CMakeLists.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e24666484a7..10f778e5a77 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -64,6 +64,8 @@ set(src flb_snappy.c flb_compression.c flb_http_common.c + flb_http_client_http1.c + flb_http_client_http2.c flb_http_client.c flb_callback.c flb_strptime.c @@ -82,6 +84,7 @@ set(src flb_reload.c flb_msgpack_append_message.c flb_notification.c + flb_lock.c ) # Config format @@ -165,6 +168,7 @@ if(FLB_SIGNV4 AND FLB_TLS) set(src ${src} "flb_signv4.c" + "flb_signv4_ng.c" ) endif() From 7f95b0d14ff1fd9f4f48555f118eccf52e248768 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 29 Oct 2024 23:08:17 +0100 Subject: [PATCH 09/21] out_opentelemetry: added http/2 and grpc support Signed-off-by: Leonardo Alminana --- plugins/out_opentelemetry/opentelemetry.c | 236 ++++++++++++++++-- plugins/out_opentelemetry/opentelemetry.h | 21 +- .../out_opentelemetry/opentelemetry_conf.c | 49 +++- .../out_opentelemetry/opentelemetry_logs.c | 9 +- 4 files changed, 279 insertions(+), 36 deletions(-) diff --git a/plugins/out_opentelemetry/opentelemetry.c b/plugins/out_opentelemetry/opentelemetry.c index 8471c8102f6..b79d78adafd 100644 --- a/plugins/out_opentelemetry/opentelemetry.c +++ b/plugins/out_opentelemetry/opentelemetry.c @@ -44,15 +44,10 @@ extern void cmt_encode_opentelemetry_destroy(cfl_sds_t text); #include "opentelemetry_conf.h" #include "opentelemetry_utils.h" -int otel_process_logs(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, - struct flb_input_instance *ins, void *out_context, - struct flb_config *config); - -int opentelemetry_http_post(struct opentelemetry_context *ctx, - const void *body, size_t body_len, - const char *tag, int tag_len, - const char *uri) +int opentelemetry_legacy_post(struct opentelemetry_context *ctx, + const void *body, size_t body_len, + const char *tag, int tag_len, + const char *uri) { size_t final_body_len; void *final_body; @@ -223,6 +218,184 @@ int opentelemetry_http_post(struct opentelemetry_context *ctx, return out_ret; } +int opentelemetry_post(struct opentelemetry_context *ctx, + const void *body, size_t body_len, + const char *tag, int tag_len, + const char *http_uri, + const char *grpc_uri) +{ + const char *compression_algorithm; + uint32_t wire_message_length; + size_t grpc_body_length; + cfl_sds_t grpc_body; + struct flb_http_response *response; + struct flb_http_request *request; + int out_ret; + int result; + + if (!ctx->enable_http2_flag) { + return opentelemetry_legacy_post(ctx, + body, body_len, + tag, tag_len, + http_uri); + } + + compression_algorithm = NULL; + + request = flb_http_client_request_builder( + &ctx->http_client, + FLB_HTTP_CLIENT_ARGUMENT_METHOD(FLB_HTTP_POST), + FLB_HTTP_CLIENT_ARGUMENT_HOST(ctx->host), + FLB_HTTP_CLIENT_ARGUMENT_USER_AGENT("Fluent-Bit"), + FLB_HTTP_CLIENT_ARGUMENT_HEADERS( + FLB_HTTP_CLIENT_HEADER_CONFIG_MAP_LIST, + ctx->headers)); + + if (request == NULL) { + flb_plg_error(ctx->ins, "error initializing http request"); + + return FLB_RETRY; + } + + if (request->protocol_version == HTTP_PROTOCOL_VERSION_20) { + grpc_body = cfl_sds_create_size(body_len + 5); + + if (grpc_body == NULL) { + return FLB_RETRY; + } + + wire_message_length = (uint32_t) body_len; + + cfl_sds_cat(grpc_body, "\x00----", 5); + + ((uint8_t *) grpc_body)[1] = (wire_message_length & 0xFF000000) >> 24; + ((uint8_t *) grpc_body)[2] = (wire_message_length & 0x00FF0000) >> 16; + ((uint8_t *) grpc_body)[3] = (wire_message_length & 0x0000FF00) >> 8; + ((uint8_t *) grpc_body)[4] = (wire_message_length & 0x000000FF) >> 0; + + cfl_sds_cat(grpc_body, body, body_len); + + grpc_body_length = cfl_sds_len(grpc_body); + + result = flb_http_request_set_parameters(request, + FLB_HTTP_CLIENT_ARGUMENT_URI(grpc_uri), + FLB_HTTP_CLIENT_ARGUMENT_CONTENT_TYPE( + "application/grpc"), + FLB_HTTP_CLIENT_ARGUMENT_BODY(grpc_body, + grpc_body_length, + compression_algorithm)); + + cfl_sds_destroy(grpc_body); + + if (result != 0) { + flb_http_client_request_destroy(request, FLB_TRUE); + + return FLB_RETRY; + } + } + else { + if (ctx->compress_gzip == FLB_TRUE) { + compression_algorithm = "gzip"; + } + + result = flb_http_request_set_parameters(request, + FLB_HTTP_CLIENT_ARGUMENT_URI(http_uri), + FLB_HTTP_CLIENT_ARGUMENT_CONTENT_TYPE( + FLB_OPENTELEMETRY_MIME_PROTOBUF_LITERAL), + FLB_HTTP_CLIENT_ARGUMENT_BODY(body, + body_len, + compression_algorithm)); + + if (result != 0) { + flb_http_client_request_destroy(request, FLB_TRUE); + + return FLB_RETRY; + } + } + + if (ctx->http_user != NULL && + ctx->http_passwd != NULL) { + result = flb_http_request_set_parameters(request, + FLB_HTTP_CLIENT_ARGUMENT_BASIC_AUTHORIZATION( + ctx->http_user, + ctx->http_passwd)); + + if (result != 0) { + flb_plg_error(ctx->ins, "error setting http authorization data"); + + return FLB_RETRY; + } + + flb_http_request_set_authorization(request, + HTTP_WWW_AUTHORIZATION_SCHEME_BASIC, + ctx->http_user, + ctx->http_passwd); + } + + response = flb_http_client_request_execute(request); + + if (response == NULL) { + flb_debug("http request execution error"); + + flb_http_client_request_destroy(request, FLB_TRUE); + + return FLB_RETRY; + } + + /* + * Only allow the following HTTP status: + * + * - 200: OK + * - 201: Created + * - 202: Accepted + * - 203: no authorative resp + * - 204: No Content + * - 205: Reset content + * + */ + if (response->status < 200 || response->status > 205) { + if (ctx->log_response_payload && + response->body != NULL && + cfl_sds_len(response->body) > 0) { + flb_plg_error(ctx->ins, "%s:%i, HTTP status=%i\n%s", + ctx->host, ctx->port, + response->status, response->body); + } + else { + flb_plg_error(ctx->ins, "%s:%i, HTTP status=%i", + ctx->host, ctx->port, response->status); + } + + out_ret = FLB_RETRY; + } + else { + if (ctx->log_response_payload && + response->body != NULL && + cfl_sds_len(response->body) > 0) { + flb_plg_info(ctx->ins, "%s:%i, HTTP status=%i\n%s", + ctx->host, ctx->port, + response->status, response->body); + } + else { + flb_plg_info(ctx->ins, "%s:%i, HTTP status=%i", + ctx->host, ctx->port, + response->status); + } + + out_ret = FLB_OK; + } + + flb_http_client_request_destroy(request, FLB_TRUE); + + return out_ret; +} + +int otel_process_logs(struct flb_event_chunk *event_chunk, + struct flb_output_flush *out_flush, + struct flb_input_instance *ins, void *out_context, + struct flb_config *config); + + static void append_labels(struct opentelemetry_context *ctx, struct cmt *cmt) { @@ -258,7 +431,7 @@ static int opentelemetry_format_test(struct flb_config *config, } static int process_metrics(struct flb_event_chunk *event_chunk, - struct flb_output_flush *out_flush, + struct flb_output_flush *out1_flush, struct flb_input_instance *ins, void *out_context, struct flb_config *config) { @@ -312,7 +485,7 @@ static int process_metrics(struct flb_event_chunk *event_chunk, diff = off; /* concat buffer */ - flb_sds_cat_safe(&buf, encoded_chunk, cfl_sds_len(encoded_chunk)); + flb_sds_cat_safe(&buf, encoded_chunk, flb_sds_len(encoded_chunk)); /* release */ cmt_encode_opentelemetry_destroy(encoded_chunk); @@ -323,10 +496,11 @@ static int process_metrics(struct flb_event_chunk *event_chunk, flb_plg_debug(ctx->ins, "final payload size: %lu", flb_sds_len(buf)); if (buf && flb_sds_len(buf) > 0) { /* Send HTTP request */ - result = opentelemetry_http_post(ctx, buf, flb_sds_len(buf), - event_chunk->tag, - flb_sds_len(event_chunk->tag), - ctx->metrics_uri_sanitized); + result = opentelemetry_post(ctx, buf, flb_sds_len(buf), + event_chunk->tag, + flb_sds_len(event_chunk->tag), + ctx->metrics_uri_sanitized, + ctx->grpc_metrics_uri); /* Debug http_post() result statuses */ if (result == FLB_OK) { @@ -413,10 +587,11 @@ static int process_traces(struct flb_event_chunk *event_chunk, flb_plg_debug(ctx->ins, "final payload size: %lu", flb_sds_len(buf)); if (buf && flb_sds_len(buf) > 0) { /* Send HTTP request */ - result = opentelemetry_http_post(ctx, buf, flb_sds_len(buf), - event_chunk->tag, - flb_sds_len(event_chunk->tag), - ctx->traces_uri_sanitized); + result = opentelemetry_post(ctx, buf, flb_sds_len(buf), + event_chunk->tag, + flb_sds_len(event_chunk->tag), + ctx->traces_uri_sanitized, + ctx->grpc_traces_uri); /* Debug http_post() result statuses */ if (result == FLB_OK) { @@ -496,7 +671,11 @@ static struct flb_config_map config_map[] = { add_labels), "Adds a custom label to the metrics use format: 'add_label name value'" }, - + { + FLB_CONFIG_MAP_STR, "http2", "on", + 0, FLB_TRUE, offsetof(struct opentelemetry_context, enable_http2), + "Enable, disable or force HTTP/2 usage. Accepted values : on, off, force" + }, { FLB_CONFIG_MAP_STR, "proxy", NULL, 0, FLB_FALSE, 0, @@ -522,6 +701,11 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct opentelemetry_context, metrics_uri), "Specify an optional HTTP URI for the target OTel endpoint." }, + { + FLB_CONFIG_MAP_STR, "grpc_metrics_uri", "/opentelemetry.proto.collector.metrics.v1.MetricsService/Export", + 0, FLB_TRUE, offsetof(struct opentelemetry_context, grpc_metrics_uri), + "Specify an optional gRPC URI for the target OTel endpoint." + }, { FLB_CONFIG_MAP_INT, "batch_size", DEFAULT_LOG_RECORD_BATCH_SIZE, @@ -543,6 +727,12 @@ static struct flb_config_map config_map[] = { "Specify an optional HTTP URI for the target OTel endpoint." }, + { + FLB_CONFIG_MAP_STR, "grpc_logs_uri", "/opentelemetry.proto.collector.logs.v1.LogsService/Export", + 0, FLB_TRUE, offsetof(struct opentelemetry_context, grpc_logs_uri), + "Specify an optional gRPCß URI for the target OTel endpoint." + }, + { FLB_CONFIG_MAP_STR, "logs_body_key", NULL, FLB_CONFIG_MAP_MULT, FLB_TRUE, offsetof(struct opentelemetry_context, log_body_key_list_str), @@ -561,6 +751,12 @@ static struct flb_config_map config_map[] = { 0, FLB_TRUE, offsetof(struct opentelemetry_context, traces_uri), "Specify an optional HTTP URI for the target OTel endpoint." }, + { + FLB_CONFIG_MAP_STR, "grpc_traces_uri", "/opentelemetry.proto.collector.trace.v1.TraceService/Export", + 0, FLB_TRUE, offsetof(struct opentelemetry_context, grpc_traces_uri), + "Specify an optional gRPC URI for the target OTel endpoint." + }, + { FLB_CONFIG_MAP_BOOL, "log_response_payload", "true", 0, FLB_TRUE, offsetof(struct opentelemetry_context, log_response_payload), diff --git a/plugins/out_opentelemetry/opentelemetry.h b/plugins/out_opentelemetry/opentelemetry.h index a43d4184caf..cb25e6da055 100644 --- a/plugins/out_opentelemetry/opentelemetry.h +++ b/plugins/out_opentelemetry/opentelemetry.h @@ -23,6 +23,7 @@ #include #include #include +#include #define FLB_OPENTELEMETRY_CONTENT_TYPE_HEADER_NAME "Content-Type" #define FLB_OPENTELEMETRY_MIME_PROTOBUF_LITERAL "application/x-protobuf" @@ -43,6 +44,9 @@ struct opentelemetry_body_key { /* Plugin context */ struct opentelemetry_context { + int enable_http2_flag; + char *enable_http2; + /* HTTP Auth */ char *http_user; char *http_passwd; @@ -57,12 +61,17 @@ struct opentelemetry_context { char *metrics_uri_sanitized; char *logs_uri_sanitized; char *traces_uri; + char *grpc_traces_uri; char *metrics_uri; + char *grpc_metrics_uri; char *logs_uri; - + char *grpc_logs_uri; char *host; int port; + /* HTTP client */ + struct flb_http_client_ng http_client; + /* record metadata parsing */ flb_sds_t logs_metadata_key; @@ -140,7 +149,6 @@ struct opentelemetry_context { /* Arbitrary HTTP headers */ struct mk_list *headers; - /* instance context */ struct flb_output_instance *ins; @@ -169,8 +177,9 @@ struct opentelemetry_context { struct flb_record_accessor *ra_log_meta_otlp_trace_flags; }; -int opentelemetry_http_post(struct opentelemetry_context *ctx, - const void *body, size_t body_len, - const char *tag, int tag_len, - const char *uri); +int opentelemetry_post(struct opentelemetry_context *ctx, + const void *body, size_t body_len, + const char *tag, int tag_len, + const char *http_uri, + const char *grpc_uri); #endif diff --git a/plugins/out_opentelemetry/opentelemetry_conf.c b/plugins/out_opentelemetry/opentelemetry_conf.c index f6c8b40ba9c..97cb5f1c1af 100644 --- a/plugins/out_opentelemetry/opentelemetry_conf.c +++ b/plugins/out_opentelemetry/opentelemetry_conf.c @@ -184,10 +184,9 @@ static int config_add_labels(struct flb_output_instance *ins, */ static int check_proxy(struct flb_output_instance *ins, - struct opentelemetry_context *ctx, - char *host, char *port, - char *protocol, char *uri) -{ + struct opentelemetry_context *ctx, + char *host, char *port, + char *protocol, char *uri){ int ret; const char *tmp = NULL; @@ -215,8 +214,7 @@ static int check_proxy(struct flb_output_instance *ins, return 0; } -static char *sanitize_uri(char *uri) -{ +static char *sanitize_uri(char *uri){ char *new_uri; int uri_len; @@ -254,6 +252,8 @@ struct opentelemetry_context *flb_opentelemetry_context_create(struct flb_output struct flb_upstream *upstream; struct opentelemetry_context *ctx = NULL; const char *tmp = NULL; + uint64_t http_client_flags; + int http_protocol_version; /* Allocate plugin context */ ctx = flb_calloc(1, sizeof(struct opentelemetry_context)); @@ -540,6 +540,41 @@ struct opentelemetry_context *flb_opentelemetry_context_create(struct flb_output flb_plg_error(ins, "failed to create record accessor for otlp trace flags"); } + http_client_flags = FLB_HTTP_CLIENT_FLAG_AUTO_DEFLATE | + FLB_HTTP_CLIENT_FLAG_AUTO_INFLATE; + + if (ctx->u->base.net.keepalive) { + http_client_flags |= FLB_HTTP_CLIENT_FLAG_KEEPALIVE; + } + + ctx->enable_http2_flag = FLB_TRUE; + + if (strcasecmp(ctx->enable_http2, "force") == 0) { + http_protocol_version = HTTP_PROTOCOL_VERSION_20; + } + else if (flb_utils_bool(ctx->enable_http2)) { + http_protocol_version = HTTP_PROTOCOL_VERSION_AUTODETECT; + } + else { + http_protocol_version = HTTP_PROTOCOL_VERSION_11; + + ctx->enable_http2_flag = FLB_FALSE; + } + + ret = flb_http_client_ng_init(&ctx->http_client, + NULL, + ctx->u, + http_protocol_version, + http_client_flags); + + if (ret != 0) { + flb_plg_debug(ctx->ins, "http client creation error"); + + flb_opentelemetry_context_destroy(ctx); + + ctx = NULL; + } + return ctx; } @@ -549,6 +584,8 @@ void flb_opentelemetry_context_destroy(struct opentelemetry_context *ctx) return; } + flb_http_client_ng_destroy(&ctx->http_client); + flb_kv_release(&ctx->kv_labels); if (ctx->u) { diff --git a/plugins/out_opentelemetry/opentelemetry_logs.c b/plugins/out_opentelemetry/opentelemetry_logs.c index f02a21b46fd..b908cf7cb3e 100644 --- a/plugins/out_opentelemetry/opentelemetry_logs.c +++ b/plugins/out_opentelemetry/opentelemetry_logs.c @@ -770,10 +770,11 @@ static int logs_flush_to_otel(struct opentelemetry_context *ctx, struct flb_even opentelemetry__proto__collector__logs__v1__export_logs_service_request__pack(export_logs, body); /* send post request to opentelemetry with content type application/x-protobuf */ - ret = opentelemetry_http_post(ctx, body, len, - event_chunk->tag, - flb_sds_len(event_chunk->tag), - ctx->logs_uri_sanitized); + ret = opentelemetry_post(ctx, body, len, + event_chunk->tag, + flb_sds_len(event_chunk->tag), + ctx->logs_uri_sanitized, + ctx->grpc_logs_uri); flb_free(body); return ret; From fd0815cc22b09c2fedcffdf26d3b137d911a075b Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 29 Oct 2024 23:08:33 +0100 Subject: [PATCH 10/21] out_splunk: updated constant names Signed-off-by: Leonardo Alminana --- plugins/in_splunk/splunk.c | 4 ++-- plugins/in_splunk/splunk_prot.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/in_splunk/splunk.c b/plugins/in_splunk/splunk.c index b10dfa8f00e..f965b3c1ed1 100644 --- a/plugins/in_splunk/splunk.c +++ b/plugins/in_splunk/splunk.c @@ -95,8 +95,8 @@ static int in_splunk_init(struct flb_input_instance *ins, if (ctx->enable_http2) { - ret = flb_http_server_init(&ctx->http_server, - HTTP_PROTOCOL_AUTODETECT, + ret = flb_http_server_init(&ctx->http_server, + HTTP_PROTOCOL_VERSION_AUTODETECT, (FLB_HTTP_SERVER_FLAG_KEEPALIVE | FLB_HTTP_SERVER_FLAG_AUTO_INFLATE), NULL, ins->host.listen, diff --git a/plugins/in_splunk/splunk_prot.c b/plugins/in_splunk/splunk_prot.c index c312d2623b8..cfc21f2b60f 100644 --- a/plugins/in_splunk/splunk_prot.c +++ b/plugins/in_splunk/splunk_prot.c @@ -1156,7 +1156,7 @@ int splunk_prot_handle_ng(struct flb_http_request *request, } /* HTTP/1.1 needs Host header */ - if (request->protocol_version == HTTP_PROTOCOL_HTTP1 && + if (request->protocol_version == HTTP_PROTOCOL_VERSION_11 && request->host == NULL) { return -1; From b0774c9ec3ee98bd693a38ceb1ea25c9fe568d6e Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 29 Oct 2024 23:08:50 +0100 Subject: [PATCH 11/21] in_opensearch: updated constant names Signed-off-by: Leonardo Alminana --- plugins/in_elasticsearch/in_elasticsearch.c | 6 +++--- plugins/in_elasticsearch/in_elasticsearch_bulk_prot.c | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/plugins/in_elasticsearch/in_elasticsearch.c b/plugins/in_elasticsearch/in_elasticsearch.c index 72096d19a06..f77390932c8 100644 --- a/plugins/in_elasticsearch/in_elasticsearch.c +++ b/plugins/in_elasticsearch/in_elasticsearch.c @@ -137,8 +137,8 @@ static int in_elasticsearch_bulk_init(struct flb_input_instance *ins, bytes_to_nodename(rand, ctx->node_name, 12); if (ctx->enable_http2) { - ret = flb_http_server_init(&ctx->http_server, - HTTP_PROTOCOL_AUTODETECT, + ret = flb_http_server_init(&ctx->http_server, + HTTP_PROTOCOL_VERSION_AUTODETECT, (FLB_HTTP_SERVER_FLAG_KEEPALIVE | FLB_HTTP_SERVER_FLAG_AUTO_INFLATE), NULL, ins->host.listen, @@ -209,7 +209,7 @@ static int in_elasticsearch_bulk_init(struct flb_input_instance *ins, return -1; } - ctx->collector_id = ret; + ctx->collector_id = ret; } return 0; diff --git a/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.c b/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.c index a2424413e19..e5818c9ddca 100644 --- a/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.c +++ b/plugins/in_elasticsearch/in_elasticsearch_bulk_prot.c @@ -1090,7 +1090,6 @@ static int process_payload_ng(struct flb_http_request *request, return -1; } - printf("Processing payload 2 : %s\n", request->body); parse_payload_ndjson(context, tag, request->body, cfl_sds_len(request->body), bulk_statuses); return 0; @@ -1117,7 +1116,7 @@ int in_elasticsearch_bulk_prot_handle_ng(struct flb_http_request *request, } /* HTTP/1.1 needs Host header */ - if (request->protocol_version == HTTP_PROTOCOL_HTTP1 && + if (request->protocol_version == HTTP_PROTOCOL_VERSION_11 && request->host == NULL) { return -1; From bcd65acbc139d4642adf03819449ddd081e32d4d Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 29 Oct 2024 23:09:03 +0100 Subject: [PATCH 12/21] in_http: updated constant names Signed-off-by: Leonardo Alminana --- plugins/in_http/http.c | 4 ++-- plugins/in_http/http_prot.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/in_http/http.c b/plugins/in_http/http.c index 2aa777cc490..bba621475c7 100644 --- a/plugins/in_http/http.c +++ b/plugins/in_http/http.c @@ -94,8 +94,8 @@ static int in_http_init(struct flb_input_instance *ins, port = (unsigned short int) strtoul(ctx->tcp_port, NULL, 10); if (ctx->enable_http2) { - ret = flb_http_server_init(&ctx->http_server, - HTTP_PROTOCOL_AUTODETECT, + ret = flb_http_server_init(&ctx->http_server, + HTTP_PROTOCOL_VERSION_AUTODETECT, (FLB_HTTP_SERVER_FLAG_KEEPALIVE | FLB_HTTP_SERVER_FLAG_AUTO_INFLATE), NULL, ins->host.listen, diff --git a/plugins/in_http/http_prot.c b/plugins/in_http/http_prot.c index 7c3a96f6c9b..175fe4461e1 100644 --- a/plugins/in_http/http_prot.c +++ b/plugins/in_http/http_prot.c @@ -1016,7 +1016,7 @@ int http_prot_handle_ng(struct flb_http_request *request, /* ToDo: Fix me */ /* HTTP/1.1 needs Host header */ - if (request->protocol_version == HTTP_PROTOCOL_HTTP1 && + if (request->protocol_version == HTTP_PROTOCOL_VERSION_11 && request->host == NULL) { flb_sds_destroy(tag); From af8cf8046b7458f078af1772ed03437ebc30cf97 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 29 Oct 2024 23:09:13 +0100 Subject: [PATCH 13/21] in_opentelemetry: updated constant names Signed-off-by: Leonardo Alminana --- plugins/in_opentelemetry/opentelemetry.c | 2 +- plugins/in_opentelemetry/opentelemetry_prot.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/in_opentelemetry/opentelemetry.c b/plugins/in_opentelemetry/opentelemetry.c index bd03df008f4..5876be34363 100644 --- a/plugins/in_opentelemetry/opentelemetry.c +++ b/plugins/in_opentelemetry/opentelemetry.c @@ -92,7 +92,7 @@ static int in_opentelemetry_init(struct flb_input_instance *ins, if (ctx->enable_http2) { ret = flb_http_server_init(&ctx->http_server, - HTTP_PROTOCOL_AUTODETECT, + HTTP_PROTOCOL_VERSION_AUTODETECT, (FLB_HTTP_SERVER_FLAG_KEEPALIVE | FLB_HTTP_SERVER_FLAG_AUTO_INFLATE), NULL, ins->host.listen, diff --git a/plugins/in_opentelemetry/opentelemetry_prot.c b/plugins/in_opentelemetry/opentelemetry_prot.c index 3a2180387e7..d87e6b759af 100644 --- a/plugins/in_opentelemetry/opentelemetry_prot.c +++ b/plugins/in_opentelemetry/opentelemetry_prot.c @@ -2485,7 +2485,7 @@ int opentelemetry_prot_handle_ng(struct flb_http_request *request, /* ToDo: Fix me */ /* HTTP/1.1 needs Host header */ - if (request->protocol_version == HTTP_PROTOCOL_HTTP1 && + if (request->protocol_version == HTTP_PROTOCOL_VERSION_11 && request->host == NULL) { return -1; } From a283666ca62728fbedd7e47addced8025a5c7ab4 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Tue, 29 Oct 2024 23:09:26 +0100 Subject: [PATCH 14/21] in_prometheus_remote_write: updated constant names Signed-off-by: Leonardo Alminana --- plugins/in_prometheus_remote_write/prom_rw.c | 2 +- plugins/in_prometheus_remote_write/prom_rw_prot.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/in_prometheus_remote_write/prom_rw.c b/plugins/in_prometheus_remote_write/prom_rw.c index e2cb852815c..c6dcda45564 100644 --- a/plugins/in_prometheus_remote_write/prom_rw.c +++ b/plugins/in_prometheus_remote_write/prom_rw.c @@ -92,7 +92,7 @@ static int prom_rw_init(struct flb_input_instance *ins, if (ctx->enable_http2) { ret = flb_http_server_init(&ctx->http_server, - HTTP_PROTOCOL_AUTODETECT, + HTTP_PROTOCOL_VERSION_AUTODETECT, (FLB_HTTP_SERVER_FLAG_KEEPALIVE | FLB_HTTP_SERVER_FLAG_AUTO_INFLATE), NULL, ins->host.listen, diff --git a/plugins/in_prometheus_remote_write/prom_rw_prot.c b/plugins/in_prometheus_remote_write/prom_rw_prot.c index d041c8f9a6d..6d7e4f6fa8e 100644 --- a/plugins/in_prometheus_remote_write/prom_rw_prot.c +++ b/plugins/in_prometheus_remote_write/prom_rw_prot.c @@ -464,7 +464,7 @@ int prom_rw_prot_handle_ng(struct flb_http_request *request, /* ToDo: Fix me */ /* HTTP/1.1 needs Host header */ - if (request->protocol_version == HTTP_PROTOCOL_HTTP1 && + if (request->protocol_version >= HTTP_PROTOCOL_VERSION_11 && request->host == NULL) { return -1; From e06f40128176ca4feb070b55a27b48c23a8c5749 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Wed, 30 Oct 2024 14:03:36 +0100 Subject: [PATCH 15/21] tests: runtime: in_http : added leftover directory handling Signed-off-by: Leonardo Alminana --- tests/runtime/in_http.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/runtime/in_http.c b/tests/runtime/in_http.c index 609bc9dadaf..d4d88faadf8 100644 --- a/tests/runtime/in_http.c +++ b/tests/runtime/in_http.c @@ -429,13 +429,13 @@ void flb_test_http_json_charset_header(char *response_code) void flb_test_http_successful_response_code_200() { - flb_test_http_successful_response_code("200"); + flb_test_http_successful_response_code("200"); flb_test_http_json_charset_header("200"); } void flb_test_http_successful_response_code_204() { - flb_test_http_successful_response_code("204"); + flb_test_http_successful_response_code("204"); flb_test_http_json_charset_header("204"); } @@ -547,9 +547,11 @@ void flb_test_http_failure_400_bad_disk_write() flb_time_msleep(5000); - rename("/tmp/http-input-test-404-bad-write", + rmdir("/tmp/http-input-test-404-bad-write.fail/http.0"); + rmdir("/tmp/http-input-test-404-bad-write.fail"); + + rename("/tmp/http-input-test-404-bad-write", "/tmp/http-input-test-404-bad-write.fail"); - TEST_CHECK(ret == 0); ctx->httpc = http_client_ctx_create(); TEST_CHECK(ctx->httpc != NULL); @@ -575,9 +577,8 @@ void flb_test_http_failure_400_bad_disk_write() TEST_MSG("http response code error. expect: %d, got: %d\n", 400, c->resp.status); } - rename("/tmp/http-input-test-404-bad-write.fail", + rename("/tmp/http-input-test-404-bad-write.fail", "/tmp/http-input-test-404-bad-write"); - TEST_CHECK(ret == 0); /* waiting to flush */ flb_time_msleep(1500); From 44db6293a2e02b9607413c681bb21b8a45b2c61a Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Wed, 30 Oct 2024 14:04:07 +0100 Subject: [PATCH 16/21] http_common: fixed operation order when converting names to lowercase Signed-off-by: Leonardo Alminana --- src/flb_http_common.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/flb_http_common.c b/src/flb_http_common.c index b39391147a3..fdfdd88041c 100644 --- a/src/flb_http_common.c +++ b/src/flb_http_common.c @@ -242,6 +242,10 @@ int flb_http_request_set_header(struct flb_http_request *request, char *lowercase_name; int result; + if (name_length == 0) { + name_length = strlen(name); + } + lowercase_name = flb_http_server_convert_string_to_lowercase( name, name_length); @@ -249,10 +253,6 @@ int flb_http_request_set_header(struct flb_http_request *request, return -1; } - if (name_length == 0) { - name_length = strlen(name); - } - if (value_length == 0) { if (value[0] == '\0') { value_length = 1; @@ -949,6 +949,10 @@ int flb_http_response_set_header(struct flb_http_response *response, int version; int result; + if (name_length == 0) { + name_length = strlen(name); + } + lowercase_name = flb_http_server_convert_string_to_lowercase( name, name_length); @@ -956,10 +960,6 @@ int flb_http_response_set_header(struct flb_http_response *response, return -1; } - if (name_length == 0) { - name_length = strlen(name); - } - if (value_length == 0) { if (value[0] == '\0') { value_length = 1; From 259a7be288f27d76d96be2e3e4ac46cebebfb66f Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Wed, 30 Oct 2024 14:04:29 +0100 Subject: [PATCH 17/21] http_server: fixed version detection when initializing sessions Signed-off-by: Leonardo Alminana --- src/http_server/flb_http_server.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/http_server/flb_http_server.c b/src/http_server/flb_http_server.c index 81f107f8083..5080eefbede 100644 --- a/src/http_server/flb_http_server.c +++ b/src/http_server/flb_http_server.c @@ -451,7 +451,8 @@ int flb_http_server_session_init(struct flb_http_server_session *session, int ve return -3; } } - else if (session->version <= HTTP_PROTOCOL_VERSION_11) { + else if (session->version > HTTP_PROTOCOL_VERSION_AUTODETECT && + session->version <= HTTP_PROTOCOL_VERSION_11) { result = flb_http1_server_session_init(&session->http1, session); if (result != 0) { From 25c3ec17506c5662c29ae337f8a911c110aa7b5d Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Wed, 30 Oct 2024 14:46:15 +0100 Subject: [PATCH 18/21] lock: added missing header Signed-off-by: Leonardo Alminana --- src/flb_lock.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/flb_lock.c b/src/flb_lock.c index 212bb19d4ea..e28f5b7e62d 100644 --- a/src/flb_lock.c +++ b/src/flb_lock.c @@ -1,4 +1,5 @@ #include +#include #include #include From 43b5b4746927ec6e61fa5519c890f9afd18f39c1 Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Wed, 30 Oct 2024 14:46:45 +0100 Subject: [PATCH 19/21] signv4: added type cast to fix msvc compilation issue Signed-off-by: Leonardo Alminana --- src/flb_signv4_ng.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flb_signv4_ng.c b/src/flb_signv4_ng.c index f7624ecfcfe..701ebea9a65 100644 --- a/src/flb_signv4_ng.c +++ b/src/flb_signv4_ng.c @@ -415,7 +415,7 @@ static void headers_sanitize_ng(struct flb_http_request *request, struct mk_list /* Sanitize value */ v_start = header_entry->val; - v_end = header_entry->val + header_entry->val_size; + v_end = &((char *) header_entry->val)[header_entry->val_size]; if (header_entry->val_size > 0) { v_end--; From 0cd728613862696f8f6f7626acf65566b981880d Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Wed, 30 Oct 2024 15:59:24 +0100 Subject: [PATCH 20/21] tls: fixed macos compilation error Signed-off-by: Leonardo Alminana --- src/tls/openssl.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/tls/openssl.c b/src/tls/openssl.c index bc78f4572c9..4d4d80c9b54 100644 --- a/src/tls/openssl.c +++ b/src/tls/openssl.c @@ -204,7 +204,7 @@ int tls_context_alpn_set(void *ctx_backend, const char *alpn) } static int tls_context_server_alpn_select_callback(SSL *ssl, - const unsigned char **out, + unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, @@ -218,7 +218,7 @@ static int tls_context_server_alpn_select_callback(SSL *ssl, result = SSL_TLSEXT_ERR_NOACK; if (ctx->alpn != NULL) { - result = SSL_select_next_proto((unsigned char **) out, + result = SSL_select_next_proto(out, outlen, (const unsigned char *) &ctx->alpn[1], (unsigned int) ctx->alpn[0], @@ -369,14 +369,18 @@ static void *tls_context_create(int verify, pthread_mutex_init(&ctx->mutex, NULL); if (mode == FLB_TLS_SERVER_MODE) { - SSL_CTX_set_alpn_select_cb(ssl_ctx, - tls_context_server_alpn_select_callback, - ctx); + SSL_CTX_set_alpn_select_cb( + ssl_ctx, + (SSL_CTX_alpn_select_cb_func) + tls_context_server_alpn_select_callback, + ctx); } else { - SSL_CTX_set_next_proto_select_cb(ssl_ctx, - tls_context_server_alpn_select_callback, - ctx); + SSL_CTX_set_next_proto_select_cb( + ssl_ctx, + (SSL_CTX_npn_select_cb_func) + tls_context_server_alpn_select_callback, + ctx); } /* Verify peer: by default OpenSSL always verify peer */ From fd40f7bea7c9efb5401dbdac01ca6c88cbedd39a Mon Sep 17 00:00:00 2001 From: Leonardo Alminana Date: Wed, 30 Oct 2024 16:08:27 +0100 Subject: [PATCH 21/21] tls: fixed centos compilation error Signed-off-by: Leonardo Alminana --- src/tls/openssl.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/tls/openssl.c b/src/tls/openssl.c index 4d4d80c9b54..54c3468fc47 100644 --- a/src/tls/openssl.c +++ b/src/tls/openssl.c @@ -204,7 +204,7 @@ int tls_context_alpn_set(void *ctx_backend, const char *alpn) } static int tls_context_server_alpn_select_callback(SSL *ssl, - unsigned char **out, + const unsigned char **out, unsigned char *outlen, const unsigned char *in, unsigned int inlen, @@ -218,7 +218,7 @@ static int tls_context_server_alpn_select_callback(SSL *ssl, result = SSL_TLSEXT_ERR_NOACK; if (ctx->alpn != NULL) { - result = SSL_select_next_proto(out, + result = SSL_select_next_proto((unsigned char **) out, outlen, (const unsigned char *) &ctx->alpn[1], (unsigned int) ctx->alpn[0], @@ -236,6 +236,21 @@ static int tls_context_server_alpn_select_callback(SSL *ssl, return result; } +static int tls_context_client_alpn_select_callback(SSL *ssl, + unsigned char **out, + unsigned char *outlen, + const unsigned char *in, + unsigned int inlen, + void *arg) +{ + return tls_context_server_alpn_select_callback(ssl, + (const unsigned char **) out, + outlen, + in, + inlen, + arg); +} + #ifdef _MSC_VER static int windows_load_system_certificates(struct tls_context *ctx) { @@ -371,15 +386,13 @@ static void *tls_context_create(int verify, if (mode == FLB_TLS_SERVER_MODE) { SSL_CTX_set_alpn_select_cb( ssl_ctx, - (SSL_CTX_alpn_select_cb_func) - tls_context_server_alpn_select_callback, + tls_context_server_alpn_select_callback, ctx); } else { SSL_CTX_set_next_proto_select_cb( ssl_ctx, - (SSL_CTX_npn_select_cb_func) - tls_context_server_alpn_select_callback, + tls_context_client_alpn_select_callback, ctx); }