diff --git a/include/fluent-bit/flb_opentelemetry.h b/include/fluent-bit/flb_opentelemetry.h index e2839360cb4..4e005d041c5 100644 --- a/include/fluent-bit/flb_opentelemetry.h +++ b/include/fluent-bit/flb_opentelemetry.h @@ -21,6 +21,7 @@ #define FLB_OPENTELEMETRY_H #include +#include #include #include @@ -173,6 +174,9 @@ int flb_opentelemetry_logs_json_to_msgpack(struct flb_log_event_encoder *encoder const char *logs_body_key, int *error_status); +int flb_opentelemetry_metrics_json_to_cmt(struct cfl_list *context_list, + const char *body, size_t len); + struct ctrace *flb_opentelemetry_json_traces_to_ctrace(const char *body, size_t len, int *error_status); @@ -207,6 +211,12 @@ int flb_otel_utils_json_payload_append_converted_kvlist(struct flb_log_event_enc int target_field, msgpack_object *object); +struct cfl_variant *flb_otel_utils_msgpack_object_to_cfl_variant( + msgpack_object *object); + +int flb_otel_utils_clone_kvlist_from_otlp_json_array(struct cfl_kvlist *target, + msgpack_object *attributes_object); + int flb_otel_utils_hex_to_id(const char *str, int len, unsigned char *out_buf, int out_size); uint64_t flb_otel_utils_convert_string_number_to_u64(char *str, size_t len); diff --git a/lib/cmetrics/CMakeLists.txt b/lib/cmetrics/CMakeLists.txt index c5253f665e9..3579ca6165c 100644 --- a/lib/cmetrics/CMakeLists.txt +++ b/lib/cmetrics/CMakeLists.txt @@ -4,9 +4,9 @@ set(CMAKE_POSITION_INDEPENDENT_CODE ON) set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # CMetrics Version -set(CMT_VERSION_MAJOR 1) +set(CMT_VERSION_MAJOR 2) set(CMT_VERSION_MINOR 0) -set(CMT_VERSION_PATCH 7) +set(CMT_VERSION_PATCH 2) set(CMT_VERSION_STR "${CMT_VERSION_MAJOR}.${CMT_VERSION_MINOR}.${CMT_VERSION_PATCH}") # Include helpers diff --git a/lib/cmetrics/include/cmetrics/cmetrics.h b/lib/cmetrics/include/cmetrics/cmetrics.h index 75ef78d621a..4b5307846c5 100644 --- a/lib/cmetrics/include/cmetrics/cmetrics.h +++ b/lib/cmetrics/include/cmetrics/cmetrics.h @@ -28,6 +28,7 @@ #define CMT_HISTOGRAM 2 #define CMT_SUMMARY 3 #define CMT_UNTYPED 4 +#define CMT_EXP_HISTOGRAM 5 #define CMT_AGGREGATION_TYPE_UNSPECIFIED 0 #define CMT_AGGREGATION_TYPE_DELTA 1 @@ -65,6 +66,7 @@ struct cmt { struct cfl_list counters; struct cfl_list gauges; struct cfl_list histograms; + struct cfl_list exp_histograms; struct cfl_list summaries; struct cfl_list untypeds; diff --git a/lib/cmetrics/include/cmetrics/cmt_cat.h b/lib/cmetrics/include/cmetrics/cmt_cat.h index 15d4b9db783..ee20e80ec09 100644 --- a/lib/cmetrics/include/cmetrics/cmt_cat.h +++ b/lib/cmetrics/include/cmetrics/cmt_cat.h @@ -26,6 +26,7 @@ struct cmt_counter; struct cmt_gauge; struct cmt_untyped; struct cmt_histogram; +struct cmt_exp_histogram; struct cmt_summary; int cmt_cat_copy_label_keys(struct cmt_map *map, char **out); @@ -34,6 +35,7 @@ int cmt_cat_counter(struct cmt *cmt, struct cmt_counter *counter, struct cmt_map int cmt_cat_gauge(struct cmt *cmt, struct cmt_gauge *gauge, struct cmt_map *filtered_map); int cmt_cat_untyped(struct cmt *cmt, struct cmt_untyped *untyped, struct cmt_map *filtered_map); int cmt_cat_histogram(struct cmt *cmt, struct cmt_histogram *histogram, struct cmt_map *filtered_map); +int cmt_cat_exp_histogram(struct cmt *cmt, struct cmt_exp_histogram *exp_histogram, struct cmt_map *filtered_map); int cmt_cat_summary(struct cmt *cmt, struct cmt_summary *summary, struct cmt_map *filtered_map); int cmt_cat(struct cmt *dst, struct cmt *src); diff --git a/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h b/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h index 68b0eb4ba1e..7daa5682ab2 100644 --- a/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h +++ b/lib/cmetrics/include/cmetrics/cmt_decode_msgpack.h @@ -55,6 +55,7 @@ struct cmt_msgpack_decode_context { uint64_t *summary_quantiles; size_t summary_quantiles_count; int aggregation_type; + int metric_value_type_set; }; int cmt_decode_msgpack_create(struct cmt **out_cmt, char *in_buf, size_t in_size, diff --git a/lib/cmetrics/include/cmetrics/cmt_encode_opentelemetry.h b/lib/cmetrics/include/cmetrics/cmt_encode_opentelemetry.h index 46beca4b2ad..fd7c10af2ba 100644 --- a/lib/cmetrics/include/cmetrics/cmt_encode_opentelemetry.h +++ b/lib/cmetrics/include/cmetrics/cmt_encode_opentelemetry.h @@ -34,6 +34,8 @@ struct cmt_opentelemetry_context { size_t resource_index; + size_t scope_metrics_count; + Opentelemetry__Proto__Metrics__V1__ScopeMetrics **scope_metrics_list; Opentelemetry__Proto__Metrics__V1__MetricsData *metrics_data; struct cmt *cmt; }; diff --git a/lib/cmetrics/include/cmetrics/cmt_exp_histogram.h b/lib/cmetrics/include/cmetrics/cmt_exp_histogram.h new file mode 100644 index 00000000000..f9b565ba803 --- /dev/null +++ b/lib/cmetrics/include/cmetrics/cmt_exp_histogram.h @@ -0,0 +1,64 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2021-2022 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef CMT_EXP_HISTOGRAM_H +#define CMT_EXP_HISTOGRAM_H + +#include +#include +#include + +struct cmt_exp_histogram { + struct cmt_opts opts; + struct cmt_map *map; + struct cfl_list _head; + struct cmt *cmt; + int aggregation_type; +}; + +struct cmt_exp_histogram *cmt_exp_histogram_create(struct cmt *cmt, + char *ns, char *subsystem, + char *name, char *help, + int label_count, char **label_keys); + +int cmt_exp_histogram_set_default(struct cmt_exp_histogram *exp_histogram, + uint64_t timestamp, + int32_t scale, + uint64_t zero_count, + double zero_threshold, + int32_t positive_offset, + size_t positive_bucket_count, + uint64_t *positive_bucket_counts, + int32_t negative_offset, + size_t negative_bucket_count, + uint64_t *negative_bucket_counts, + int sum_set, + double sum, + uint64_t count, + int labels_count, char **label_vals); + +int cmt_exp_histogram_destroy(struct cmt_exp_histogram *exp_histogram); + +int cmt_exp_histogram_to_explicit(struct cmt_metric *metric, + double **upper_bounds, + size_t *upper_bounds_count, + uint64_t **bucket_counts, + size_t *bucket_count); + +#endif diff --git a/lib/cmetrics/include/cmetrics/cmt_metric.h b/lib/cmetrics/include/cmetrics/cmt_metric.h index ae066b2939b..c9e4534ce3a 100644 --- a/lib/cmetrics/include/cmetrics/cmt_metric.h +++ b/lib/cmetrics/include/cmetrics/cmt_metric.h @@ -21,16 +21,40 @@ #define CMT_METRIC_H #include +#include + +enum cmt_metric_value_type { + CMT_METRIC_VALUE_DOUBLE = 0, + CMT_METRIC_VALUE_INT64 = 1, + CMT_METRIC_VALUE_UINT64 = 2 +}; struct cmt_metric { /* counters and gauges */ uint64_t val; + uint64_t value_type; + uint64_t val_int64; + uint64_t val_uint64; /* histogram */ uint64_t *hist_buckets; uint64_t hist_count; uint64_t hist_sum; + /* exponential histogram */ + int exp_hist_sum_set; + int32_t exp_hist_scale; + uint64_t exp_hist_zero_count; + double exp_hist_zero_threshold; + int32_t exp_hist_positive_offset; + uint64_t *exp_hist_positive_buckets; + size_t exp_hist_positive_count; + int32_t exp_hist_negative_offset; + uint64_t *exp_hist_negative_buckets; + size_t exp_hist_negative_count; + uint64_t exp_hist_count; + uint64_t exp_hist_sum; + /* summary */ int sum_quantiles_set; /* specify if quantive values has been set */ uint64_t *sum_quantiles; /* 0, 0.25, 0.5, 0.75 and 1 */ @@ -46,11 +70,21 @@ struct cmt_metric { }; void cmt_metric_set(struct cmt_metric *metric, uint64_t timestamp, double val); +void cmt_metric_set_double(struct cmt_metric *metric, uint64_t timestamp, double val); +void cmt_metric_set_int64(struct cmt_metric *metric, uint64_t timestamp, int64_t val); +void cmt_metric_set_uint64(struct cmt_metric *metric, uint64_t timestamp, uint64_t val); void cmt_metric_inc(struct cmt_metric *metric, uint64_t timestamp); void cmt_metric_dec(struct cmt_metric *metric, uint64_t timestamp); void cmt_metric_add(struct cmt_metric *metric, uint64_t timestamp, double val); void cmt_metric_sub(struct cmt_metric *metric, uint64_t timestamp, double val); double cmt_metric_get_value(struct cmt_metric *metric); +int cmt_metric_get_value_type(struct cmt_metric *metric); +int64_t cmt_metric_get_int64_value(struct cmt_metric *metric); +uint64_t cmt_metric_get_uint64_value(struct cmt_metric *metric); +void cmt_metric_get_value_snapshot(struct cmt_metric *metric, + int *out_type, + int64_t *out_int64, + uint64_t *out_uint64); uint64_t cmt_metric_get_timestamp(struct cmt_metric *metric); void cmt_metric_hist_inc(struct cmt_metric *metric, uint64_t timestamp, diff --git a/lib/cmetrics/include/cmetrics/cmt_mpack_utils.h b/lib/cmetrics/include/cmetrics/cmt_mpack_utils.h index 408eb4e328a..264c2f45a5c 100644 --- a/lib/cmetrics/include/cmetrics/cmt_mpack_utils.h +++ b/lib/cmetrics/include/cmetrics/cmt_mpack_utils.h @@ -33,6 +33,7 @@ struct cmt_mpack_map_entry_callback_t { }; int cmt_mpack_consume_double_tag(mpack_reader_t *reader, double *output_buffer); +int cmt_mpack_consume_int_tag(mpack_reader_t *reader, int64_t *output_buffer); int cmt_mpack_consume_uint_tag(mpack_reader_t *reader, uint64_t *output_buffer); int cmt_mpack_consume_string_tag(mpack_reader_t *reader, cfl_sds_t *output_buffer); int cmt_mpack_unpack_map(mpack_reader_t *reader, diff --git a/lib/cmetrics/include/cmetrics/cmt_variant_utils.h b/lib/cmetrics/include/cmetrics/cmt_variant_utils.h index cb1cb443ef7..41d738894ac 100644 --- a/lib/cmetrics/include/cmetrics/cmt_variant_utils.h +++ b/lib/cmetrics/include/cmetrics/cmt_variant_utils.h @@ -88,6 +88,14 @@ static inline int pack_cfl_variant_int64(mpack_writer_t *writer, return 0; } +static inline int pack_cfl_variant_uint64(mpack_writer_t *writer, + uint64_t value) +{ + mpack_write_u64(writer, value); + + return 0; +} + static inline int pack_cfl_variant_double(mpack_writer_t *writer, double value) { @@ -96,6 +104,13 @@ static inline int pack_cfl_variant_double(mpack_writer_t *writer, return 0; } +static inline int pack_cfl_variant_null(mpack_writer_t *writer) +{ + mpack_write_nil(writer); + + return 0; +} + static inline int pack_cfl_variant_array(mpack_writer_t *writer, struct cfl_array *array) { @@ -169,9 +184,15 @@ static inline int pack_cfl_variant(mpack_writer_t *writer, else if (value->type == CFL_VARIANT_INT) { result = pack_cfl_variant_int64(writer, value->data.as_int64); } + else if (value->type == CFL_VARIANT_UINT) { + result = pack_cfl_variant_uint64(writer, value->data.as_uint64); + } else if (value->type == CFL_VARIANT_DOUBLE) { result = pack_cfl_variant_double(writer, value->data.as_double); } + else if (value->type == CFL_VARIANT_NULL) { + result = pack_cfl_variant_null(writer); + } else if (value->type == CFL_VARIANT_ARRAY) { result = pack_cfl_variant_array(writer, value->data.as_array); } @@ -552,6 +573,27 @@ static inline int unpack_cfl_variant_double(mpack_reader_t *reader, return 0; } +static inline int unpack_cfl_variant_null(mpack_reader_t *reader, + struct cfl_variant **value) +{ + mpack_tag_t tag; + int result; + + result = unpack_cfl_variant_read_tag(reader, &tag, mpack_type_nil); + + if (result != 0) { + return result; + } + + *value = cfl_variant_create_from_null(); + + if (*value == NULL) { + return -3; + } + + return 0; +} + static inline int unpack_cfl_variant_array(mpack_reader_t *reader, struct cfl_variant **value) { @@ -624,6 +666,9 @@ static inline int unpack_cfl_variant(mpack_reader_t *reader, else if (value_type == mpack_type_double) { result = unpack_cfl_variant_double(reader, value); } + else if (value_type == mpack_type_nil) { + result = unpack_cfl_variant_null(reader, value); + } else if (value_type == mpack_type_array) { result = unpack_cfl_variant_array(reader, value); } diff --git a/lib/cmetrics/src/CMakeLists.txt b/lib/cmetrics/src/CMakeLists.txt index 4f46ccfdb60..b79dcb373f0 100644 --- a/lib/cmetrics/src/CMakeLists.txt +++ b/lib/cmetrics/src/CMakeLists.txt @@ -14,6 +14,7 @@ set(src cmt_untyped.c cmt_summary.c cmt_histogram.c + cmt_exp_histogram.c cmt_metric.c cmt_metric_histogram.c cmt_map.c @@ -76,6 +77,9 @@ endif() # Static Library add_library(cmetrics-static STATIC ${src}) target_link_libraries(cmetrics-static mpack-static cfl-static fluent-otel-proto) +if(NOT MSVC) + target_link_libraries(cmetrics-static m) +endif() # Install Library if(MSVC) diff --git a/lib/cmetrics/src/cmetrics.c b/lib/cmetrics/src/cmetrics.c index 9817809d12c..a2bb0ffd1b3 100644 --- a/lib/cmetrics/src/cmetrics.c +++ b/lib/cmetrics/src/cmetrics.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include #include @@ -76,6 +77,7 @@ struct cmt *cmt_create() cfl_list_init(&cmt->counters); cfl_list_init(&cmt->gauges); cfl_list_init(&cmt->histograms); + cfl_list_init(&cmt->exp_histograms); cfl_list_init(&cmt->summaries); cfl_list_init(&cmt->untypeds); @@ -94,6 +96,7 @@ void cmt_destroy(struct cmt *cmt) struct cmt_gauge *g; struct cmt_summary *s; struct cmt_histogram *h; + struct cmt_exp_histogram *eh; struct cmt_untyped *u; cfl_list_foreach_safe(head, tmp, &cmt->counters) { @@ -116,6 +119,11 @@ void cmt_destroy(struct cmt *cmt) cmt_histogram_destroy(h); } + cfl_list_foreach_safe(head, tmp, &cmt->exp_histograms) { + eh = cfl_list_entry(head, struct cmt_exp_histogram, _head); + cmt_exp_histogram_destroy(eh); + } + cfl_list_foreach_safe(head, tmp, &cmt->untypeds) { u = cfl_list_entry(head, struct cmt_untyped, _head); cmt_untyped_destroy(u); diff --git a/lib/cmetrics/src/cmt_cat.c b/lib/cmetrics/src/cmt_cat.c index 595225bbd40..8e65ff289be 100644 --- a/lib/cmetrics/src/cmt_cat.c +++ b/lib/cmetrics/src/cmt_cat.c @@ -24,6 +24,7 @@ #include #include #include +#include #include int cmt_cat_copy_label_keys(struct cmt_map *map, char **out) @@ -173,12 +174,231 @@ static inline int cat_summary_values(struct cmt_metric *metric_dst, struct cmt_s return 0; } +static inline int cat_exp_histogram_values(struct cmt_metric *metric_dst, + struct cmt_metric *metric_src) +{ + int64_t dst_start; + int64_t dst_end; + int64_t src_start; + int64_t src_end; + int64_t merged_start; + int64_t merged_end; + size_t index; + size_t merged_count; + uint64_t *merged_buckets; + uint64_t *tmp_buckets; + + if (metric_dst->exp_hist_positive_count > 0 && + metric_dst->exp_hist_positive_buckets == NULL) { + return -1; + } + + if (metric_dst->exp_hist_negative_count > 0 && + metric_dst->exp_hist_negative_buckets == NULL) { + return -1; + } + + if (metric_src->exp_hist_positive_count > 0 && + metric_src->exp_hist_positive_buckets == NULL) { + return -1; + } + + if (metric_src->exp_hist_negative_count > 0 && + metric_src->exp_hist_negative_buckets == NULL) { + return -1; + } + + if (metric_dst->exp_hist_positive_buckets == NULL && + metric_dst->exp_hist_negative_buckets == NULL && + metric_dst->exp_hist_positive_count == 0 && + metric_dst->exp_hist_negative_count == 0 && + metric_dst->exp_hist_count == 0 && + metric_dst->exp_hist_zero_count == 0 && + metric_dst->exp_hist_sum == 0 && + metric_dst->exp_hist_scale == 0 && + metric_dst->exp_hist_positive_offset == 0 && + metric_dst->exp_hist_negative_offset == 0 && + metric_dst->exp_hist_zero_threshold == 0.0) { + if (metric_src->exp_hist_positive_count > 0) { + metric_dst->exp_hist_positive_buckets = calloc(metric_src->exp_hist_positive_count, + sizeof(uint64_t)); + if (metric_dst->exp_hist_positive_buckets == NULL) { + return -1; + } + + memcpy(metric_dst->exp_hist_positive_buckets, + metric_src->exp_hist_positive_buckets, + sizeof(uint64_t) * metric_src->exp_hist_positive_count); + } + + if (metric_src->exp_hist_negative_count > 0) { + metric_dst->exp_hist_negative_buckets = calloc(metric_src->exp_hist_negative_count, + sizeof(uint64_t)); + if (metric_dst->exp_hist_negative_buckets == NULL) { + free(metric_dst->exp_hist_positive_buckets); + metric_dst->exp_hist_positive_buckets = NULL; + + return -1; + } + + memcpy(metric_dst->exp_hist_negative_buckets, + metric_src->exp_hist_negative_buckets, + sizeof(uint64_t) * metric_src->exp_hist_negative_count); + } + + metric_dst->exp_hist_scale = metric_src->exp_hist_scale; + metric_dst->exp_hist_zero_count = metric_src->exp_hist_zero_count; + metric_dst->exp_hist_zero_threshold = metric_src->exp_hist_zero_threshold; + metric_dst->exp_hist_positive_offset = metric_src->exp_hist_positive_offset; + metric_dst->exp_hist_positive_count = metric_src->exp_hist_positive_count; + metric_dst->exp_hist_negative_offset = metric_src->exp_hist_negative_offset; + metric_dst->exp_hist_negative_count = metric_src->exp_hist_negative_count; + metric_dst->exp_hist_count = metric_src->exp_hist_count; + metric_dst->exp_hist_sum_set = metric_src->exp_hist_sum_set; + metric_dst->exp_hist_sum = metric_src->exp_hist_sum; + + return 0; + } + + if (metric_dst->exp_hist_scale != metric_src->exp_hist_scale || + metric_dst->exp_hist_zero_threshold != metric_src->exp_hist_zero_threshold) { + return -1; + } + + if (metric_src->exp_hist_positive_count > 0) { + if (metric_dst->exp_hist_positive_count == 0) { + metric_dst->exp_hist_positive_buckets = calloc(metric_src->exp_hist_positive_count, + sizeof(uint64_t)); + if (metric_dst->exp_hist_positive_buckets == NULL) { + return -1; + } + + memcpy(metric_dst->exp_hist_positive_buckets, + metric_src->exp_hist_positive_buckets, + sizeof(uint64_t) * metric_src->exp_hist_positive_count); + metric_dst->exp_hist_positive_offset = metric_src->exp_hist_positive_offset; + metric_dst->exp_hist_positive_count = metric_src->exp_hist_positive_count; + } + else { + dst_start = metric_dst->exp_hist_positive_offset; + dst_end = dst_start + metric_dst->exp_hist_positive_count; + src_start = metric_src->exp_hist_positive_offset; + src_end = src_start + metric_src->exp_hist_positive_count; + + merged_start = dst_start < src_start ? dst_start : src_start; + merged_end = dst_end > src_end ? dst_end : src_end; + merged_count = (size_t) (merged_end - merged_start); + + merged_buckets = calloc(merged_count, sizeof(uint64_t)); + if (merged_buckets == NULL) { + return -1; + } + + for (index = 0; index < metric_dst->exp_hist_positive_count; index++) { + merged_buckets[(size_t) (dst_start + index - merged_start)] += + metric_dst->exp_hist_positive_buckets[index]; + } + + for (index = 0; index < metric_src->exp_hist_positive_count; index++) { + merged_buckets[(size_t) (src_start + index - merged_start)] += + metric_src->exp_hist_positive_buckets[index]; + } + + tmp_buckets = metric_dst->exp_hist_positive_buckets; + metric_dst->exp_hist_positive_buckets = merged_buckets; + metric_dst->exp_hist_positive_offset = (int32_t) merged_start; + metric_dst->exp_hist_positive_count = merged_count; + free(tmp_buckets); + } + } + + if (metric_src->exp_hist_negative_count > 0) { + if (metric_dst->exp_hist_negative_count == 0) { + metric_dst->exp_hist_negative_buckets = calloc(metric_src->exp_hist_negative_count, + sizeof(uint64_t)); + if (metric_dst->exp_hist_negative_buckets == NULL) { + return -1; + } + + memcpy(metric_dst->exp_hist_negative_buckets, + metric_src->exp_hist_negative_buckets, + sizeof(uint64_t) * metric_src->exp_hist_negative_count); + metric_dst->exp_hist_negative_offset = metric_src->exp_hist_negative_offset; + metric_dst->exp_hist_negative_count = metric_src->exp_hist_negative_count; + } + else { + dst_start = metric_dst->exp_hist_negative_offset; + dst_end = dst_start + metric_dst->exp_hist_negative_count; + src_start = metric_src->exp_hist_negative_offset; + src_end = src_start + metric_src->exp_hist_negative_count; + + merged_start = dst_start < src_start ? dst_start : src_start; + merged_end = dst_end > src_end ? dst_end : src_end; + merged_count = (size_t) (merged_end - merged_start); + + merged_buckets = calloc(merged_count, sizeof(uint64_t)); + if (merged_buckets == NULL) { + return -1; + } + + for (index = 0; index < metric_dst->exp_hist_negative_count; index++) { + merged_buckets[(size_t) (dst_start + index - merged_start)] += + metric_dst->exp_hist_negative_buckets[index]; + } + + for (index = 0; index < metric_src->exp_hist_negative_count; index++) { + merged_buckets[(size_t) (src_start + index - merged_start)] += + metric_src->exp_hist_negative_buckets[index]; + } + + tmp_buckets = metric_dst->exp_hist_negative_buckets; + metric_dst->exp_hist_negative_buckets = merged_buckets; + metric_dst->exp_hist_negative_offset = (int32_t) merged_start; + metric_dst->exp_hist_negative_count = merged_count; + free(tmp_buckets); + } + } + + metric_dst->exp_hist_zero_count += metric_src->exp_hist_zero_count; + metric_dst->exp_hist_count += metric_src->exp_hist_count; + + if (metric_dst->exp_hist_sum_set && metric_src->exp_hist_sum_set) { + metric_dst->exp_hist_sum = cmt_math_d64_to_uint64( + cmt_math_uint64_to_d64(metric_dst->exp_hist_sum) + + cmt_math_uint64_to_d64(metric_src->exp_hist_sum)); + } + else if (metric_src->exp_hist_sum_set) { + metric_dst->exp_hist_sum_set = CMT_TRUE; + metric_dst->exp_hist_sum = metric_src->exp_hist_sum; + } + + return 0; +} + +static inline void cat_scalar_value(struct cmt_metric *metric_dst, + struct cmt_metric *metric_src) +{ + uint64_t ts; + double val; + + ts = cmt_metric_get_timestamp(metric_src); + + if (cmt_metric_get_value_type(metric_src) == CMT_METRIC_VALUE_INT64) { + cmt_metric_set_int64(metric_dst, ts, cmt_metric_get_int64_value(metric_src)); + } + else if (cmt_metric_get_value_type(metric_src) == CMT_METRIC_VALUE_UINT64) { + cmt_metric_set_uint64(metric_dst, ts, cmt_metric_get_uint64_value(metric_src)); + } + else { + val = cmt_metric_get_value(metric_src); + cmt_metric_set_double(metric_dst, ts, val); + } +} + int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map *src) { int c; int ret; - uint64_t ts; - double val; char **labels = NULL; struct cfl_list *head; struct cmt_metric *metric_dst; @@ -210,11 +430,14 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map return -1; } } + else if (src->type == CMT_EXP_HISTOGRAM) { + ret = cat_exp_histogram_values(metric_dst, metric_src); + if (ret == -1) { + return -1; + } + } - ts = cmt_metric_get_timestamp(metric_src); - val = cmt_metric_get_value(metric_src); - - cmt_metric_set(metric_dst, ts, val); + cat_scalar_value(metric_dst, metric_src); } /* Process map dynamic metrics */ @@ -249,11 +472,14 @@ int cmt_cat_copy_map(struct cmt_opts *opts, struct cmt_map *dst, struct cmt_map return -1; } } + else if (src->type == CMT_EXP_HISTOGRAM) { + ret = cat_exp_histogram_values(metric_dst, metric_src); + if (ret == -1) { + return -1; + } + } - ts = cmt_metric_get_timestamp(metric_src); - val = cmt_metric_get_value(metric_src); - - cmt_metric_set(metric_dst, ts, val); + cat_scalar_value(metric_dst, metric_src); } return 0; @@ -342,6 +568,21 @@ static struct cmt_histogram *histogram_lookup(struct cmt *cmt, struct cmt_opts * return NULL; } +static struct cmt_exp_histogram *exp_histogram_lookup(struct cmt *cmt, struct cmt_opts *opts) +{ + struct cmt_exp_histogram *exp_histogram; + struct cfl_list *head; + + cfl_list_foreach(head, &cmt->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + if (cmt_opts_compare(&exp_histogram->opts, opts) == 0) { + return exp_histogram; + } + } + + return NULL; +} + int cmt_cat_counter(struct cmt *cmt, struct cmt_counter *counter, struct cmt_map *filtered_map) { @@ -599,6 +840,50 @@ int cmt_cat_summary(struct cmt *cmt, struct cmt_summary *summary, return 0; } +int cmt_cat_exp_histogram(struct cmt *cmt, struct cmt_exp_histogram *exp_histogram, + struct cmt_map *filtered_map) +{ + int ret; + char **labels = NULL; + struct cmt_map *map; + struct cmt_opts *opts; + struct cmt_exp_histogram *eh; + + map = exp_histogram->map; + opts = map->opts; + + ret = cmt_cat_copy_label_keys(map, (char **) &labels); + if (ret == -1) { + return -1; + } + + eh = exp_histogram_lookup(cmt, opts); + if (!eh) { + eh = cmt_exp_histogram_create(cmt, + opts->ns, opts->subsystem, + opts->name, opts->description, + map->label_count, labels); + } + + free(labels); + if (!eh) { + return -1; + } + + if (filtered_map != NULL) { + ret = cmt_cat_copy_map(&eh->opts, eh->map, filtered_map); + } + else { + ret = cmt_cat_copy_map(&eh->opts, eh->map, map); + } + + if (ret == -1) { + return -1; + } + + return 0; +} + static int append_context(struct cmt *dst, struct cmt *src) { int ret; @@ -607,6 +892,7 @@ static int append_context(struct cmt *dst, struct cmt *src) struct cmt_gauge *gauge; struct cmt_untyped *untyped; struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; struct cmt_summary *summary; /* Counters */ @@ -645,6 +931,15 @@ static int append_context(struct cmt *dst, struct cmt *src) } } + /* Exponential Histogram */ + cfl_list_foreach(head, &src->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + ret = cmt_cat_exp_histogram(dst, exp_histogram, NULL); + if (ret == -1) { + return -1; + } + } + /* Summary */ cfl_list_foreach(head, &src->summaries) { summary = cfl_list_entry(head, struct cmt_summary, _head); diff --git a/lib/cmetrics/src/cmt_decode_msgpack.c b/lib/cmetrics/src/cmt_decode_msgpack.c index 558bbb91b00..c36a12db3b9 100644 --- a/lib/cmetrics/src/cmt_decode_msgpack.c +++ b/lib/cmetrics/src/cmt_decode_msgpack.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -30,6 +31,7 @@ #include #include #include +#include static int create_counter_instance(struct cmt_map *map) @@ -137,6 +139,26 @@ static int create_histogram_instance(struct cmt_map *map) return CMT_DECODE_MSGPACK_SUCCESS; } +static int create_exp_histogram_instance(struct cmt_map *map) +{ + struct cmt_exp_histogram *exp_histogram; + + if (NULL == map) { + return CMT_DECODE_MSGPACK_INVALID_ARGUMENT_ERROR; + } + + exp_histogram = calloc(1, sizeof(struct cmt_exp_histogram)); + + if (NULL == exp_histogram) { + return CMT_DECODE_MSGPACK_ALLOCATION_ERROR; + } + + exp_histogram->map = map; + map->parent = (void *) exp_histogram; + + return CMT_DECODE_MSGPACK_SUCCESS; +} + static int create_metric_instance(struct cmt_map *map) { switch(map->type) { @@ -148,6 +170,8 @@ static int create_metric_instance(struct cmt_map *map) return create_summary_instance(map); case CMT_HISTOGRAM: return create_histogram_instance(map); + case CMT_EXP_HISTOGRAM: + return create_exp_histogram_instance(map); case CMT_UNTYPED: return create_untyped_instance(map); } @@ -409,8 +433,67 @@ static int unpack_metric_value(mpack_reader_t *reader, size_t index, void *conte result = cmt_mpack_consume_double_tag(reader, &value); - if(CMT_DECODE_MSGPACK_SUCCESS == result) { - decode_context->metric->val = cmt_math_d64_to_uint64(value); + if (CMT_DECODE_MSGPACK_SUCCESS == result && + decode_context->metric_value_type_set == CMT_FALSE) { + cmt_metric_set_double(decode_context->metric, + decode_context->metric->timestamp, + value); + } + + return result; +} + +static int unpack_metric_value_type(mpack_reader_t *reader, size_t index, void *context) +{ + uint64_t value; + int result; + struct cmt_msgpack_decode_context *decode_context; + + decode_context = (struct cmt_msgpack_decode_context *) context; + + result = cmt_mpack_consume_uint_tag(reader, &value); + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + if (value == CMT_METRIC_VALUE_INT64 || + value == CMT_METRIC_VALUE_UINT64 || + value == CMT_METRIC_VALUE_DOUBLE) { + cmt_atomic_store(&decode_context->metric->value_type, value); + } + } + + return result; +} + +static int unpack_metric_value_int64(mpack_reader_t *reader, size_t index, void *context) +{ + int64_t value; + int result; + struct cmt_msgpack_decode_context *decode_context; + + decode_context = (struct cmt_msgpack_decode_context *) context; + result = cmt_mpack_consume_int_tag(reader, &value); + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + cmt_metric_set_int64(decode_context->metric, + decode_context->metric->timestamp, + value); + decode_context->metric_value_type_set = CMT_TRUE; + } + + return result; +} + +static int unpack_metric_value_uint64(mpack_reader_t *reader, size_t index, void *context) +{ + uint64_t value; + int result; + struct cmt_msgpack_decode_context *decode_context; + + decode_context = (struct cmt_msgpack_decode_context *) context; + result = cmt_mpack_consume_uint_tag(reader, &value); + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + cmt_metric_set_uint64(decode_context->metric, + decode_context->metric->timestamp, + value); + decode_context->metric_value_type_set = CMT_TRUE; } return result; @@ -461,16 +544,40 @@ static int unpack_summary_quantile(mpack_reader_t *reader, size_t index, void *c } decode_context = (struct cmt_msgpack_decode_context *) context; + + if (decode_context->metric == NULL || + decode_context->metric->sum_quantiles == NULL || + index >= decode_context->metric->sum_quantiles_count) { + return CMT_DECODE_MSGPACK_CORRUPT_INPUT_DATA_ERROR; + } + return cmt_mpack_consume_uint_tag(reader, &decode_context->metric->sum_quantiles[index]); } static int unpack_summary_quantiles(mpack_reader_t *reader, size_t index, void *context) { + size_t expected_count; + size_t entry_count; + struct cmt_msgpack_decode_context *decode_context; + if (NULL == reader || NULL == context ) { return CMT_DECODE_MSGPACK_INVALID_ARGUMENT_ERROR; } + decode_context = (struct cmt_msgpack_decode_context *) context; + expected_count = 0; + + if (decode_context->metric != NULL) { + expected_count = decode_context->metric->sum_quantiles_count; + } + + entry_count = cmt_mpack_peek_array_length(reader); + + if (entry_count != expected_count) { + return CMT_DECODE_MSGPACK_CORRUPT_INPUT_DATA_ERROR; + } + return cmt_mpack_unpack_array(reader, unpack_summary_quantile, context); } @@ -567,6 +674,8 @@ static int unpack_histogram_count(mpack_reader_t *reader, size_t index, void *co static int unpack_histogram_bucket(mpack_reader_t *reader, size_t index, void *context) { struct cmt_msgpack_decode_context *decode_context; + struct cmt_histogram *histogram; + size_t expected_count; if (NULL == reader || NULL == context) { @@ -575,12 +684,32 @@ static int unpack_histogram_bucket(mpack_reader_t *reader, size_t index, void *c decode_context = (struct cmt_msgpack_decode_context *) context; + if (decode_context->map == NULL || + decode_context->metric == NULL || + decode_context->metric->hist_buckets == NULL || + decode_context->map->parent == NULL) { + return CMT_DECODE_MSGPACK_CORRUPT_INPUT_DATA_ERROR; + } + + histogram = (struct cmt_histogram *) decode_context->map->parent; + if (histogram->buckets == NULL) { + return CMT_DECODE_MSGPACK_CORRUPT_INPUT_DATA_ERROR; + } + + expected_count = histogram->buckets->count + 1; + if (index >= expected_count) { + return CMT_DECODE_MSGPACK_CORRUPT_INPUT_DATA_ERROR; + } + return cmt_mpack_consume_uint_tag(reader, &decode_context->metric->hist_buckets[index]); } static int unpack_histogram_buckets(mpack_reader_t *reader, size_t index, void *context) { struct cmt_msgpack_decode_context *decode_context; + struct cmt_histogram *histogram; + size_t expected_count; + size_t entry_count; if (NULL == reader || NULL == context ) { @@ -589,6 +718,22 @@ static int unpack_histogram_buckets(mpack_reader_t *reader, size_t index, void * decode_context = (struct cmt_msgpack_decode_context *) context; + if (decode_context->map == NULL || + decode_context->map->parent == NULL) { + return CMT_DECODE_MSGPACK_CORRUPT_INPUT_DATA_ERROR; + } + + histogram = (struct cmt_histogram *) decode_context->map->parent; + if (histogram->buckets == NULL) { + return CMT_DECODE_MSGPACK_CORRUPT_INPUT_DATA_ERROR; + } + + entry_count = cmt_mpack_peek_array_length(reader); + expected_count = histogram->buckets->count + 1; + if (entry_count != expected_count) { + return CMT_DECODE_MSGPACK_CORRUPT_INPUT_DATA_ERROR; + } + return cmt_mpack_unpack_array(reader, unpack_histogram_bucket, decode_context); } @@ -616,6 +761,175 @@ static int unpack_metric_histogram(mpack_reader_t *reader, size_t index, void *c return result; } +static int unpack_exp_histogram_scale(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_msgpack_decode_context *decode_context; + int64_t value; + int result; + + decode_context = (struct cmt_msgpack_decode_context *) context; + result = cmt_mpack_consume_int_tag(reader, &value); + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + decode_context->metric->exp_hist_scale = (int32_t) value; + } + return result; +} + +static int unpack_exp_histogram_zero_count(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_msgpack_decode_context *decode_context; + decode_context = (struct cmt_msgpack_decode_context *) context; + return cmt_mpack_consume_uint_tag(reader, &decode_context->metric->exp_hist_zero_count); +} + +static int unpack_exp_histogram_zero_threshold(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_msgpack_decode_context *decode_context; + decode_context = (struct cmt_msgpack_decode_context *) context; + return cmt_mpack_consume_double_tag(reader, &decode_context->metric->exp_hist_zero_threshold); +} + +static int unpack_exp_histogram_positive_offset(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_msgpack_decode_context *decode_context; + int64_t value; + int result; + + decode_context = (struct cmt_msgpack_decode_context *) context; + result = cmt_mpack_consume_int_tag(reader, &value); + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + decode_context->metric->exp_hist_positive_offset = (int32_t) value; + } + return result; +} + +static int unpack_exp_histogram_negative_offset(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_msgpack_decode_context *decode_context; + int64_t value; + int result; + + decode_context = (struct cmt_msgpack_decode_context *) context; + result = cmt_mpack_consume_int_tag(reader, &value); + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + decode_context->metric->exp_hist_negative_offset = (int32_t) value; + } + return result; +} + +static int unpack_exp_histogram_positive_bucket(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_msgpack_decode_context *decode_context; + decode_context = (struct cmt_msgpack_decode_context *) context; + return cmt_mpack_consume_uint_tag(reader, &decode_context->metric->exp_hist_positive_buckets[index]); +} + +static int unpack_exp_histogram_negative_bucket(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_msgpack_decode_context *decode_context; + decode_context = (struct cmt_msgpack_decode_context *) context; + return cmt_mpack_consume_uint_tag(reader, &decode_context->metric->exp_hist_negative_buckets[index]); +} + +static int unpack_exp_histogram_positive_buckets(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_msgpack_decode_context *decode_context; + size_t count; + + decode_context = (struct cmt_msgpack_decode_context *) context; + count = cmt_mpack_peek_array_length(reader); + + if (decode_context->metric->exp_hist_positive_buckets != NULL) { + free(decode_context->metric->exp_hist_positive_buckets); + decode_context->metric->exp_hist_positive_buckets = NULL; + decode_context->metric->exp_hist_positive_count = 0; + } + + if (count > 0) { + decode_context->metric->exp_hist_positive_buckets = calloc(count, sizeof(uint64_t)); + if (decode_context->metric->exp_hist_positive_buckets == NULL) { + return CMT_DECODE_MSGPACK_ALLOCATION_ERROR; + } + decode_context->metric->exp_hist_positive_count = count; + } + + return cmt_mpack_unpack_array(reader, unpack_exp_histogram_positive_bucket, context); +} + +static int unpack_exp_histogram_negative_buckets(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_msgpack_decode_context *decode_context; + size_t count; + + decode_context = (struct cmt_msgpack_decode_context *) context; + count = cmt_mpack_peek_array_length(reader); + + if (decode_context->metric->exp_hist_negative_buckets != NULL) { + free(decode_context->metric->exp_hist_negative_buckets); + decode_context->metric->exp_hist_negative_buckets = NULL; + decode_context->metric->exp_hist_negative_count = 0; + } + + if (count > 0) { + decode_context->metric->exp_hist_negative_buckets = calloc(count, sizeof(uint64_t)); + if (decode_context->metric->exp_hist_negative_buckets == NULL) { + return CMT_DECODE_MSGPACK_ALLOCATION_ERROR; + } + decode_context->metric->exp_hist_negative_count = count; + } + + return cmt_mpack_unpack_array(reader, unpack_exp_histogram_negative_bucket, context); +} + +static int unpack_exp_histogram_count(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_msgpack_decode_context *decode_context; + decode_context = (struct cmt_msgpack_decode_context *) context; + return cmt_mpack_consume_uint_tag(reader, &decode_context->metric->exp_hist_count); +} + +static int unpack_exp_histogram_sum_set(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_msgpack_decode_context *decode_context; + uint64_t value; + int result; + + decode_context = (struct cmt_msgpack_decode_context *) context; + result = cmt_mpack_consume_uint_tag(reader, &value); + + if (result == CMT_DECODE_MSGPACK_SUCCESS) { + decode_context->metric->exp_hist_sum_set = value ? CMT_TRUE : CMT_FALSE; + } + + return result; +} + +static int unpack_exp_histogram_sum(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_msgpack_decode_context *decode_context; + decode_context = (struct cmt_msgpack_decode_context *) context; + return cmt_mpack_consume_uint_tag(reader, &decode_context->metric->exp_hist_sum); +} + +static int unpack_metric_exp_histogram(mpack_reader_t *reader, size_t index, void *context) +{ + struct cmt_mpack_map_entry_callback_t callbacks[] = { + {"scale", unpack_exp_histogram_scale}, + {"zero_count", unpack_exp_histogram_zero_count}, + {"zero_threshold", unpack_exp_histogram_zero_threshold}, + {"positive_offset", unpack_exp_histogram_positive_offset}, + {"positive_buckets",unpack_exp_histogram_positive_buckets}, + {"negative_offset", unpack_exp_histogram_negative_offset}, + {"negative_buckets",unpack_exp_histogram_negative_buckets}, + {"count", unpack_exp_histogram_count}, + {"sum_set", unpack_exp_histogram_sum_set}, + {"sum", unpack_exp_histogram_sum}, + {NULL, NULL} + }; + + return cmt_mpack_unpack_map(reader, callbacks, context); +} + static int unpack_metric_hash(mpack_reader_t *reader, size_t index, void *context) { @@ -643,9 +957,13 @@ static int unpack_metric(mpack_reader_t *reader, { {"ts", unpack_metric_ts}, {"value", unpack_metric_value}, + {"value_type", unpack_metric_value_type}, + {"value_int64", unpack_metric_value_int64}, + {"value_uint64", unpack_metric_value_uint64}, {"labels", unpack_metric_labels}, {"summary", unpack_metric_summary}, {"histogram", unpack_metric_histogram}, + {"exp_histogram", unpack_metric_exp_histogram}, {"hash", unpack_metric_hash}, {NULL, NULL} }; @@ -701,6 +1019,7 @@ static int unpack_metric(mpack_reader_t *reader, cfl_list_init(&metric->labels); decode_context->metric = metric; + decode_context->metric_value_type_set = CMT_FALSE; result = cmt_mpack_unpack_map(reader, callbacks, (void *) decode_context); @@ -714,6 +1033,12 @@ static int unpack_metric(mpack_reader_t *reader, if (NULL != metric->sum_quantiles) { free(metric->sum_quantiles); } + if (NULL != metric->exp_hist_positive_buckets) { + free(metric->exp_hist_positive_buckets); + } + if (NULL != metric->exp_hist_negative_buckets) { + free(metric->exp_hist_negative_buckets); + } free(metric); } @@ -758,8 +1083,25 @@ static int unpack_metric_array_entry(mpack_reader_t *reader, size_t index, void decode_context->map->metric.sum_count = metric->sum_count; decode_context->map->metric.sum_sum = metric->sum_sum; } + else if (decode_context->map->type == CMT_EXP_HISTOGRAM) { + decode_context->map->metric.exp_hist_scale = metric->exp_hist_scale; + decode_context->map->metric.exp_hist_zero_count = metric->exp_hist_zero_count; + decode_context->map->metric.exp_hist_zero_threshold = metric->exp_hist_zero_threshold; + decode_context->map->metric.exp_hist_positive_offset = metric->exp_hist_positive_offset; + decode_context->map->metric.exp_hist_positive_count = metric->exp_hist_positive_count; + decode_context->map->metric.exp_hist_positive_buckets = metric->exp_hist_positive_buckets; + decode_context->map->metric.exp_hist_negative_offset = metric->exp_hist_negative_offset; + decode_context->map->metric.exp_hist_negative_count = metric->exp_hist_negative_count; + decode_context->map->metric.exp_hist_negative_buckets = metric->exp_hist_negative_buckets; + decode_context->map->metric.exp_hist_count = metric->exp_hist_count; + decode_context->map->metric.exp_hist_sum_set = metric->exp_hist_sum_set; + decode_context->map->metric.exp_hist_sum = metric->exp_hist_sum; + } decode_context->map->metric.val = metric->val; + decode_context->map->metric.value_type = metric->value_type; + decode_context->map->metric.val_int64 = metric->val_int64; + decode_context->map->metric.val_uint64 = metric->val_uint64; decode_context->map->metric.hash = metric->hash; decode_context->map->metric.timestamp = metric->timestamp; @@ -1297,6 +1639,34 @@ static int append_unpacked_histogram_to_metrics_context( return CMT_DECODE_MSGPACK_SUCCESS; } +static int append_unpacked_exp_histogram_to_metrics_context( + struct cmt *context, + struct cmt_map *map) +{ + struct cmt_exp_histogram *exp_histogram; + + if (NULL == context || NULL == map) { + return CMT_DECODE_MSGPACK_INVALID_ARGUMENT_ERROR; + } + + exp_histogram = map->parent; + if (NULL == exp_histogram) { + return CMT_DECODE_MSGPACK_ALLOCATION_ERROR; + } + + exp_histogram->cmt = context; + exp_histogram->map = map; + map->parent = (void *) exp_histogram; + + memcpy(&exp_histogram->opts, map->opts, sizeof(struct cmt_opts)); + free(map->opts); + map->opts = &exp_histogram->opts; + + cfl_list_add(&exp_histogram->_head, &context->exp_histograms); + + return CMT_DECODE_MSGPACK_SUCCESS; +} + static int unpack_basic_type_entry(mpack_reader_t *reader, size_t index, void *context) { int result; @@ -1325,6 +1695,9 @@ static int unpack_basic_type_entry(mpack_reader_t *reader, size_t index, void *c else if (CMT_HISTOGRAM == map->type) { result = append_unpacked_histogram_to_metrics_context(cmt, map); } + else if (CMT_EXP_HISTOGRAM == map->type) { + result = append_unpacked_exp_histogram_to_metrics_context(cmt, map); + } else if (CMT_UNTYPED == map->type) { result = append_unpacked_untyped_to_metrics_context(cmt, map); } diff --git a/lib/cmetrics/src/cmt_decode_opentelemetry.c b/lib/cmetrics/src/cmt_decode_opentelemetry.c index 5ead2b92a86..54c5cd29e38 100644 --- a/lib/cmetrics/src/cmt_decode_opentelemetry.c +++ b/lib/cmetrics/src/cmt_decode_opentelemetry.c @@ -24,10 +24,14 @@ #include #include #include +#include #include #include #include +#include +#include + static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyValue *source); static int clone_array(struct cfl_array *target, @@ -42,6 +46,16 @@ static int clone_kvlist_entry(struct cfl_kvlist *target, static struct cmt_map_label *create_label(char *caption, size_t length); static int append_new_map_label_key(struct cmt_map *map, char *name); static int append_new_metric_label_value(struct cmt_metric *metric, char *name, size_t length); +static int decode_metric_unit(struct cmt_map *map, char *unit); +static uint64_t compute_metric_hash(struct cmt_map *map, struct cmt_metric *sample); +static struct cfl_kvlist *get_or_create_metric_metadata_context(struct cmt *cmt, struct cmt_map *map); +static struct cfl_kvlist *get_or_create_data_point_metadata_context(struct cmt *cmt, + struct cmt_map *map, + struct cmt_metric *sample, + uint64_t timestamp); +static int clone_exemplars_to_kvlist(struct cfl_kvlist *target, + Opentelemetry__Proto__Metrics__V1__Exemplar **exemplars, + size_t exemplar_count); static struct cfl_variant *clone_variant(Opentelemetry__Proto__Common__V1__AnyValue *source) { @@ -254,6 +268,248 @@ static struct cmt_map_label *create_label(char *caption, size_t length) return instance; } +static int decode_metric_unit(struct cmt_map *map, char *unit) +{ + if (map == NULL) { + return CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR; + } + + if (map->unit != NULL) { + cfl_sds_destroy(map->unit); + map->unit = NULL; + } + + if (unit == NULL || unit[0] == '\0') { + return CMT_DECODE_OPENTELEMETRY_SUCCESS; + } + + map->unit = cfl_sds_create(unit); + if (map->unit == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + return CMT_DECODE_OPENTELEMETRY_SUCCESS; +} + +static uint64_t compute_metric_hash(struct cmt_map *map, struct cmt_metric *sample) +{ + struct cfl_list *head; + struct cmt_map_label *label_value; + cfl_hash_state_t state; + + if (sample == NULL || map == NULL) { + return 0; + } + + if (cfl_list_size(&sample->labels) == 0) { + return 0; + } + + cfl_hash_64bits_reset(&state); + cfl_hash_64bits_update(&state, map->opts->fqname, cfl_sds_len(map->opts->fqname)); + + cfl_list_foreach(head, &sample->labels) { + label_value = cfl_list_entry(head, struct cmt_map_label, _head); + cfl_hash_64bits_update(&state, label_value->name, cfl_sds_len(label_value->name)); + } + + return cfl_hash_64bits_digest(&state); +} + +static char *map_type_to_key(int map_type) +{ + switch (map_type) { + case CMT_COUNTER: + return "counter"; + case CMT_GAUGE: + return "gauge"; + case CMT_UNTYPED: + return "untyped"; + case CMT_SUMMARY: + return "summary"; + case CMT_HISTOGRAM: + return "histogram"; + case CMT_EXP_HISTOGRAM: + return "exp_histogram"; + default: + return "unknown"; + } +} + +static struct cfl_kvlist *get_or_create_metric_metadata_context(struct cmt *cmt, struct cmt_map *map) +{ + struct cfl_kvlist *otlp_root; + struct cfl_kvlist *metrics_root; + struct cfl_kvlist *type_root; + + if (cmt == NULL || map == NULL || map->opts == NULL || map->opts->fqname == NULL) { + return NULL; + } + + otlp_root = get_or_create_external_metadata_kvlist(cmt->external_metadata, "otlp"); + if (otlp_root == NULL) { + return NULL; + } + + metrics_root = get_or_create_external_metadata_kvlist(otlp_root, "metrics"); + if (metrics_root == NULL) { + return NULL; + } + + type_root = get_or_create_external_metadata_kvlist(metrics_root, map_type_to_key(map->type)); + if (type_root == NULL) { + return NULL; + } + + return get_or_create_external_metadata_kvlist(type_root, map->opts->fqname); +} + +static struct cfl_kvlist *get_or_create_data_point_metadata_context(struct cmt *cmt, + struct cmt_map *map, + struct cmt_metric *sample, + uint64_t timestamp) +{ + char key[128]; + struct cfl_kvlist *metric_context; + struct cfl_kvlist *datapoints_context; + + if (sample != NULL && sample->hash == 0 && cfl_list_size(&sample->labels) > 0) { + sample->hash = compute_metric_hash(map, sample); + } + + metric_context = get_or_create_metric_metadata_context(cmt, map); + if (metric_context == NULL) { + return NULL; + } + + datapoints_context = get_or_create_external_metadata_kvlist(metric_context, "datapoints"); + if (datapoints_context == NULL) { + return NULL; + } + + snprintf(key, sizeof(key) - 1, "%" PRIx64 ":%" PRIu64, + sample != NULL ? sample->hash : 0, timestamp); + + return get_or_create_external_metadata_kvlist(datapoints_context, key); +} + +static int clone_exemplars_to_kvlist(struct cfl_kvlist *target, + Opentelemetry__Proto__Metrics__V1__Exemplar **exemplars, + size_t exemplar_count) +{ + size_t index; + size_t entry_index; + int result; + struct cfl_array *array; + struct cfl_kvlist *entry; + struct cfl_kvlist *filtered_attributes; + + if (target == NULL || exemplars == NULL || exemplar_count == 0) { + return 0; + } + + array = cfl_array_create(exemplar_count); + if (array == NULL) { + return -1; + } + + for (index = 0 ; index < exemplar_count ; index++) { + entry = cfl_kvlist_create(); + if (entry == NULL) { + cfl_array_destroy(array); + return -1; + } + + result = cfl_kvlist_insert_uint64(entry, "time_unix_nano", exemplars[index]->time_unix_nano); + if (result != 0) { + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + + if (exemplars[index]->span_id.len > 0) { + result = cfl_kvlist_insert_bytes(entry, "span_id", + (char *) exemplars[index]->span_id.data, + exemplars[index]->span_id.len, CFL_FALSE); + if (result != 0) { + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + } + + if (exemplars[index]->trace_id.len > 0) { + result = cfl_kvlist_insert_bytes(entry, "trace_id", + (char *) exemplars[index]->trace_id.data, + exemplars[index]->trace_id.len, CFL_FALSE); + if (result != 0) { + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + } + + if (exemplars[index]->value_case == OPENTELEMETRY__PROTO__METRICS__V1__EXEMPLAR__VALUE_AS_DOUBLE) { + result = cfl_kvlist_insert_double(entry, "as_double", exemplars[index]->as_double); + if (result != 0) { + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + } + else if (exemplars[index]->value_case == OPENTELEMETRY__PROTO__METRICS__V1__EXEMPLAR__VALUE_AS_INT) { + result = cfl_kvlist_insert_int64(entry, "as_int", exemplars[index]->as_int); + if (result != 0) { + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + } + + if (exemplars[index]->n_filtered_attributes > 0) { + filtered_attributes = cfl_kvlist_create(); + if (filtered_attributes == NULL) { + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + + for (entry_index = 0 ; entry_index < exemplars[index]->n_filtered_attributes ; entry_index++) { + result = clone_kvlist_entry(filtered_attributes, exemplars[index]->filtered_attributes[entry_index]); + if (result != 0) { + cfl_kvlist_destroy(filtered_attributes); + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + } + + result = cfl_kvlist_insert_kvlist(entry, "filtered_attributes", filtered_attributes); + if (result != 0) { + cfl_kvlist_destroy(filtered_attributes); + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + } + + result = cfl_array_append_kvlist(array, entry); + if (result != 0) { + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + } + + result = cfl_kvlist_insert_array(target, "exemplars", array); + if (result != 0) { + cfl_array_destroy(array); + return -1; + } + + return 0; +} + static int append_new_map_label_key(struct cmt_map *map, char *name) { struct cmt_map_label *label; @@ -406,7 +662,6 @@ static int decode_numerical_data_point(struct cmt *cmt, int static_metric_detected; struct cmt_metric *sample; int result; - double value; static_metric_detected = CMT_FALSE; @@ -449,21 +704,43 @@ static int decode_numerical_data_point(struct cmt *cmt, } if (result == CMT_DECODE_OPENTELEMETRY_SUCCESS) { - value = 0; + struct cfl_kvlist *point_metadata; + int number_value_case; + + number_value_case = -1; if (data_point->value_case == OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT) { - if (data_point->as_int < 0) { - value = 0; + number_value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT; + + if (map->type == CMT_COUNTER && + ((struct cmt_counter *) map->parent)->allow_reset == CMT_FALSE && + data_point->as_int < 0) { + cmt_metric_set_double(sample, data_point->time_unix_nano, 0.0); } else { - value = data_point->as_int; + cmt_metric_set_int64(sample, data_point->time_unix_nano, data_point->as_int); } } else if (data_point->value_case == OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_DOUBLE) { - value = data_point->as_double; + number_value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_DOUBLE; + cmt_metric_set_double(sample, data_point->time_unix_nano, data_point->as_double); + } + else { + return CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR; } - cmt_metric_set(sample, data_point->time_unix_nano, value); + point_metadata = get_or_create_data_point_metadata_context(cmt, map, sample, data_point->time_unix_nano); + if (point_metadata != NULL) { + cfl_kvlist_insert_uint64(point_metadata, "start_time_unix_nano", data_point->start_time_unix_nano); + cfl_kvlist_insert_uint64(point_metadata, "flags", data_point->flags); + if (number_value_case == OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT) { + cfl_kvlist_insert_string(point_metadata, "number_value_case", "int"); + } + else { + cfl_kvlist_insert_string(point_metadata, "number_value_case", "double"); + } + clone_exemplars_to_kvlist(point_metadata, data_point->exemplars, data_point->n_exemplars); + } } return result; @@ -560,6 +837,8 @@ static int decode_summary_data_point(struct cmt *cmt, } if (result == CMT_DECODE_OPENTELEMETRY_SUCCESS) { + struct cfl_kvlist *point_metadata; + if (sample->sum_quantiles_set == CMT_FALSE) { sample->sum_quantiles = calloc(data_point->n_quantile_values, sizeof(uint64_t)); @@ -581,6 +860,12 @@ static int decode_summary_data_point(struct cmt *cmt, sample->sum_sum = cmt_math_d64_to_uint64(data_point->sum); sample->sum_count = data_point->count; + + point_metadata = get_or_create_data_point_metadata_context(cmt, map, sample, data_point->time_unix_nano); + if (point_metadata != NULL) { + cfl_kvlist_insert_uint64(point_metadata, "start_time_unix_nano", data_point->start_time_unix_nano); + cfl_kvlist_insert_uint64(point_metadata, "flags", data_point->flags); + } } return result; @@ -673,6 +958,8 @@ static int decode_histogram_data_point(struct cmt *cmt, } if (result == CMT_DECODE_OPENTELEMETRY_SUCCESS) { + struct cfl_kvlist *point_metadata; + if (sample->hist_buckets == NULL) { sample->hist_buckets = calloc(data_point->n_bucket_counts + 1, sizeof(uint64_t)); @@ -691,6 +978,22 @@ static int decode_histogram_data_point(struct cmt *cmt, sample->hist_sum = cmt_math_d64_to_uint64(data_point->sum); sample->hist_count = data_point->count; + + point_metadata = get_or_create_data_point_metadata_context(cmt, map, sample, data_point->time_unix_nano); + if (point_metadata != NULL) { + cfl_kvlist_insert_uint64(point_metadata, "start_time_unix_nano", data_point->start_time_unix_nano); + cfl_kvlist_insert_uint64(point_metadata, "flags", data_point->flags); + cfl_kvlist_insert_bool(point_metadata, "has_sum", data_point->has_sum ? CFL_TRUE : CFL_FALSE); + if (data_point->has_min) { + cfl_kvlist_insert_bool(point_metadata, "has_min", CFL_TRUE); + cfl_kvlist_insert_double(point_metadata, "min", data_point->min); + } + if (data_point->has_max) { + cfl_kvlist_insert_bool(point_metadata, "has_max", CFL_TRUE); + cfl_kvlist_insert_double(point_metadata, "max", data_point->max); + } + clone_exemplars_to_kvlist(point_metadata, data_point->exemplars, data_point->n_exemplars); + } } return result; @@ -727,6 +1030,7 @@ static int decode_counter_entry(struct cmt *cmt, counter = (struct cmt_counter *) instance; counter->map->metric_static_set = 0; + counter->allow_reset = !metric->is_monotonic; result = decode_numerical_data_point_list(cmt, counter->map, @@ -744,7 +1048,6 @@ static int decode_counter_entry(struct cmt *cmt, counter->aggregation_type = CMT_AGGREGATION_TYPE_UNSPECIFIED; } - counter->allow_reset = !metric->is_monotonic; } return result; @@ -826,6 +1129,188 @@ static int decode_histogram_entry(struct cmt *cmt, return result; } +static int decode_exponential_histogram_data_point(struct cmt *cmt, + struct cmt_map *map, + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint *data_point) +{ + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets *positive; + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets *negative; + struct cmt_metric *sample; + int static_metric_detected; + int result; + size_t index; + + result = CMT_DECODE_OPENTELEMETRY_SUCCESS; + positive = data_point->positive; + negative = data_point->negative; + + static_metric_detected = CMT_FALSE; + + if (data_point->n_attributes == 0) { + if (map->metric_static_set == CMT_FALSE) { + static_metric_detected = CMT_TRUE; + } + } + + if (static_metric_detected == CMT_FALSE) { + sample = calloc(1, sizeof(struct cmt_metric)); + + if (sample == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + cfl_list_init(&sample->labels); + + result = decode_data_point_labels(cmt, + map, + sample, + data_point->n_attributes, + data_point->attributes); + + if (result != 0) { + destroy_label_list(&sample->labels); + free(sample); + return result; + } + else { + cfl_list_add(&sample->_head, &map->metrics); + } + } + else { + sample = &map->metric; + map->metric_static_set = CMT_TRUE; + } + + if (sample->exp_hist_positive_buckets != NULL) { + free(sample->exp_hist_positive_buckets); + sample->exp_hist_positive_buckets = NULL; + sample->exp_hist_positive_count = 0; + } + + if (sample->exp_hist_negative_buckets != NULL) { + free(sample->exp_hist_negative_buckets); + sample->exp_hist_negative_buckets = NULL; + sample->exp_hist_negative_count = 0; + } + + if (positive != NULL && positive->n_bucket_counts > 0) { + sample->exp_hist_positive_buckets = calloc(positive->n_bucket_counts, sizeof(uint64_t)); + if (sample->exp_hist_positive_buckets == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + for (index = 0 ; index < positive->n_bucket_counts ; index++) { + sample->exp_hist_positive_buckets[index] = positive->bucket_counts[index]; + } + sample->exp_hist_positive_count = positive->n_bucket_counts; + sample->exp_hist_positive_offset = positive->offset; + } + else { + sample->exp_hist_positive_offset = 0; + } + + if (negative != NULL && negative->n_bucket_counts > 0) { + sample->exp_hist_negative_buckets = calloc(negative->n_bucket_counts, sizeof(uint64_t)); + if (sample->exp_hist_negative_buckets == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + for (index = 0 ; index < negative->n_bucket_counts ; index++) { + sample->exp_hist_negative_buckets[index] = negative->bucket_counts[index]; + } + sample->exp_hist_negative_count = negative->n_bucket_counts; + sample->exp_hist_negative_offset = negative->offset; + } + else { + sample->exp_hist_negative_offset = 0; + } + + sample->exp_hist_scale = data_point->scale; + sample->exp_hist_zero_count = data_point->zero_count; + sample->exp_hist_zero_threshold = data_point->zero_threshold; + sample->exp_hist_count = data_point->count; + sample->exp_hist_sum_set = data_point->has_sum ? CMT_TRUE : CMT_FALSE; + if (sample->exp_hist_sum_set == CMT_TRUE) { + sample->exp_hist_sum = cmt_math_d64_to_uint64(data_point->sum); + } + else { + sample->exp_hist_sum = 0; + } + sample->timestamp = data_point->time_unix_nano; + + { + struct cfl_kvlist *point_metadata; + + point_metadata = get_or_create_data_point_metadata_context(cmt, map, sample, data_point->time_unix_nano); + if (point_metadata != NULL) { + cfl_kvlist_insert_uint64(point_metadata, "start_time_unix_nano", data_point->start_time_unix_nano); + cfl_kvlist_insert_uint64(point_metadata, "flags", data_point->flags); + cfl_kvlist_insert_bool(point_metadata, "has_sum", data_point->has_sum ? CFL_TRUE : CFL_FALSE); + if (data_point->has_min) { + cfl_kvlist_insert_bool(point_metadata, "has_min", CFL_TRUE); + cfl_kvlist_insert_double(point_metadata, "min", data_point->min); + } + if (data_point->has_max) { + cfl_kvlist_insert_bool(point_metadata, "has_max", CFL_TRUE); + cfl_kvlist_insert_double(point_metadata, "max", data_point->max); + } + clone_exemplars_to_kvlist(point_metadata, data_point->exemplars, data_point->n_exemplars); + } + } + + return result; +} + +static int decode_exponential_histogram_data_point_list( + struct cmt *cmt, + struct cmt_map *map, + size_t data_point_count, + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint **data_point_list) +{ + size_t index; + int result; + + result = CMT_DECODE_OPENTELEMETRY_SUCCESS; + + for (index = 0 ; + result == CMT_DECODE_OPENTELEMETRY_SUCCESS && + index < data_point_count ; + index++) { + result = decode_exponential_histogram_data_point(cmt, map, data_point_list[index]); + } + + return result; +} + +static int decode_exponential_histogram_entry(struct cmt *cmt, + void *instance, + Opentelemetry__Proto__Metrics__V1__ExponentialHistogram *metric) +{ + struct cmt_exp_histogram *exp_histogram; + int result; + + exp_histogram = (struct cmt_exp_histogram *) instance; + + exp_histogram->map->metric_static_set = 0; + + result = decode_exponential_histogram_data_point_list(cmt, + exp_histogram->map, + metric->n_data_points, + metric->data_points); + + if (result == CMT_DECODE_OPENTELEMETRY_SUCCESS) { + if (metric->aggregation_temporality == OPENTELEMETRY__PROTO__METRICS__V1__AGGREGATION_TEMPORALITY__AGGREGATION_TEMPORALITY_DELTA) { + exp_histogram->aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + } + else if (metric->aggregation_temporality == OPENTELEMETRY__PROTO__METRICS__V1__AGGREGATION_TEMPORALITY__AGGREGATION_TEMPORALITY_CUMULATIVE) { + exp_histogram->aggregation_type = CMT_AGGREGATION_TYPE_CUMULATIVE; + } + else { + exp_histogram->aggregation_type = CMT_AGGREGATION_TYPE_UNSPECIFIED; + } + } + + return result; +} + static int decode_metrics_entry(struct cmt *cmt, Opentelemetry__Proto__Metrics__V1__Metric *metric) { @@ -833,12 +1318,15 @@ static int decode_metrics_entry(struct cmt *cmt, char *metric_namespace; char *metric_subsystem; char *metric_name; + char *metric_unit; + struct cmt_map *map; void *instance; int result; result = CMT_DECODE_OPENTELEMETRY_SUCCESS; metric_name = metric->name; + metric_unit = metric->unit; metric_namespace = ""; metric_subsystem = ""; metric_description = metric->description; @@ -862,11 +1350,33 @@ static int decode_metrics_entry(struct cmt *cmt, return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; } + map = ((struct cmt_counter *) instance)->map; + result = decode_metric_unit(map, metric_unit); + if (result != CMT_DECODE_OPENTELEMETRY_SUCCESS) { + cmt_counter_destroy(instance); + return result; + } + result = decode_counter_entry(cmt, instance, metric->sum); if (result) { cmt_counter_destroy(instance); } + else if (metric->n_metadata > 0) { + struct cfl_kvlist *metric_context; + struct cfl_kvlist *metric_metadata; + size_t index; + + metric_context = get_or_create_metric_metadata_context(cmt, ((struct cmt_counter *) instance)->map); + if (metric_context != NULL) { + metric_metadata = get_or_create_external_metadata_kvlist(metric_context, "metadata"); + if (metric_metadata != NULL) { + for (index = 0; index < metric->n_metadata; index++) { + clone_kvlist_entry(metric_metadata, metric->metadata[index]); + } + } + } + } } else if (metric->data_case == OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_GAUGE) { instance = cmt_gauge_create(cmt, @@ -880,11 +1390,33 @@ static int decode_metrics_entry(struct cmt *cmt, return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; } + map = ((struct cmt_gauge *) instance)->map; + result = decode_metric_unit(map, metric_unit); + if (result != CMT_DECODE_OPENTELEMETRY_SUCCESS) { + cmt_gauge_destroy(instance); + return result; + } + result = decode_gauge_entry(cmt, instance, metric->gauge); if (result) { cmt_gauge_destroy(instance); } + else if (metric->n_metadata > 0) { + struct cfl_kvlist *metric_context; + struct cfl_kvlist *metric_metadata; + size_t index; + + metric_context = get_or_create_metric_metadata_context(cmt, ((struct cmt_gauge *) instance)->map); + if (metric_context != NULL) { + metric_metadata = get_or_create_external_metadata_kvlist(metric_context, "metadata"); + if (metric_metadata != NULL) { + for (index = 0; index < metric->n_metadata; index++) { + clone_kvlist_entry(metric_metadata, metric->metadata[index]); + } + } + } + } } else if (metric->data_case == OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_SUMMARY) { instance = cmt_summary_create(cmt, @@ -899,6 +1431,13 @@ static int decode_metrics_entry(struct cmt *cmt, return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; } + map = ((struct cmt_summary *) instance)->map; + result = decode_metric_unit(map, metric_unit); + if (result != CMT_DECODE_OPENTELEMETRY_SUCCESS) { + cmt_summary_destroy(instance); + return result; + } + /* We are forced to create at least one quantile by the constructor but we * don't know the details about it at the moment so we just leave it "open" */ @@ -908,6 +1447,21 @@ static int decode_metrics_entry(struct cmt *cmt, if (result) { cmt_summary_destroy(instance); } + else if (metric->n_metadata > 0) { + struct cfl_kvlist *metric_context; + struct cfl_kvlist *metric_metadata; + size_t index; + + metric_context = get_or_create_metric_metadata_context(cmt, ((struct cmt_summary *) instance)->map); + if (metric_context != NULL) { + metric_metadata = get_or_create_external_metadata_kvlist(metric_context, "metadata"); + if (metric_metadata != NULL) { + for (index = 0; index < metric->n_metadata; index++) { + clone_kvlist_entry(metric_metadata, metric->metadata[index]); + } + } + } + } } else if (metric->data_case == OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_HISTOGRAM) { instance = cmt_histogram_create(cmt, @@ -922,11 +1476,75 @@ static int decode_metrics_entry(struct cmt *cmt, return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; } + map = ((struct cmt_histogram *) instance)->map; + result = decode_metric_unit(map, metric_unit); + if (result != CMT_DECODE_OPENTELEMETRY_SUCCESS) { + cmt_histogram_destroy(instance); + return result; + } + result = decode_histogram_entry(cmt, instance, metric->histogram); if (result) { cmt_histogram_destroy(instance); } + else if (metric->n_metadata > 0) { + struct cfl_kvlist *metric_context; + struct cfl_kvlist *metric_metadata; + size_t index; + + metric_context = get_or_create_metric_metadata_context(cmt, ((struct cmt_histogram *) instance)->map); + if (metric_context != NULL) { + metric_metadata = get_or_create_external_metadata_kvlist(metric_context, "metadata"); + if (metric_metadata != NULL) { + for (index = 0; index < metric->n_metadata; index++) { + clone_kvlist_entry(metric_metadata, metric->metadata[index]); + } + } + } + } + } + else if (metric->data_case == OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_EXPONENTIAL_HISTOGRAM) { + instance = cmt_exp_histogram_create(cmt, + metric_namespace, + metric_subsystem, + metric_name, + metric_description, + 0, NULL); + + if (instance == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + map = ((struct cmt_exp_histogram *) instance)->map; + result = decode_metric_unit(map, metric_unit); + if (result != CMT_DECODE_OPENTELEMETRY_SUCCESS) { + cmt_exp_histogram_destroy(instance); + return result; + } + + result = decode_exponential_histogram_entry(cmt, + instance, + metric->exponential_histogram); + + if (result) { + cmt_exp_histogram_destroy(instance); + } + else if (metric->n_metadata > 0) { + struct cfl_kvlist *metric_context; + struct cfl_kvlist *metric_metadata; + size_t index; + + metric_context = get_or_create_metric_metadata_context(cmt, ((struct cmt_exp_histogram *) instance)->map); + if (metric_context != NULL) { + metric_metadata = get_or_create_external_metadata_kvlist(metric_context, "metadata"); + if (metric_metadata != NULL) { + for (index = 0; index < metric->n_metadata; index++) { + clone_kvlist_entry(metric_metadata, metric->metadata[index]); + } + } + } + } } return result; diff --git a/lib/cmetrics/src/cmt_encode_cloudwatch_emf.c b/lib/cmetrics/src/cmt_encode_cloudwatch_emf.c index af36eaa0798..dae82ee8419 100644 --- a/lib/cmetrics/src/cmt_encode_cloudwatch_emf.c +++ b/lib/cmetrics/src/cmt_encode_cloudwatch_emf.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -107,6 +108,9 @@ static void pack_cmetrics_type(mpack_writer_t *writer, struct cmt *cmt, else if (map->type == CMT_HISTOGRAM) { mpack_write_cstr(writer, "histogram"); } + else if (map->type == CMT_EXP_HISTOGRAM) { + mpack_write_cstr(writer, "histogram"); + } else { mpack_write_cstr(writer, ""); } @@ -120,23 +124,56 @@ static void pack_histogram_metric(mpack_writer_t *writer, struct cmt *cmt, int index = 0; double val = 0.0; double tmp; - uint64_t *hist_metrics; + uint64_t *hist_metrics = NULL; + uint64_t *exp_bucket_counts = NULL; + double *exp_upper_bounds = NULL; + size_t exp_bucket_count = 0; + size_t exp_upper_bounds_count = 0; + size_t bucket_count = 0; struct cmt_opts *opts = map->opts; struct cmt_histogram *histogram = NULL; struct cmt_histogram_buckets *buckets = NULL; - histogram = (struct cmt_histogram *) map->parent; - buckets = histogram->buckets; - hist_metrics = calloc(buckets->count + 1, sizeof(uint64_t)); + if (map->type == CMT_HISTOGRAM) { + histogram = (struct cmt_histogram *) map->parent; + buckets = histogram->buckets; + bucket_count = buckets->count; + } + else if (map->type == CMT_EXP_HISTOGRAM) { + if (cmt_exp_histogram_to_explicit(metric, + &exp_upper_bounds, + &exp_upper_bounds_count, + &exp_bucket_counts, + &exp_bucket_count) != 0) { + return; + } + + bucket_count = exp_upper_bounds_count; + } + else { + return; + } - for (i = 0; i <= buckets->count; i++) { - hist_metrics[i] = cmt_metric_hist_get_value(metric, i); + hist_metrics = calloc(bucket_count + 1, sizeof(uint64_t)); + if (hist_metrics == NULL) { + free(exp_bucket_counts); + free(exp_upper_bounds); + return; } - for (i = 0; i <= buckets->count; i++) { + for (i = 0; i <= bucket_count; i++) { + if (map->type == CMT_HISTOGRAM) { + hist_metrics[i] = cmt_metric_hist_get_value(metric, i); + } + else { + hist_metrics[i] = exp_bucket_counts[i]; + } + } + + for (i = 0; i <= bucket_count; i++) { index = i; - for (k = i + 1; k <= buckets->count; k++) { + for (k = i + 1; k <= bucket_count; k++) { if (hist_metrics[k] < hist_metrics[index]) { index = k; } @@ -151,16 +188,28 @@ static void pack_histogram_metric(mpack_writer_t *writer, struct cmt *cmt, mpack_write_cstr(writer, "Min"); mpack_write_double(writer, hist_metrics[0]); mpack_write_cstr(writer, "Max"); - mpack_write_double(writer, hist_metrics[buckets->count - 1]); + mpack_write_double(writer, hist_metrics[bucket_count - 1]); mpack_write_cstr(writer, "Sum"); - val = cmt_metric_hist_get_sum_value(metric); + if (map->type == CMT_HISTOGRAM) { + val = cmt_metric_hist_get_sum_value(metric); + } + else { + val = cmt_math_uint64_to_d64(metric->exp_hist_sum); + } mpack_write_double(writer, val); mpack_write_cstr(writer, "Count"); - val = cmt_metric_hist_get_count_value(metric); + if (map->type == CMT_HISTOGRAM) { + val = cmt_metric_hist_get_count_value(metric); + } + else { + val = metric->exp_hist_count; + } mpack_write_double(writer, val); mpack_finish_map(writer); free(hist_metrics); + free(exp_bucket_counts); + free(exp_upper_bounds); } static void pack_summary_metric(mpack_writer_t *writer, struct cmt *cmt, @@ -272,7 +321,7 @@ static int pack_metric(mpack_writer_t *writer, struct cmt *cmt, if (map->type == CMT_SUMMARY) { pack_summary_metric(writer, cmt, map, metric); } - else if (map->type == CMT_HISTOGRAM) { + else if (map->type == CMT_HISTOGRAM || map->type == CMT_EXP_HISTOGRAM) { pack_histogram_metric(writer, cmt, map, metric); } else { @@ -308,6 +357,7 @@ static size_t count_metrics(struct cmt *cmt) { size_t metric_count; struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; struct cmt_summary *summary; struct cmt_untyped *untyped; struct cmt_counter *counter; @@ -366,6 +416,16 @@ static size_t count_metrics(struct cmt *cmt) metric_count += cfl_list_size(&map->metrics); } + /* Exponential Histogram */ + cfl_list_foreach(head, &cmt->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + map = exp_histogram->map; + if (map->metric_static_set == 1) { + metric_count++; + } + metric_count += cfl_list_size(&map->metrics); + } + return metric_count; } @@ -373,6 +433,7 @@ static int pack_context_metrics(mpack_writer_t *writer, struct cmt *cmt, int wra { size_t metric_count; struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; struct cmt_summary *summary; struct cmt_untyped *untyped; struct cmt_counter *counter; @@ -414,6 +475,12 @@ static int pack_context_metrics(mpack_writer_t *writer, struct cmt *cmt, int wra pack_metrics(writer, cmt, histogram->map); } + /* Exponential Histogram */ + cfl_list_foreach(head, &cmt->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + pack_metrics(writer, cmt, exp_histogram->map); + } + if (wrap_array == CMT_TRUE) { mpack_finish_array(writer); /* outermost context scope */ } diff --git a/lib/cmetrics/src/cmt_encode_influx.c b/lib/cmetrics/src/cmt_encode_influx.c index 747355d3176..97d9c52c1c6 100644 --- a/lib/cmetrics/src/cmt_encode_influx.c +++ b/lib/cmetrics/src/cmt_encode_influx.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include @@ -159,10 +160,50 @@ static void append_metric_value(struct cmt_map *map, double val; char tmp[256]; struct cmt_opts *opts; + struct cmt_map fake_map; + struct cmt_metric fake_metric; + struct cmt_histogram fake_histogram; + struct cmt_histogram_buckets fake_buckets; + size_t bucket_count; + size_t upper_bounds_count; + uint64_t *bucket_values; + double *upper_bounds; if (map->type == CMT_HISTOGRAM) { return append_histogram_metric_value(map, buf, metric); } + else if (map->type == CMT_EXP_HISTOGRAM) { + if (cmt_exp_histogram_to_explicit(metric, + &upper_bounds, + &upper_bounds_count, + &bucket_values, + &bucket_count) == 0) { + memset(&fake_map, 0, sizeof(struct cmt_map)); + memset(&fake_metric, 0, sizeof(struct cmt_metric)); + memset(&fake_histogram, 0, sizeof(struct cmt_histogram)); + memset(&fake_buckets, 0, sizeof(struct cmt_histogram_buckets)); + + fake_buckets.count = upper_bounds_count; + fake_buckets.upper_bounds = upper_bounds; + fake_histogram.buckets = &fake_buckets; + + fake_map = *map; + fake_map.type = CMT_HISTOGRAM; + fake_map.parent = &fake_histogram; + + fake_metric = *metric; + fake_metric.hist_buckets = bucket_values; + fake_metric.hist_count = metric->exp_hist_count; + fake_metric.hist_sum = metric->exp_hist_sum; + + append_histogram_metric_value(&fake_map, buf, &fake_metric); + + free(bucket_values); + free(upper_bounds); + } + + return; + } else if (map->type == CMT_SUMMARY) { return append_summary_metric_value(map, buf, metric); } @@ -340,6 +381,7 @@ cfl_sds_t cmt_encode_influx_create(struct cmt *cmt) struct cmt_untyped *untyped; struct cmt_summary *summary; struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; /* Allocate a 1KB of buffer */ buf = cfl_sds_create_size(1024); @@ -371,6 +413,12 @@ cfl_sds_t cmt_encode_influx_create(struct cmt *cmt) format_metrics(cmt, &buf, histogram->map); } + /* Exponential Histograms */ + cfl_list_foreach(head, &cmt->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + format_metrics(cmt, &buf, exp_histogram->map); + } + /* Untyped */ cfl_list_foreach(head, &cmt->untypeds) { untyped = cfl_list_entry(head, struct cmt_untyped, _head); diff --git a/lib/cmetrics/src/cmt_encode_msgpack.c b/lib/cmetrics/src/cmt_encode_msgpack.c index b31a78e3b4f..f6cb07a17fd 100644 --- a/lib/cmetrics/src/cmt_encode_msgpack.c +++ b/lib/cmetrics/src/cmt_encode_msgpack.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -183,6 +184,13 @@ static int pack_metric(mpack_writer_t *writer, struct cmt_map *map, struct cmt_m s++; } + if (map->type != CMT_HISTOGRAM && + map->type != CMT_EXP_HISTOGRAM && + map->type != CMT_SUMMARY && + cmt_metric_get_value_type(metric) != CMT_METRIC_VALUE_DOUBLE) { + s += 2; + } + mpack_start_map(writer, s); mpack_write_cstr(writer, "ts"); @@ -212,6 +220,50 @@ static int pack_metric(mpack_writer_t *writer, struct cmt_map *map, struct cmt_m mpack_finish_map(writer); /* 'histogram' */ } + else if (map->type == CMT_EXP_HISTOGRAM) { + mpack_write_cstr(writer, "exp_histogram"); + mpack_start_map(writer, 10); + + mpack_write_cstr(writer, "scale"); + mpack_write_int(writer, metric->exp_hist_scale); + + mpack_write_cstr(writer, "zero_count"); + mpack_write_uint(writer, metric->exp_hist_zero_count); + + mpack_write_cstr(writer, "zero_threshold"); + mpack_write_double(writer, metric->exp_hist_zero_threshold); + + mpack_write_cstr(writer, "positive_offset"); + mpack_write_int(writer, metric->exp_hist_positive_offset); + + mpack_write_cstr(writer, "positive_buckets"); + mpack_start_array(writer, metric->exp_hist_positive_count); + for (index = 0 ; index < metric->exp_hist_positive_count ; index++) { + mpack_write_uint(writer, metric->exp_hist_positive_buckets[index]); + } + mpack_finish_array(writer); + + mpack_write_cstr(writer, "negative_offset"); + mpack_write_int(writer, metric->exp_hist_negative_offset); + + mpack_write_cstr(writer, "negative_buckets"); + mpack_start_array(writer, metric->exp_hist_negative_count); + for (index = 0 ; index < metric->exp_hist_negative_count ; index++) { + mpack_write_uint(writer, metric->exp_hist_negative_buckets[index]); + } + mpack_finish_array(writer); + + mpack_write_cstr(writer, "count"); + mpack_write_uint(writer, metric->exp_hist_count); + + mpack_write_cstr(writer, "sum_set"); + mpack_write_uint(writer, metric->exp_hist_sum_set); + + mpack_write_cstr(writer, "sum"); + mpack_write_uint(writer, metric->exp_hist_sum); + + mpack_finish_map(writer); /* 'exp_histogram' */ + } else if (map->type == CMT_SUMMARY) { summary = (struct cmt_summary *) map->parent; @@ -242,6 +294,19 @@ static int pack_metric(mpack_writer_t *writer, struct cmt_map *map, struct cmt_m mpack_write_cstr(writer, "value"); val = cmt_metric_get_value(metric); mpack_write_double(writer, val); + + if (cmt_metric_get_value_type(metric) == CMT_METRIC_VALUE_INT64) { + mpack_write_cstr(writer, "value_type"); + mpack_write_uint(writer, CMT_METRIC_VALUE_INT64); + mpack_write_cstr(writer, "value_int64"); + mpack_write_i64(writer, cmt_metric_get_int64_value(metric)); + } + else if (cmt_metric_get_value_type(metric) == CMT_METRIC_VALUE_UINT64) { + mpack_write_cstr(writer, "value_type"); + mpack_write_uint(writer, CMT_METRIC_VALUE_UINT64); + mpack_write_cstr(writer, "value_uint64"); + mpack_write_u64(writer, cmt_metric_get_uint64_value(metric)); + } } s = cfl_list_size(&metric->labels); @@ -374,6 +439,7 @@ static int pack_context_metrics(mpack_writer_t *writer, struct cmt *cmt) { size_t metric_count; struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; struct cmt_summary *summary; struct cmt_untyped *untyped; struct cmt_counter *counter; @@ -386,6 +452,7 @@ static int pack_context_metrics(mpack_writer_t *writer, struct cmt *cmt) metric_count += cfl_list_size(&cmt->untypeds); metric_count += cfl_list_size(&cmt->summaries); metric_count += cfl_list_size(&cmt->histograms); + metric_count += cfl_list_size(&cmt->exp_histograms); mpack_write_cstr(writer, "metrics"); mpack_start_array(writer, metric_count); @@ -420,6 +487,12 @@ static int pack_context_metrics(mpack_writer_t *writer, struct cmt *cmt) pack_basic_type(writer, cmt, histogram->map); } + /* Exponential Histogram */ + cfl_list_foreach(head, &cmt->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + pack_basic_type(writer, cmt, exp_histogram->map); + } + mpack_finish_array(writer); return 0; diff --git a/lib/cmetrics/src/cmt_encode_opentelemetry.c b/lib/cmetrics/src/cmt_encode_opentelemetry.c index f192b76f8ae..87957c84da3 100644 --- a/lib/cmetrics/src/cmt_encode_opentelemetry.c +++ b/lib/cmetrics/src/cmt_encode_opentelemetry.c @@ -24,7 +24,9 @@ #include #include #include +#include #include +#include static Opentelemetry__Proto__Metrics__V1__ScopeMetrics ** initialize_scope_metrics_list( @@ -54,6 +56,664 @@ struct cfl_kvlist *fetch_metadata_kvlist_key(struct cfl_kvlist *kvlist, char *ke return entry_kvlist; } +static struct cfl_array *fetch_metadata_array_key(struct cfl_kvlist *kvlist, char *key) +{ + struct cfl_variant *entry_variant; + + if (kvlist == NULL) { + return NULL; + } + + entry_variant = cfl_kvlist_fetch(kvlist, key); + if (entry_variant == NULL || entry_variant->type != CFL_VARIANT_ARRAY) { + return NULL; + } + + return entry_variant->data.as_array; +} + +static struct cfl_kvlist *fetch_array_kvlist_entry(struct cfl_array *array, size_t index) +{ + struct cfl_variant *entry_variant; + + if (array == NULL || index >= array->entry_count) { + return NULL; + } + + entry_variant = cfl_array_fetch_by_index(array, index); + if (entry_variant == NULL || entry_variant->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + return entry_variant->data.as_kvlist; +} + +static char *map_type_to_key(int map_type) +{ + switch (map_type) { + case CMT_COUNTER: + return "counter"; + case CMT_GAUGE: + return "gauge"; + case CMT_UNTYPED: + return "untyped"; + case CMT_SUMMARY: + return "summary"; + case CMT_HISTOGRAM: + return "histogram"; + case CMT_EXP_HISTOGRAM: + return "exp_histogram"; + default: + return "unknown"; + } +} + +static struct cfl_kvlist *get_metric_otlp_metadata_context(struct cmt *cmt, struct cmt_map *map) +{ + struct cfl_kvlist *otlp_root; + struct cfl_kvlist *metrics_root; + struct cfl_kvlist *type_root; + + if (cmt == NULL || map == NULL || map->opts == NULL || map->opts->fqname == NULL) { + return NULL; + } + + otlp_root = fetch_metadata_kvlist_key(cmt->external_metadata, "otlp"); + if (otlp_root == NULL) { + return NULL; + } + + metrics_root = fetch_metadata_kvlist_key(otlp_root, "metrics"); + if (metrics_root == NULL) { + return NULL; + } + + type_root = fetch_metadata_kvlist_key(metrics_root, map_type_to_key(map->type)); + if (type_root == NULL) { + return NULL; + } + + return fetch_metadata_kvlist_key(type_root, map->opts->fqname); +} + +static struct cfl_kvlist *get_data_point_otlp_metadata_context(struct cmt *cmt, + struct cmt_map *map, + struct cmt_metric *sample) +{ + char key[128]; + struct cfl_kvlist *metric_context; + struct cfl_kvlist *datapoints_context; + + metric_context = get_metric_otlp_metadata_context(cmt, map); + if (metric_context == NULL) { + return NULL; + } + + datapoints_context = fetch_metadata_kvlist_key(metric_context, "datapoints"); + if (datapoints_context == NULL) { + return NULL; + } + + snprintf(key, sizeof(key) - 1, "%" PRIx64 ":%" PRIu64, + sample != NULL ? sample->hash : 0, + sample != NULL ? cmt_metric_get_timestamp(sample) : 0); + + return fetch_metadata_kvlist_key(datapoints_context, key); +} + +static int kvlist_fetch_uint64(struct cfl_kvlist *kvlist, const char *key, uint64_t *out) +{ + struct cfl_variant *value; + + value = cfl_kvlist_fetch(kvlist, (char *) key); + if (value == NULL) { + return -1; + } + + if (value->type == CFL_VARIANT_UINT) { + *out = value->data.as_uint64; + return 0; + } + + if (value->type == CFL_VARIANT_INT && value->data.as_int64 >= 0) { + *out = (uint64_t) value->data.as_int64; + return 0; + } + + return -1; +} + +static int kvlist_fetch_double(struct cfl_kvlist *kvlist, const char *key, double *out) +{ + struct cfl_variant *value; + + value = cfl_kvlist_fetch(kvlist, (char *) key); + if (value == NULL) { + return -1; + } + + if (value->type == CFL_VARIANT_DOUBLE) { + *out = value->data.as_double; + return 0; + } + else if (value->type == CFL_VARIANT_UINT) { + *out = (double) value->data.as_uint64; + return 0; + } + else if (value->type == CFL_VARIANT_INT) { + *out = (double) value->data.as_int64; + return 0; + } + + return -1; +} + +static int kvlist_fetch_bool(struct cfl_kvlist *kvlist, const char *key, int *out) +{ + struct cfl_variant *value; + + value = cfl_kvlist_fetch(kvlist, (char *) key); + if (value == NULL) { + return -1; + } + + if (value->type == CFL_VARIANT_BOOL) { + *out = value->data.as_bool ? CMT_TRUE : CMT_FALSE; + return 0; + } + + if (value->type == CFL_VARIANT_UINT) { + *out = value->data.as_uint64 != 0 ? CMT_TRUE : CMT_FALSE; + return 0; + } + + if (value->type == CFL_VARIANT_INT) { + *out = value->data.as_int64 != 0 ? CMT_TRUE : CMT_FALSE; + return 0; + } + + return -1; +} + +static int kvlist_fetch_string(struct cfl_kvlist *kvlist, const char *key, char **out) +{ + struct cfl_variant *value; + + value = cfl_kvlist_fetch(kvlist, (char *) key); + if (value == NULL) { + return -1; + } + + if (value->type == CFL_VARIANT_STRING && value->data.as_string != NULL) { + *out = value->data.as_string; + return 0; + } + + return -1; +} + +static inline void otlp_kvpair_list_destroy(Opentelemetry__Proto__Common__V1__KeyValue **pair_list, + size_t entry_count); +static inline Opentelemetry__Proto__Common__V1__KeyValue **cfl_kvlist_to_otlp_kvpair_list( + struct cfl_kvlist *kvlist); + +static Opentelemetry__Proto__Metrics__V1__Exemplar *create_exemplar_from_kvlist( + struct cfl_kvlist *kvlist) +{ + int result; + size_t index; + uint64_t uint64_value; + Opentelemetry__Proto__Metrics__V1__Exemplar *exemplar; + struct cfl_variant *variant; + struct cfl_kvlist *filtered_attributes; + + exemplar = calloc(1, sizeof(Opentelemetry__Proto__Metrics__V1__Exemplar)); + if (exemplar == NULL) { + return NULL; + } + + opentelemetry__proto__metrics__v1__exemplar__init(exemplar); + + result = kvlist_fetch_uint64(kvlist, "time_unix_nano", &uint64_value); + if (result == 0) { + exemplar->time_unix_nano = uint64_value; + } + + variant = cfl_kvlist_fetch(kvlist, "span_id"); + if (variant != NULL && variant->type == CFL_VARIANT_BYTES) { + exemplar->span_id.len = cfl_sds_len(variant->data.as_bytes); + exemplar->span_id.data = calloc(1, exemplar->span_id.len); + if (exemplar->span_id.data == NULL) { + free(exemplar); + return NULL; + } + memcpy(exemplar->span_id.data, variant->data.as_bytes, exemplar->span_id.len); + } + + variant = cfl_kvlist_fetch(kvlist, "trace_id"); + if (variant != NULL && variant->type == CFL_VARIANT_BYTES) { + exemplar->trace_id.len = cfl_sds_len(variant->data.as_bytes); + exemplar->trace_id.data = calloc(1, exemplar->trace_id.len); + if (exemplar->trace_id.data == NULL) { + if (exemplar->span_id.data != NULL) { + free(exemplar->span_id.data); + } + free(exemplar); + return NULL; + } + memcpy(exemplar->trace_id.data, variant->data.as_bytes, exemplar->trace_id.len); + } + + variant = cfl_kvlist_fetch(kvlist, "as_double"); + if (variant != NULL) { + if (variant->type == CFL_VARIANT_DOUBLE) { + exemplar->value_case = OPENTELEMETRY__PROTO__METRICS__V1__EXEMPLAR__VALUE_AS_DOUBLE; + exemplar->as_double = variant->data.as_double; + } + } + else { + variant = cfl_kvlist_fetch(kvlist, "as_int"); + if (variant != NULL) { + if (variant->type == CFL_VARIANT_INT) { + exemplar->value_case = OPENTELEMETRY__PROTO__METRICS__V1__EXEMPLAR__VALUE_AS_INT; + exemplar->as_int = variant->data.as_int64; + } + else if (variant->type == CFL_VARIANT_UINT) { + exemplar->value_case = OPENTELEMETRY__PROTO__METRICS__V1__EXEMPLAR__VALUE_AS_INT; + exemplar->as_int = (int64_t) variant->data.as_uint64; + } + } + } + + filtered_attributes = fetch_metadata_kvlist_key(kvlist, "filtered_attributes"); + if (filtered_attributes != NULL && cfl_kvlist_count(filtered_attributes) > 0) { + exemplar->filtered_attributes = cfl_kvlist_to_otlp_kvpair_list(filtered_attributes); + if (exemplar->filtered_attributes == NULL) { + if (exemplar->span_id.data != NULL) { + free(exemplar->span_id.data); + } + if (exemplar->trace_id.data != NULL) { + free(exemplar->trace_id.data); + } + free(exemplar); + return NULL; + } + + exemplar->n_filtered_attributes = cfl_kvlist_count(filtered_attributes); + + for (index = 0; index < exemplar->n_filtered_attributes; index++) { + if (exemplar->filtered_attributes[index] == NULL) { + otlp_kvpair_list_destroy(exemplar->filtered_attributes, exemplar->n_filtered_attributes); + if (exemplar->span_id.data != NULL) { + free(exemplar->span_id.data); + } + if (exemplar->trace_id.data != NULL) { + free(exemplar->trace_id.data); + } + free(exemplar); + return NULL; + } + } + } + + return exemplar; +} + +static void destroy_exemplar(Opentelemetry__Proto__Metrics__V1__Exemplar *exemplar) +{ + if (exemplar == NULL) { + return; + } + + if (exemplar->span_id.data != NULL) { + free(exemplar->span_id.data); + } + + if (exemplar->trace_id.data != NULL) { + free(exemplar->trace_id.data); + } + + if (exemplar->filtered_attributes != NULL) { + otlp_kvpair_list_destroy(exemplar->filtered_attributes, exemplar->n_filtered_attributes); + } + + free(exemplar); +} + +static int initialize_exemplars_from_metadata(struct cfl_kvlist *point_metadata, + Opentelemetry__Proto__Metrics__V1__Exemplar ***out_exemplars, + size_t *out_count) +{ + size_t index; + size_t dest_index; + struct cfl_array *array; + struct cfl_variant *variant; + Opentelemetry__Proto__Metrics__V1__Exemplar **exemplars; + + *out_exemplars = NULL; + *out_count = 0; + + variant = cfl_kvlist_fetch(point_metadata, "exemplars"); + if (variant == NULL || variant->type != CFL_VARIANT_ARRAY || variant->data.as_array == NULL) { + return 0; + } + + array = variant->data.as_array; + if (array->entry_count == 0) { + return 0; + } + + exemplars = calloc(array->entry_count, sizeof(Opentelemetry__Proto__Metrics__V1__Exemplar *)); + if (exemplars == NULL) { + return -1; + } + + dest_index = 0; + + for (index = 0; index < array->entry_count; index++) { + variant = cfl_array_fetch_by_index(array, index); + if (variant == NULL || variant->type != CFL_VARIANT_KVLIST || variant->data.as_kvlist == NULL) { + continue; + } + + exemplars[dest_index] = create_exemplar_from_kvlist(variant->data.as_kvlist); + if (exemplars[dest_index] == NULL) { + while (dest_index > 0) { + dest_index--; + if (exemplars[dest_index] != NULL) { + destroy_exemplar(exemplars[dest_index]); + } + } + free(exemplars); + return -1; + } + + dest_index++; + } + + if (dest_index == 0) { + free(exemplars); + return 0; + } + + *out_exemplars = exemplars; + *out_count = dest_index; + + return 0; +} + +static size_t compute_flat_scope_index(struct cmt_opentelemetry_context *context, + size_t resource_index, + size_t scope_index) +{ + size_t index; + size_t offset; + + offset = 0; + + if (context == NULL || context->metrics_data == NULL) { + return 0; + } + + if (resource_index >= context->metrics_data->n_resource_metrics) { + return 0; + } + + for (index = 0; index < resource_index; index++) { + offset += context->metrics_data->resource_metrics[index]->n_scope_metrics; + } + + if (scope_index >= context->metrics_data->resource_metrics[resource_index]->n_scope_metrics) { + return 0; + } + + return offset + scope_index; +} + +static size_t resolve_target_scope_index(struct cmt_opentelemetry_context *context, + struct cmt_map *map) +{ + uint64_t flat_scope_index; + uint64_t resource_index; + uint64_t scope_index; + int result; + struct cfl_kvlist *metric_context; + struct cfl_kvlist *metadata; + + if (context == NULL || context->scope_metrics_count == 0) { + return 0; + } + + metric_context = get_metric_otlp_metadata_context(context->cmt, map); + if (metric_context == NULL) { + return 0; + } + + metadata = fetch_metadata_kvlist_key(metric_context, "metadata"); + if (metadata == NULL) { + return 0; + } + + result = kvlist_fetch_uint64(metadata, "scope_flat_index", &flat_scope_index); + if (result == 0 && flat_scope_index < context->scope_metrics_count) { + return (size_t) flat_scope_index; + } + + result = kvlist_fetch_uint64(metadata, "resource_index", &resource_index); + if (result != 0) { + return 0; + } + + result = kvlist_fetch_uint64(metadata, "scope_index", &scope_index); + if (result != 0) { + return 0; + } + + flat_scope_index = compute_flat_scope_index(context, (size_t) resource_index, (size_t) scope_index); + if (flat_scope_index < context->scope_metrics_count) { + return (size_t) flat_scope_index; + } + + return 0; +} + +static void apply_metric_metadata_from_otlp_context(struct cmt *cmt, + struct cmt_map *map, + Opentelemetry__Proto__Metrics__V1__Metric *metric) +{ + struct cfl_kvlist *metric_context; + struct cfl_kvlist *metadata; + + metric_context = get_metric_otlp_metadata_context(cmt, map); + if (metric_context == NULL) { + return; + } + + metadata = fetch_metadata_kvlist_key(metric_context, "metadata"); + if (metadata == NULL || cfl_kvlist_count(metadata) == 0) { + return; + } + + metric->metadata = cfl_kvlist_to_otlp_kvpair_list(metadata); + if (metric->metadata != NULL) { + metric->n_metadata = cfl_kvlist_count(metadata); + } +} + +static void apply_data_point_metadata_from_otlp_context(struct cmt *cmt, + struct cmt_map *map, + struct cmt_metric *sample, + void *data_point) +{ + int bool_value; + int result; + uint64_t uint64_value; + double double_value; + size_t exemplar_count; + Opentelemetry__Proto__Metrics__V1__Exemplar **exemplars; + struct cfl_kvlist *point_metadata; + + point_metadata = get_data_point_otlp_metadata_context(cmt, map, sample); + if (point_metadata == NULL || data_point == NULL) { + return; + } + + if (map->type == CMT_COUNTER || + map->type == CMT_GAUGE || + map->type == CMT_UNTYPED) { + Opentelemetry__Proto__Metrics__V1__NumberDataPoint *number_data_point; + char *number_value_case; + + number_data_point = (Opentelemetry__Proto__Metrics__V1__NumberDataPoint *) data_point; + number_value_case = NULL; + + result = kvlist_fetch_uint64(point_metadata, "start_time_unix_nano", &uint64_value); + if (result == 0) { + number_data_point->start_time_unix_nano = uint64_value; + } + + result = kvlist_fetch_uint64(point_metadata, "flags", &uint64_value); + if (result == 0) { + number_data_point->flags = (uint32_t) uint64_value; + } + + if (cmt_metric_get_value_type(sample) == CMT_METRIC_VALUE_INT64) { + number_data_point->value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT; + number_data_point->as_int = cmt_metric_get_int64_value(sample); + } + else if (cmt_metric_get_value_type(sample) == CMT_METRIC_VALUE_UINT64) { + if (cmt_metric_get_uint64_value(sample) <= INT64_MAX) { + number_data_point->value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT; + number_data_point->as_int = (int64_t) cmt_metric_get_uint64_value(sample); + } + else { + number_data_point->value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_DOUBLE; + number_data_point->as_double = (double) cmt_metric_get_uint64_value(sample); + } + } + else { + result = kvlist_fetch_string(point_metadata, "number_value_case", &number_value_case); + if (result == 0 && strcmp(number_value_case, "int") == 0) { + number_data_point->value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT; + number_data_point->as_int = cmt_metric_get_int64_value(sample); + } + else { + number_data_point->value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_DOUBLE; + number_data_point->as_double = cmt_metric_get_value(sample); + } + } + + result = initialize_exemplars_from_metadata(point_metadata, &exemplars, &exemplar_count); + if (result == 0 && exemplars != NULL) { + number_data_point->exemplars = exemplars; + number_data_point->n_exemplars = exemplar_count; + } + } + else if (map->type == CMT_SUMMARY) { + Opentelemetry__Proto__Metrics__V1__SummaryDataPoint *summary_data_point; + + summary_data_point = (Opentelemetry__Proto__Metrics__V1__SummaryDataPoint *) data_point; + + result = kvlist_fetch_uint64(point_metadata, "start_time_unix_nano", &uint64_value); + if (result == 0) { + summary_data_point->start_time_unix_nano = uint64_value; + } + + result = kvlist_fetch_uint64(point_metadata, "flags", &uint64_value); + if (result == 0) { + summary_data_point->flags = (uint32_t) uint64_value; + } + } + else if (map->type == CMT_HISTOGRAM) { + Opentelemetry__Proto__Metrics__V1__HistogramDataPoint *histogram_data_point; + + histogram_data_point = (Opentelemetry__Proto__Metrics__V1__HistogramDataPoint *) data_point; + + result = kvlist_fetch_uint64(point_metadata, "start_time_unix_nano", &uint64_value); + if (result == 0) { + histogram_data_point->start_time_unix_nano = uint64_value; + } + + result = kvlist_fetch_uint64(point_metadata, "flags", &uint64_value); + if (result == 0) { + histogram_data_point->flags = (uint32_t) uint64_value; + } + + result = kvlist_fetch_bool(point_metadata, "has_sum", &bool_value); + if (result == 0 && bool_value == CMT_FALSE) { + histogram_data_point->has_sum = CMT_FALSE; + } + + result = kvlist_fetch_bool(point_metadata, "has_min", &bool_value); + if (result == 0 && bool_value == CMT_TRUE) { + result = kvlist_fetch_double(point_metadata, "min", &double_value); + if (result == 0) { + histogram_data_point->has_min = CMT_TRUE; + histogram_data_point->min = double_value; + } + } + + result = kvlist_fetch_bool(point_metadata, "has_max", &bool_value); + if (result == 0 && bool_value == CMT_TRUE) { + result = kvlist_fetch_double(point_metadata, "max", &double_value); + if (result == 0) { + histogram_data_point->has_max = CMT_TRUE; + histogram_data_point->max = double_value; + } + } + + result = initialize_exemplars_from_metadata(point_metadata, &exemplars, &exemplar_count); + if (result == 0 && exemplars != NULL) { + histogram_data_point->exemplars = exemplars; + histogram_data_point->n_exemplars = exemplar_count; + } + } + else if (map->type == CMT_EXP_HISTOGRAM) { + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint *exp_data_point; + + exp_data_point = (Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint *) data_point; + + result = kvlist_fetch_uint64(point_metadata, "start_time_unix_nano", &uint64_value); + if (result == 0) { + exp_data_point->start_time_unix_nano = uint64_value; + } + + result = kvlist_fetch_uint64(point_metadata, "flags", &uint64_value); + if (result == 0) { + exp_data_point->flags = (uint32_t) uint64_value; + } + + result = kvlist_fetch_bool(point_metadata, "has_sum", &bool_value); + if (result == 0 && bool_value == CMT_FALSE) { + exp_data_point->has_sum = CMT_FALSE; + } + + result = kvlist_fetch_bool(point_metadata, "has_min", &bool_value); + if (result == 0 && bool_value == CMT_TRUE) { + result = kvlist_fetch_double(point_metadata, "min", &double_value); + if (result == 0) { + exp_data_point->has_min = CMT_TRUE; + exp_data_point->min = double_value; + } + } + + result = kvlist_fetch_bool(point_metadata, "has_max", &bool_value); + if (result == 0 && bool_value == CMT_TRUE) { + result = kvlist_fetch_double(point_metadata, "max", &double_value); + if (result == 0) { + exp_data_point->has_max = CMT_TRUE; + exp_data_point->max = double_value; + } + } + + result = initialize_exemplars_from_metadata(point_metadata, &exemplars, &exemplar_count); + if (result == 0 && exemplars != NULL) { + exp_data_point->exemplars = exemplars; + exp_data_point->n_exemplars = exemplar_count; + } + } +} + static int is_string_releaseable(char *address); @@ -117,7 +777,7 @@ static Opentelemetry__Proto__Metrics__V1__ScopeMetrics * static int append_metric_to_scope_metrics( Opentelemetry__Proto__Metrics__V1__ScopeMetrics *instrumentation_scope_metrics, Opentelemetry__Proto__Metrics__V1__Metric *metric, - size_t metric_slot_hint); + size_t metric_slot_limit); static void destroy_scope_metric_list( Opentelemetry__Proto__Metrics__V1__ScopeMetrics **metric_list); @@ -144,6 +804,9 @@ static void destroy_summary_data_point( static void destroy_histogram_data_point( Opentelemetry__Proto__Metrics__V1__HistogramDataPoint *data_point); +static void destroy_exponential_histogram_data_point( + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint *data_point); + static void destroy_data_point( void *data_point, int data_point_type); @@ -157,11 +820,14 @@ static void destroy_summary_data_point_list( static void destroy_histogram_data_point_list( Opentelemetry__Proto__Metrics__V1__HistogramDataPoint **data_point_list); +static void destroy_exponential_histogram_data_point_list( + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint **data_point_list); + static Opentelemetry__Proto__Metrics__V1__NumberDataPoint * initialize_numerical_data_point( uint64_t start_time, uint64_t timestamp, - double value, + struct cmt_metric *sample, Opentelemetry__Proto__Common__V1__KeyValue **attribute_list, size_t attribute_count); @@ -226,6 +892,10 @@ static Opentelemetry__Proto__Metrics__V1__HistogramDataPoint ** initialize_histogram_data_point_list( size_t element_count); +static Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint ** + initialize_exponential_histogram_data_point_list( + size_t element_count); + static void destroy_metric( Opentelemetry__Proto__Metrics__V1__Metric *metric); @@ -822,6 +1492,7 @@ static size_t get_metric_count(struct cmt *cmt) { size_t metric_count; struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; struct cmt_summary *summary; struct cmt_untyped *untyped; struct cmt_counter *counter; @@ -860,6 +1531,12 @@ static size_t get_metric_count(struct cmt *cmt) metric_count += !is_metric_empty(histogram->map); } + cfl_list_foreach(head, &cmt->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + + metric_count += !is_metric_empty(exp_histogram->map); + } + return metric_count; } @@ -1135,9 +1812,6 @@ static Opentelemetry__Proto__Metrics__V1__ScopeMetrics *initialize_scope_metrics if (scope_metrics->metrics == NULL) { error_detection_flag = CMT_TRUE; } - else { - scope_metrics->n_metrics = metric_element_count; - } } if (!error_detection_flag && metadata != NULL) { @@ -1156,15 +1830,18 @@ static Opentelemetry__Proto__Metrics__V1__ScopeMetrics *initialize_scope_metrics static int append_metric_to_scope_metrics( Opentelemetry__Proto__Metrics__V1__ScopeMetrics *metrics, Opentelemetry__Proto__Metrics__V1__Metric *metric, - size_t metric_slot_hint) + size_t metric_slot_limit) { size_t metric_slot_index; - for (metric_slot_index = metric_slot_hint ; - metric_slot_index < metrics->n_metrics; + for (metric_slot_index = 0 ; + metric_slot_index < metric_slot_limit; metric_slot_index++) { if (metrics->metrics[metric_slot_index] == NULL) { metrics->metrics[metric_slot_index] = metric; + if (metric_slot_index + 1 > metrics->n_metrics) { + metrics->n_metrics = metric_slot_index + 1; + } return CMT_ENCODE_OPENTELEMETRY_SUCCESS; } @@ -1302,9 +1979,21 @@ static Opentelemetry__Proto__Common__V1__KeyValue ** static void destroy_numerical_data_point( Opentelemetry__Proto__Metrics__V1__NumberDataPoint *data_point) { + size_t index; + if (data_point != NULL) { destroy_attribute_list(data_point->attributes); + if (data_point->exemplars != NULL) { + for (index = 0; index < data_point->n_exemplars; index++) { + if (data_point->exemplars[index] != NULL) { + destroy_exemplar(data_point->exemplars[index]); + } + } + + free(data_point->exemplars); + } + free(data_point); } } @@ -1326,6 +2015,8 @@ static void destroy_summary_data_point( static void destroy_histogram_data_point( Opentelemetry__Proto__Metrics__V1__HistogramDataPoint *data_point) { + size_t index; + if (data_point != NULL) { destroy_attribute_list(data_point->attributes); @@ -1337,6 +2028,52 @@ static void destroy_histogram_data_point( free(data_point->explicit_bounds); } + if (data_point->exemplars != NULL) { + for (index = 0; index < data_point->n_exemplars; index++) { + if (data_point->exemplars[index] != NULL) { + destroy_exemplar(data_point->exemplars[index]); + } + } + + free(data_point->exemplars); + } + + free(data_point); + } +} + +static void destroy_exponential_histogram_data_point( + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint *data_point) +{ + size_t index; + + if (data_point != NULL) { + destroy_attribute_list(data_point->attributes); + + if (data_point->positive != NULL) { + if (data_point->positive->bucket_counts != NULL) { + free(data_point->positive->bucket_counts); + } + free(data_point->positive); + } + + if (data_point->negative != NULL) { + if (data_point->negative->bucket_counts != NULL) { + free(data_point->negative->bucket_counts); + } + free(data_point->negative); + } + + if (data_point->exemplars != NULL) { + for (index = 0; index < data_point->n_exemplars; index++) { + if (data_point->exemplars[index] != NULL) { + destroy_exemplar(data_point->exemplars[index]); + } + } + + free(data_point->exemplars); + } + free(data_point); } } @@ -1354,6 +2091,8 @@ static void destroy_data_point( return destroy_summary_data_point(data_point); case CMT_HISTOGRAM: return destroy_histogram_data_point(data_point); + case CMT_EXP_HISTOGRAM: + return destroy_exponential_histogram_data_point(data_point); } } @@ -1411,11 +2150,28 @@ static void destroy_histogram_data_point_list( } } +static void destroy_exponential_histogram_data_point_list( + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint **data_point_list) +{ + size_t element_index; + + if (data_point_list != NULL) { + for (element_index = 0 ; + data_point_list[element_index] != NULL ; + element_index++) { + destroy_exponential_histogram_data_point(data_point_list[element_index]); + data_point_list[element_index] = NULL; + } + + free(data_point_list); + } +} + static Opentelemetry__Proto__Metrics__V1__NumberDataPoint * initialize_numerical_data_point( uint64_t start_time, uint64_t timestamp, - double value, + struct cmt_metric *sample, Opentelemetry__Proto__Common__V1__KeyValue **attribute_list, size_t attribute_count) { @@ -1432,8 +2188,24 @@ static Opentelemetry__Proto__Metrics__V1__NumberDataPoint * data_point->start_time_unix_nano = start_time; data_point->time_unix_nano = timestamp; - data_point->value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_DOUBLE; - data_point->as_double = value; + if (sample != NULL && cmt_metric_get_value_type(sample) == CMT_METRIC_VALUE_INT64) { + data_point->value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT; + data_point->as_int = cmt_metric_get_int64_value(sample); + } + else if (sample != NULL && cmt_metric_get_value_type(sample) == CMT_METRIC_VALUE_UINT64) { + if (cmt_metric_get_uint64_value(sample) <= INT64_MAX) { + data_point->value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT; + data_point->as_int = (int64_t) cmt_metric_get_uint64_value(sample); + } + else { + data_point->value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_DOUBLE; + data_point->as_double = (double) cmt_metric_get_uint64_value(sample); + } + } + else { + data_point->value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_DOUBLE; + data_point->as_double = sample != NULL ? cmt_metric_get_value(sample) : 0; + } data_point->attributes = attribute_list; data_point->n_attributes = attribute_count; @@ -1713,6 +2485,26 @@ static int append_attribute_to_histogram_data_point( return CMT_ENCODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR; } +static int append_attribute_to_exponential_histogram_data_point( + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint *data_point, + Opentelemetry__Proto__Common__V1__KeyValue *attribute, + size_t attribute_slot_hint) +{ + size_t attribute_slot_index; + + for (attribute_slot_index = attribute_slot_hint ; + attribute_slot_index < data_point->n_attributes; + attribute_slot_index++) { + if (data_point->attributes[attribute_slot_index] == NULL) { + data_point->attributes[attribute_slot_index] = attribute; + + return CMT_ENCODE_OPENTELEMETRY_SUCCESS; + } + } + + return CMT_ENCODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR; +} + static int append_attribute_to_data_point( void *data_point, int data_point_type, @@ -1734,6 +2526,10 @@ static int append_attribute_to_data_point( return append_attribute_to_histogram_data_point(data_point, attribute, attribute_slot_hint); + case CMT_EXP_HISTOGRAM: + return append_attribute_to_exponential_histogram_data_point(data_point, + attribute, + attribute_slot_hint); } return CMT_ENCODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR; @@ -1779,6 +2575,22 @@ static Opentelemetry__Proto__Metrics__V1__HistogramDataPoint ** return data_point_list; } +static Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint ** + initialize_exponential_histogram_data_point_list( + size_t element_count) +{ + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint **data_point_list; + + data_point_list = calloc(element_count + 1, + sizeof(Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint *)); + + if (data_point_list == NULL) { + return NULL; + } + + return data_point_list; +} + static void destroy_metric( Opentelemetry__Proto__Metrics__V1__Metric *metric) { @@ -1798,6 +2610,12 @@ static void destroy_metric( metric->unit = NULL; } + if (metric->metadata != NULL) { + otlp_kvpair_list_destroy(metric->metadata, metric->n_metadata); + metric->metadata = NULL; + metric->n_metadata = 0; + } + if (metric->data_case == OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_SUM) { destroy_numerical_data_point_list(metric->sum->data_points); @@ -1811,13 +2629,18 @@ static void destroy_metric( else if (metric->data_case == OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_SUMMARY) { destroy_summary_data_point_list(metric->summary->data_points); - free(metric->histogram); + free(metric->summary); } else if (metric->data_case == OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_HISTOGRAM) { destroy_histogram_data_point_list(metric->histogram->data_points); free(metric->histogram); } + else if (metric->data_case == OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_EXPONENTIAL_HISTOGRAM) { + destroy_exponential_histogram_data_point_list(metric->exponential_histogram->data_points); + + free(metric->exponential_histogram); + } free(metric); } @@ -1985,6 +2808,29 @@ static Opentelemetry__Proto__Metrics__V1__Metric * metric->histogram->n_data_points = data_point_count; metric->histogram->aggregation_temporality = aggregation_temporality_type; } + else if (type == CMT_EXP_HISTOGRAM) { + metric->exponential_histogram = calloc(1, sizeof(Opentelemetry__Proto__Metrics__V1__ExponentialHistogram)); + + if (metric->exponential_histogram == NULL) { + destroy_metric(metric); + + return NULL; + } + + opentelemetry__proto__metrics__v1__exponential_histogram__init(metric->exponential_histogram); + + metric->data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_EXPONENTIAL_HISTOGRAM; + metric->exponential_histogram->data_points = initialize_exponential_histogram_data_point_list(data_point_count); + + if (metric->exponential_histogram->data_points == NULL) { + destroy_metric(metric); + + return NULL; + } + + metric->exponential_histogram->n_data_points = data_point_count; + metric->exponential_histogram->aggregation_temporality = aggregation_temporality_type; + } return metric; } @@ -2018,6 +2864,10 @@ static int append_data_point_to_metric( data_point_list = (void **) metric->histogram->data_points; data_point_slot_count = metric->histogram->n_data_points; } + else if (metric->data_case == OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_EXPONENTIAL_HISTOGRAM) { + data_point_list = (void **) metric->exponential_histogram->data_points; + data_point_slot_count = metric->exponential_histogram->n_data_points; + } } for (data_point_slot_index = data_point_slot_hint ; @@ -2071,6 +2921,10 @@ static void destroy_opentelemetry_context( struct cmt_opentelemetry_context *context) { if (context != NULL) { + if (context->scope_metrics_list != NULL) { + free(context->scope_metrics_list); + } + if (context->metrics_data != NULL) { destroy_metrics_data(context->metrics_data); } @@ -2135,20 +2989,32 @@ static Opentelemetry__Proto__Resource__V1__Resource *initialize_resource(struct static struct cmt_opentelemetry_context *initialize_opentelemetry_context( struct cmt *cmt) { + struct cfl_array *resource_metrics_list; + struct cfl_array *scope_metrics_list; + struct cfl_kvlist *resource_entry; + struct cfl_kvlist *scope_entry; struct cfl_kvlist *resource_metrics_root; struct cfl_kvlist *scope_metrics_root; struct cfl_kvlist *resource_root; struct cfl_kvlist *scope_root; Opentelemetry__Proto__Resource__V1__Resource *resource; struct cmt_opentelemetry_context *context; + size_t resource_count; + size_t scope_count; + size_t total_scope_count; + size_t resource_index; + size_t scope_index; + size_t flat_scope_index; int result; resource_metrics_root = fetch_metadata_kvlist_key(cmt->external_metadata, "resource_metrics"); resource_root = fetch_metadata_kvlist_key(cmt->external_metadata, "resource"); scope_metrics_root = fetch_metadata_kvlist_key(cmt->external_metadata, "scope_metrics"); scope_root = fetch_metadata_kvlist_key(cmt->external_metadata, "scope"); + resource_metrics_list = fetch_metadata_array_key(cmt->external_metadata, "resource_metrics_list"); result = CMT_ENCODE_OPENTELEMETRY_SUCCESS; + total_scope_count = 0; context = calloc(1, sizeof(struct cmt_opentelemetry_context)); @@ -2162,17 +3028,14 @@ static struct cmt_opentelemetry_context *initialize_opentelemetry_context( context->cmt = cmt; - resource = initialize_resource(resource_root, &result); - - if (resource == NULL) { - if (result) { - result = CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; - - goto cleanup; - } + if (resource_metrics_list != NULL && resource_metrics_list->entry_count > 0) { + resource_count = resource_metrics_list->entry_count; + } + else { + resource_count = 1; } - context->metrics_data = initialize_metrics_data(1); + context->metrics_data = initialize_metrics_data(resource_count); if (context->metrics_data == NULL) { result = CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; @@ -2180,38 +3043,100 @@ static struct cmt_opentelemetry_context *initialize_opentelemetry_context( goto cleanup; } - context->metrics_data->resource_metrics[0] = \ - initialize_resource_metrics(resource_metrics_root, resource, 1); + for (resource_index = 0; resource_index < resource_count; resource_index++) { + resource_entry = NULL; + scope_metrics_list = NULL; - if (context->metrics_data->resource_metrics[0] == NULL) { - result = CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + if (resource_metrics_list != NULL && resource_metrics_list->entry_count > 0) { + resource_entry = fetch_array_kvlist_entry(resource_metrics_list, resource_index); + } - goto cleanup; - } + if (resource_entry != NULL) { + resource_metrics_root = fetch_metadata_kvlist_key(resource_entry, "resource_metrics"); + resource_root = fetch_metadata_kvlist_key(resource_entry, "resource"); + scope_metrics_root = fetch_metadata_kvlist_key(resource_entry, "scope_metrics"); + scope_root = fetch_metadata_kvlist_key(resource_entry, "scope"); + scope_metrics_list = fetch_metadata_array_key(resource_entry, "scope_metrics_list"); + } + else { + resource_metrics_root = fetch_metadata_kvlist_key(cmt->external_metadata, "resource_metrics"); + resource_root = fetch_metadata_kvlist_key(cmt->external_metadata, "resource"); + scope_metrics_root = fetch_metadata_kvlist_key(cmt->external_metadata, "scope_metrics"); + scope_root = fetch_metadata_kvlist_key(cmt->external_metadata, "scope"); + } - context->metrics_data->resource_metrics[0]->scope_metrics[0] = \ - initialize_scope_metrics(scope_metrics_root, - get_metric_count(cmt)); + if (scope_metrics_list != NULL && scope_metrics_list->entry_count > 0) { + scope_count = scope_metrics_list->entry_count; + } + else { + scope_count = 1; + } - if (context->metrics_data->resource_metrics[0]->scope_metrics[0] == NULL) { - result = CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + total_scope_count += scope_count; - goto cleanup; - } + resource = initialize_resource(resource_root, &result); + + if (resource == NULL && result) { + result = CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + goto cleanup; + } - context->metrics_data->\ - resource_metrics[0]->\ - scope_metrics[0]->\ - scope = \ - initialize_instrumentation_scope( - scope_root, &result); + context->metrics_data->resource_metrics[resource_index] = + initialize_resource_metrics(resource_metrics_root, resource, scope_count); - if (result) { - result = CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + if (context->metrics_data->resource_metrics[resource_index] == NULL) { + result = CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + goto cleanup; + } + + for (scope_index = 0; scope_index < scope_count; scope_index++) { + scope_entry = NULL; + + if (scope_metrics_list != NULL && scope_metrics_list->entry_count > 0) { + scope_entry = fetch_array_kvlist_entry(scope_metrics_list, scope_index); + } + + if (scope_entry != NULL) { + scope_metrics_root = fetch_metadata_kvlist_key(scope_entry, "scope_metrics"); + scope_root = fetch_metadata_kvlist_key(scope_entry, "scope"); + } + + context->metrics_data->resource_metrics[resource_index]->scope_metrics[scope_index] = + initialize_scope_metrics(scope_metrics_root, get_metric_count(cmt)); + + if (context->metrics_data->resource_metrics[resource_index]->scope_metrics[scope_index] == NULL) { + result = CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + goto cleanup; + } + + context->metrics_data->resource_metrics[resource_index]->scope_metrics[scope_index]->scope = + initialize_instrumentation_scope(scope_root, &result); + + if (result) { + result = CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; + goto cleanup; + } + } + } + context->scope_metrics_list = + calloc(total_scope_count, sizeof(Opentelemetry__Proto__Metrics__V1__ScopeMetrics *)); + if (context->scope_metrics_list == NULL) { + result = CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; goto cleanup; } + context->scope_metrics_count = total_scope_count; + flat_scope_index = 0; + for (resource_index = 0; resource_index < resource_count; resource_index++) { + for (scope_index = 0; + scope_index < context->metrics_data->resource_metrics[resource_index]->n_scope_metrics; + scope_index++) { + context->scope_metrics_list[flat_scope_index++] = + context->metrics_data->resource_metrics[resource_index]->scope_metrics[scope_index]; + } + } + cleanup: if (result != CMT_ENCODE_OPENTELEMETRY_SUCCESS) { if (context != NULL) { @@ -2239,6 +3164,9 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, void *data_point = NULL; Opentelemetry__Proto__Common__V1__KeyValue *attribute; struct cmt_histogram *histogram; + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint *exponential_data_point; + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets *positive; + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets *negative; struct cmt_summary *summary; int result; struct cfl_list *head; @@ -2257,7 +3185,7 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, map->type == CMT_UNTYPED ) { data_point = initialize_numerical_data_point(0, cmt_metric_get_timestamp(sample), - cmt_metric_get_value(sample), + sample, attribute_list, attribute_count); } @@ -2289,6 +3217,73 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, attribute_list, attribute_count); } + else if (map->type == CMT_EXP_HISTOGRAM) { + exponential_data_point = calloc(1, sizeof(Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint)); + if (exponential_data_point == NULL) { + destroy_attribute_list(attribute_list); + return CMT_ENCODE_OPENTELEMETRY_DATA_POINT_INIT_ERROR; + } + + opentelemetry__proto__metrics__v1__exponential_histogram_data_point__init(exponential_data_point); + + exponential_data_point->start_time_unix_nano = 0; + exponential_data_point->time_unix_nano = cmt_metric_get_timestamp(sample); + exponential_data_point->count = sample->exp_hist_count; + exponential_data_point->scale = sample->exp_hist_scale; + exponential_data_point->zero_count = sample->exp_hist_zero_count; + exponential_data_point->zero_threshold = sample->exp_hist_zero_threshold; + exponential_data_point->attributes = attribute_list; + exponential_data_point->n_attributes = attribute_count; + + if (sample->exp_hist_sum_set) { + exponential_data_point->has_sum = CMT_TRUE; + exponential_data_point->sum = cmt_math_uint64_to_d64(sample->exp_hist_sum); + } + + if (sample->exp_hist_positive_count > 0) { + positive = calloc(1, sizeof(Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets)); + if (positive == NULL) { + destroy_data_point(exponential_data_point, map->type); + return CMT_ENCODE_OPENTELEMETRY_DATA_POINT_INIT_ERROR; + } + + opentelemetry__proto__metrics__v1__exponential_histogram_data_point__buckets__init(positive); + positive->offset = sample->exp_hist_positive_offset; + positive->n_bucket_counts = sample->exp_hist_positive_count; + positive->bucket_counts = calloc(sample->exp_hist_positive_count, sizeof(uint64_t)); + if (positive->bucket_counts == NULL) { + free(positive); + destroy_data_point(exponential_data_point, map->type); + return CMT_ENCODE_OPENTELEMETRY_DATA_POINT_INIT_ERROR; + } + memcpy(positive->bucket_counts, sample->exp_hist_positive_buckets, + sizeof(uint64_t) * sample->exp_hist_positive_count); + exponential_data_point->positive = positive; + } + + if (sample->exp_hist_negative_count > 0) { + negative = calloc(1, sizeof(Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets)); + if (negative == NULL) { + destroy_data_point(exponential_data_point, map->type); + return CMT_ENCODE_OPENTELEMETRY_DATA_POINT_INIT_ERROR; + } + + opentelemetry__proto__metrics__v1__exponential_histogram_data_point__buckets__init(negative); + negative->offset = sample->exp_hist_negative_offset; + negative->n_bucket_counts = sample->exp_hist_negative_count; + negative->bucket_counts = calloc(sample->exp_hist_negative_count, sizeof(uint64_t)); + if (negative->bucket_counts == NULL) { + free(negative); + destroy_data_point(exponential_data_point, map->type); + return CMT_ENCODE_OPENTELEMETRY_DATA_POINT_INIT_ERROR; + } + memcpy(negative->bucket_counts, sample->exp_hist_negative_buckets, + sizeof(uint64_t) * sample->exp_hist_negative_count); + exponential_data_point->negative = negative; + } + + data_point = exponential_data_point; + } if (data_point == NULL) { destroy_attribute_list(attribute_list); @@ -2353,6 +3348,8 @@ int append_sample_to_metric(struct cmt_opentelemetry_context *context, _head, &map->label_keys); } + apply_data_point_metadata_from_otlp_context(context->cmt, map, sample, data_point); + result = append_data_point_to_metric(metric, (void *) data_point, sample_index); if (result != CMT_ENCODE_OPENTELEMETRY_SUCCESS) { @@ -2373,10 +3370,13 @@ int pack_basic_type(struct cmt_opentelemetry_context *context, size_t sample_index; size_t sample_count; struct cmt_counter *counter; + struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; struct cmt_metric *sample; Opentelemetry__Proto__Metrics__V1__Metric *metric; int result; struct cfl_list *head; + size_t target_scope_index; sample_count = 0; @@ -2407,6 +3407,35 @@ int pack_basic_type(struct cmt_opentelemetry_context *context, monotonism_flag = !counter->allow_reset; } } + else if (map->type == CMT_HISTOGRAM) { + if (map->parent != NULL) { + histogram = (struct cmt_histogram *) map->parent; + + if (histogram->aggregation_type == CMT_AGGREGATION_TYPE_DELTA) { + aggregation_temporality_type = OPENTELEMETRY__PROTO__METRICS__V1__AGGREGATION_TEMPORALITY__AGGREGATION_TEMPORALITY_DELTA; + } + else if (histogram->aggregation_type == CMT_AGGREGATION_TYPE_CUMULATIVE) { + aggregation_temporality_type = OPENTELEMETRY__PROTO__METRICS__V1__AGGREGATION_TEMPORALITY__AGGREGATION_TEMPORALITY_CUMULATIVE; + } + } + } + else if (map->type == CMT_EXP_HISTOGRAM) { + if (map->parent != NULL) { + exp_histogram = (struct cmt_exp_histogram *) map->parent; + + if (exp_histogram->aggregation_type == CMT_AGGREGATION_TYPE_DELTA) { + aggregation_temporality_type = OPENTELEMETRY__PROTO__METRICS__V1__AGGREGATION_TEMPORALITY__AGGREGATION_TEMPORALITY_DELTA; + } + else if (exp_histogram->aggregation_type == CMT_AGGREGATION_TYPE_CUMULATIVE) { + aggregation_temporality_type = OPENTELEMETRY__PROTO__METRICS__V1__AGGREGATION_TEMPORALITY__AGGREGATION_TEMPORALITY_CUMULATIVE; + } + } + } + + target_scope_index = resolve_target_scope_index(context, map); + if (target_scope_index >= context->scope_metrics_count) { + return CMT_ENCODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR; + } metric = initialize_metric(map->type, map->opts->fqname, @@ -2420,6 +3449,8 @@ int pack_basic_type(struct cmt_opentelemetry_context *context, return CMT_ENCODE_OPENTELEMETRY_ALLOCATION_ERROR; } + apply_metric_metadata_from_otlp_context(context->cmt, map, metric); + sample_index = 0; if (map->metric_static_set) { @@ -2452,13 +3483,9 @@ int pack_basic_type(struct cmt_opentelemetry_context *context, } } - result = append_metric_to_scope_metrics( - context->\ - metrics_data->\ - resource_metrics[0]->\ - scope_metrics[0], - metric, - *metric_index); + result = append_metric_to_scope_metrics(context->scope_metrics_list[target_scope_index], + metric, + get_metric_count(context->cmt)); if (result != CMT_ENCODE_OPENTELEMETRY_SUCCESS) { destroy_metric(metric); @@ -2498,6 +3525,7 @@ cfl_sds_t cmt_encode_opentelemetry_create(struct cmt *cmt) size_t metric_index; struct cmt_opentelemetry_context *context; struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; struct cmt_summary *summary; struct cmt_untyped *untyped; struct cmt_counter *counter; @@ -2572,6 +3600,17 @@ cfl_sds_t cmt_encode_opentelemetry_create(struct cmt *cmt) } } + if (result == CMT_ENCODE_OPENTELEMETRY_SUCCESS) { + cfl_list_foreach(head, &cmt->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + result = pack_basic_type(context, exp_histogram->map, &metric_index); + + if (result != CMT_ENCODE_OPENTELEMETRY_SUCCESS) { + break; + } + } + } + if (result == CMT_ENCODE_OPENTELEMETRY_SUCCESS) { buf = render_opentelemetry_context_to_sds(context); } diff --git a/lib/cmetrics/src/cmt_encode_prometheus.c b/lib/cmetrics/src/cmt_encode_prometheus.c index f020288323b..d1a8cab87b9 100644 --- a/lib/cmetrics/src/cmt_encode_prometheus.c +++ b/lib/cmetrics/src/cmt_encode_prometheus.c @@ -26,6 +26,7 @@ #include #include #include +#include #include #include @@ -128,6 +129,9 @@ static void metric_banner(cfl_sds_t *buf, struct cmt_map *map, else if (map->type == CMT_HISTOGRAM) { cfl_sds_cat_safe(buf, " histogram\n", 11); } + else if (map->type == CMT_EXP_HISTOGRAM) { + cfl_sds_cat_safe(buf, " histogram\n", 11); + } else if (map->type == CMT_UNTYPED) { cfl_sds_cat_safe(buf, " untyped\n", 9); } @@ -171,6 +175,14 @@ static void append_metric_value(cfl_sds_t *buf, val = cmt_metric_hist_get_count_value(metric); } } + else if (map->type == CMT_EXP_HISTOGRAM) { + if (fmt->value_from == PROM_FMT_VAL_FROM_SUM) { + val = cmt_math_uint64_to_d64(metric->exp_hist_sum); + } + else if (fmt->value_from == PROM_FMT_VAL_FROM_COUNT) { + val = metric->exp_hist_count; + } + } else if (map->type == CMT_SUMMARY) { if (fmt->value_from == PROM_FMT_VAL_FROM_SUM) { val = cmt_summary_get_sum_value(metric); @@ -320,7 +332,8 @@ static cfl_sds_t bucket_value_to_string(double val) static void format_histogram_bucket(struct cmt *cmt, cfl_sds_t *buf, struct cmt_map *map, - struct cmt_metric *metric, int add_timestamp) + struct cmt_metric *metric, int add_timestamp, + int include_sum) { int i; cfl_sds_t val; @@ -362,16 +375,20 @@ static void format_histogram_bucket(struct cmt *cmt, format_metric(cmt, buf, map, metric, add_timestamp, &fmt); } - /* sum */ - prom_fmt_init(&fmt); - fmt.metric_name = CMT_TRUE; - fmt.value_from = PROM_FMT_VAL_FROM_SUM; + if (include_sum) { + /* sum */ + prom_fmt_init(&fmt); + fmt.metric_name = CMT_TRUE; + fmt.value_from = PROM_FMT_VAL_FROM_SUM; - cfl_sds_cat_safe(buf, opts->fqname, cfl_sds_len(opts->fqname)); - cfl_sds_cat_safe(buf, "_sum", 4); - format_metric(cmt, buf, map, metric, add_timestamp, &fmt); + cfl_sds_cat_safe(buf, opts->fqname, cfl_sds_len(opts->fqname)); + cfl_sds_cat_safe(buf, "_sum", 4); + format_metric(cmt, buf, map, metric, add_timestamp, &fmt); + } /* count */ + prom_fmt_init(&fmt); + fmt.metric_name = CMT_TRUE; fmt.labels_count = 0; fmt.value_from = PROM_FMT_VAL_FROM_COUNT; @@ -450,7 +467,56 @@ static void format_metrics(struct cmt *cmt, cfl_sds_t *buf, struct cmt_map *map, if (map->type == CMT_HISTOGRAM) { /* Histogram needs to format the buckets, one line per bucket */ - format_histogram_bucket(cmt, buf, map, &map->metric, add_timestamp); + format_histogram_bucket(cmt, buf, map, &map->metric, add_timestamp, + CMT_TRUE); + } + else if (map->type == CMT_EXP_HISTOGRAM) { + struct cmt_map fake_map; + struct cmt_histogram fake_histogram; + struct cmt_histogram_buckets fake_buckets; + uint64_t *original_hist_buckets; + uint64_t original_hist_count; + uint64_t original_hist_sum; + size_t bucket_count; + size_t upper_bounds_count; + uint64_t *bucket_values; + double *upper_bounds; + + if (cmt_exp_histogram_to_explicit(&map->metric, + &upper_bounds, + &upper_bounds_count, + &bucket_values, + &bucket_count) == 0) { + memset(&fake_map, 0, sizeof(struct cmt_map)); + memset(&fake_histogram, 0, sizeof(struct cmt_histogram)); + memset(&fake_buckets, 0, sizeof(struct cmt_histogram_buckets)); + + fake_buckets.count = upper_bounds_count; + fake_buckets.upper_bounds = upper_bounds; + fake_histogram.buckets = &fake_buckets; + + fake_map = *map; + fake_map.type = CMT_HISTOGRAM; + fake_map.parent = &fake_histogram; + + original_hist_buckets = map->metric.hist_buckets; + original_hist_count = map->metric.hist_count; + original_hist_sum = map->metric.hist_sum; + + map->metric.hist_buckets = bucket_values; + map->metric.hist_count = map->metric.exp_hist_count; + map->metric.hist_sum = map->metric.exp_hist_sum; + + format_histogram_bucket(cmt, buf, &fake_map, &map->metric, + add_timestamp, map->metric.exp_hist_sum_set); + + map->metric.hist_buckets = original_hist_buckets; + map->metric.hist_count = original_hist_count; + map->metric.hist_sum = original_hist_sum; + + free(bucket_values); + free(upper_bounds); + } } else if (map->type == CMT_SUMMARY) { /* Histogram needs to format the buckets, one line per bucket */ @@ -475,7 +541,55 @@ static void format_metrics(struct cmt *cmt, cfl_sds_t *buf, struct cmt_map *map, /* Format the metric based on its type */ if (map->type == CMT_HISTOGRAM) { /* Histogram needs to format the buckets, one line per bucket */ - format_histogram_bucket(cmt, buf, map, metric, add_timestamp); + format_histogram_bucket(cmt, buf, map, metric, add_timestamp, CMT_TRUE); + } + else if (map->type == CMT_EXP_HISTOGRAM) { + struct cmt_map fake_map; + struct cmt_histogram fake_histogram; + struct cmt_histogram_buckets fake_buckets; + uint64_t *original_hist_buckets; + uint64_t original_hist_count; + uint64_t original_hist_sum; + size_t bucket_count; + size_t upper_bounds_count; + uint64_t *bucket_values; + double *upper_bounds; + + if (cmt_exp_histogram_to_explicit(metric, + &upper_bounds, + &upper_bounds_count, + &bucket_values, + &bucket_count) == 0) { + memset(&fake_map, 0, sizeof(struct cmt_map)); + memset(&fake_histogram, 0, sizeof(struct cmt_histogram)); + memset(&fake_buckets, 0, sizeof(struct cmt_histogram_buckets)); + + fake_buckets.count = upper_bounds_count; + fake_buckets.upper_bounds = upper_bounds; + fake_histogram.buckets = &fake_buckets; + + fake_map = *map; + fake_map.type = CMT_HISTOGRAM; + fake_map.parent = &fake_histogram; + + original_hist_buckets = metric->hist_buckets; + original_hist_count = metric->hist_count; + original_hist_sum = metric->hist_sum; + + metric->hist_buckets = bucket_values; + metric->hist_count = metric->exp_hist_count; + metric->hist_sum = metric->exp_hist_sum; + + format_histogram_bucket(cmt, buf, &fake_map, metric, add_timestamp, + metric->exp_hist_sum_set); + + metric->hist_buckets = original_hist_buckets; + metric->hist_count = original_hist_count; + metric->hist_sum = original_hist_sum; + + free(bucket_values); + free(upper_bounds); + } } else if (map->type == CMT_SUMMARY) { format_summary_quantiles(cmt, buf, map, metric, add_timestamp); @@ -496,6 +610,7 @@ cfl_sds_t cmt_encode_prometheus_create(struct cmt *cmt, int add_timestamp) struct cmt_gauge *gauge; struct cmt_summary *summary; struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; struct cmt_untyped *untyped; /* Allocate a 1KB of buffer */ @@ -528,6 +643,12 @@ cfl_sds_t cmt_encode_prometheus_create(struct cmt *cmt, int add_timestamp) format_metrics(cmt, &buf, histogram->map, add_timestamp); } + /* Exponential Histograms */ + cfl_list_foreach(head, &cmt->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + format_metrics(cmt, &buf, exp_histogram->map, add_timestamp); + } + /* Untyped */ cfl_list_foreach(head, &cmt->untypeds) { untyped = cfl_list_entry(head, struct cmt_untyped, _head); diff --git a/lib/cmetrics/src/cmt_encode_prometheus_remote_write.c b/lib/cmetrics/src/cmt_encode_prometheus_remote_write.c index 0a109868697..8eea8206393 100644 --- a/lib/cmetrics/src/cmt_encode_prometheus_remote_write.c +++ b/lib/cmetrics/src/cmt_encode_prometheus_remote_write.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -520,7 +521,7 @@ int set_up_time_series_for_label_set(struct cmt_prometheus_remote_write_context *time_series = time_series_entry; - return CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; + return result; } @@ -552,6 +553,9 @@ int pack_metric_metadata(struct cmt_prometheus_remote_write_context *context, else if (map->type == CMT_SUMMARY) { metadata_entry->data.type = PROMETHEUS__METRIC_METADATA__METRIC_TYPE__SUMMARY; } + else if (map->type == CMT_HISTOGRAM || map->type == CMT_EXP_HISTOGRAM) { + metadata_entry->data.type = PROMETHEUS__METRIC_METADATA__METRIC_TYPE__HISTOGRAM; + } else { return CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_UNEXPECTED_METRIC_TYPE; } @@ -721,13 +725,25 @@ int pack_complex_metric_sample(struct cmt_prometheus_remote_write_context *conte struct cmt_metric dummy_metric; struct cmt_prometheus_time_series *time_series; struct cmt_map_label *dummy_label; - struct cmt_histogram *histogram; + struct cmt_histogram *histogram = NULL; struct cmt_summary *summary; + double *exp_upper_bounds; + uint64_t *exp_bucket_counts; + size_t exp_upper_bounds_count; + size_t exp_bucket_count; + size_t bucket_count; + double bucket_value; + double sum_value; + double count_value; int result; size_t index; uint64_t now; now = cfl_time_now(); + exp_upper_bounds = NULL; + exp_bucket_counts = NULL; + exp_upper_bounds_count = 0; + exp_bucket_count = 0; if (check_staled_timestamp(metric, now, CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_THRESHOLD)) { @@ -873,38 +889,61 @@ int pack_complex_metric_sample(struct cmt_prometheus_remote_write_context *conte } } } - else if (map->type == CMT_HISTOGRAM) { - histogram = (struct cmt_histogram *) map->parent; + else if (map->type == CMT_HISTOGRAM || map->type == CMT_EXP_HISTOGRAM) { + if (map->type == CMT_HISTOGRAM) { + histogram = (struct cmt_histogram *) map->parent; + bucket_count = histogram->buckets->count; + } + else { + result = cmt_exp_histogram_to_explicit(metric, + &exp_upper_bounds, + &exp_upper_bounds_count, + &exp_bucket_counts, + &exp_bucket_count); + if (result != 0) { + result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_ALLOCATION_ERROR; + } + else { + bucket_count = exp_upper_bounds_count; + } + } - context->sequence_number += SYNTHETIC_METRIC_HISTOGRAM_COUNT_SEQUENCE_DELTA; + if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { + context->sequence_number += SYNTHETIC_METRIC_HISTOGRAM_COUNT_SEQUENCE_DELTA; + map->opts->fqname = synthetized_metric_name; - map->opts->fqname = synthetized_metric_name; + cfl_sds_len_set(synthetized_metric_name, + snprintf(synthetized_metric_name, + cfl_sds_alloc(synthetized_metric_name) -1, + "%s_count", + original_metric_name)); - cfl_sds_len_set(synthetized_metric_name, - snprintf(synthetized_metric_name, - cfl_sds_alloc(synthetized_metric_name) -1, - "%s_count", - original_metric_name)); + if (map->type == CMT_HISTOGRAM) { + count_value = cmt_metric_hist_get_count_value(metric); + } + else { + count_value = metric->exp_hist_count; + } - cmt_metric_set(&dummy_metric, - dummy_metric.timestamp, - cmt_metric_hist_get_count_value(metric)); + cmt_metric_set(&dummy_metric, dummy_metric.timestamp, count_value); + result = set_up_time_series_for_label_set(context, map, metric, &time_series); - result = set_up_time_series_for_label_set(context, map, metric, &time_series); + if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { + if (add_metadata == CMT_TRUE) { + result = pack_metric_metadata(context, map, &dummy_metric); + } - if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { - if (add_metadata == CMT_TRUE) { - result = pack_metric_metadata(context, map, &dummy_metric); + if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { + result = append_metric_to_timeseries(time_series, &dummy_metric); + } } - if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { - result = append_metric_to_timeseries(time_series, &dummy_metric); - } + context->sequence_number -= SYNTHETIC_METRIC_HISTOGRAM_COUNT_SEQUENCE_DELTA; } - context->sequence_number -= SYNTHETIC_METRIC_HISTOGRAM_COUNT_SEQUENCE_DELTA; - - if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { + if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS && + (map->type == CMT_HISTOGRAM || + (map->type == CMT_EXP_HISTOGRAM && metric->exp_hist_sum_set == CMT_TRUE))) { context->sequence_number += SYNTHETIC_METRIC_HISTOGRAM_SUM_SEQUENCE_DELTA; cfl_sds_len_set(synthetized_metric_name, @@ -913,10 +952,14 @@ int pack_complex_metric_sample(struct cmt_prometheus_remote_write_context *conte "%s_sum", original_metric_name)); - cmt_metric_set(&dummy_metric, - dummy_metric.timestamp, - cmt_metric_hist_get_sum_value(metric)); + if (map->type == CMT_HISTOGRAM) { + sum_value = cmt_metric_hist_get_sum_value(metric); + } + else { + sum_value = cmt_math_uint64_to_d64(metric->exp_hist_sum); + } + cmt_metric_set(&dummy_metric, dummy_metric.timestamp, sum_value); result = set_up_time_series_for_label_set(context, map, metric, &time_series); if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { @@ -957,29 +1000,43 @@ int pack_complex_metric_sample(struct cmt_prometheus_remote_write_context *conte if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { additional_label = cfl_list_entry_last(&metric->labels, struct cmt_map_label, _head); - additional_label->name = (cfl_sds_t) additional_label_caption; for(index = 0 ; result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS && - index <= histogram->buckets->count ; + index <= bucket_count ; index++) { - if (index < histogram->buckets->count) { - cfl_sds_len_set(additional_label_caption, + if (index < bucket_count) { + if (map->type == CMT_HISTOGRAM) { + cfl_sds_len_set(additional_label_caption, + snprintf(additional_label_caption, + cfl_sds_alloc(additional_label_caption) - 1, + "%.17g", + histogram->buckets->upper_bounds[index])); + } + else { + cfl_sds_len_set(additional_label_caption, snprintf(additional_label_caption, - cfl_sds_alloc(additional_label_caption) - 1, - "%.17g", histogram->buckets->upper_bounds[index])); + cfl_sds_alloc(additional_label_caption) - 1, + "%.17g", + exp_upper_bounds[index])); + } } else { cfl_sds_len_set(additional_label_caption, - snprintf(additional_label_caption, - cfl_sds_alloc(additional_label_caption) - 1, - "+Inf")); + snprintf(additional_label_caption, + cfl_sds_alloc(additional_label_caption) - 1, + "+Inf")); } + if (map->type == CMT_HISTOGRAM) { + bucket_value = cmt_metric_hist_get_value(metric, index); + } + else { + bucket_value = exp_bucket_counts[index]; + } - dummy_metric.val = cmt_math_d64_to_uint64(cmt_metric_hist_get_value(metric, index)); - + dummy_metric.val = cmt_math_d64_to_uint64(bucket_value); result = set_up_time_series_for_label_set(context, map, metric, &time_series); if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { @@ -1014,6 +1071,9 @@ int pack_complex_metric_sample(struct cmt_prometheus_remote_write_context *conte } } + free(exp_upper_bounds); + free(exp_bucket_counts); + if (additional_label_caption != NULL) { cfl_sds_destroy(additional_label_caption); } @@ -1037,11 +1097,13 @@ int pack_complex_type(struct cmt_prometheus_remote_write_context *context, add_metadata = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_ADD_METADATA; result = CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS; - if (map->type == CMT_SUMMARY || map->type == CMT_HISTOGRAM) { + if (map->type == CMT_SUMMARY || + map->type == CMT_HISTOGRAM || + map->type == CMT_EXP_HISTOGRAM) { if (map->type == CMT_SUMMARY) { additional_label.name = (cfl_sds_t) "quantile"; } - else if (map->type == CMT_HISTOGRAM) { + else { additional_label.name = (cfl_sds_t) "le"; } @@ -1078,7 +1140,9 @@ int pack_complex_type(struct cmt_prometheus_remote_write_context *context, } } - if (map->type == CMT_SUMMARY || map->type == CMT_HISTOGRAM) { + if (map->type == CMT_SUMMARY || + map->type == CMT_HISTOGRAM || + map->type == CMT_EXP_HISTOGRAM) { cfl_list_del(&additional_label._head); } @@ -1089,6 +1153,7 @@ int pack_complex_type(struct cmt_prometheus_remote_write_context *context, cfl_sds_t cmt_encode_prometheus_remote_write_create(struct cmt *cmt) { struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; struct cmt_prometheus_remote_write_context context; struct cmt_untyped *untyped; struct cmt_counter *counter; @@ -1184,6 +1249,22 @@ cfl_sds_t cmt_encode_prometheus_remote_write_create(struct cmt *cmt) } } + if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { + /* Exponential Histograms */ + cfl_list_foreach(head, &cmt->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + result = pack_complex_type(&context, exp_histogram->map); + + if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR) { + continue; + } + + if (result != CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS) { + break; + } + } + } + if (result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_SUCCESS || result == CMT_ENCODE_PROMETHEUS_REMOTE_WRITE_CUTOFF_ERROR) { buf = render_remote_write_context_to_sds(&context); diff --git a/lib/cmetrics/src/cmt_encode_splunk_hec.c b/lib/cmetrics/src/cmt_encode_splunk_hec.c index 49def335fa3..2f5730f9af0 100644 --- a/lib/cmetrics/src/cmt_encode_splunk_hec.c +++ b/lib/cmetrics/src/cmt_encode_splunk_hec.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -297,7 +298,7 @@ static void format_histogram_bucket(struct cmt_splunk_hec_context *context, cfl_ int len = 0; char tmp[128]; cfl_sds_t val; - uint64_t metric_val; + double metric_val; struct cmt_histogram *histogram; struct cmt_histogram_buckets *buckets; cfl_sds_t metric_str; @@ -565,6 +566,53 @@ static void format_metric(struct cmt_splunk_hec_context *context, cfl_sds_t *buf if (map->type == CMT_HISTOGRAM) { return format_histogram_bucket(context, buf, map, metric); } + else if (map->type == CMT_EXP_HISTOGRAM) { + struct cmt_map fake_map; + struct cmt_histogram fake_histogram; + struct cmt_histogram_buckets fake_buckets; + uint64_t *bucket_counts = NULL; + double *upper_bounds = NULL; + size_t upper_bounds_count = 0; + size_t bucket_count = 0; + uint64_t *orig_hist_buckets; + uint64_t orig_hist_count; + uint64_t orig_hist_sum; + + if (cmt_exp_histogram_to_explicit(metric, + &upper_bounds, + &upper_bounds_count, + &bucket_counts, + &bucket_count) != 0) { + return; + } + + fake_buckets.count = upper_bounds_count; + fake_buckets.upper_bounds = upper_bounds; + fake_histogram.buckets = &fake_buckets; + + memcpy(&fake_map, map, sizeof(struct cmt_map)); + fake_map.type = CMT_HISTOGRAM; + fake_map.parent = &fake_histogram; + + orig_hist_buckets = metric->hist_buckets; + orig_hist_count = metric->hist_count; + orig_hist_sum = metric->hist_sum; + + metric->hist_buckets = bucket_counts; + metric->hist_count = metric->exp_hist_count; + metric->hist_sum = metric->exp_hist_sum; + + format_histogram_bucket(context, buf, &fake_map, metric); + + metric->hist_buckets = orig_hist_buckets; + metric->hist_count = orig_hist_count; + metric->hist_sum = orig_hist_sum; + + free(bucket_counts); + free(upper_bounds); + + return; + } else if (map->type == CMT_SUMMARY) { return format_summary_metric(context, buf, map, metric); } @@ -656,6 +704,7 @@ cfl_sds_t cmt_encode_splunk_hec_create(struct cmt *cmt, const char *host, struct cmt_untyped *untyped; struct cmt_summary *summary; struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; struct cmt_splunk_hec_context *context; context = initialize_splunk_hec_context(cmt, host, index, source, source_type); @@ -695,6 +744,12 @@ cfl_sds_t cmt_encode_splunk_hec_create(struct cmt *cmt, const char *host, format_metrics(context, &buf, histogram->map); } + /* Exponential Histograms */ + cfl_list_foreach(head, &cmt->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + format_metrics(context, &buf, exp_histogram->map); + } + /* Untyped */ cfl_list_foreach(head, &cmt->untypeds) { untyped = cfl_list_entry(head, struct cmt_untyped, _head); diff --git a/lib/cmetrics/src/cmt_encode_text.c b/lib/cmetrics/src/cmt_encode_text.c index ffdfe314c20..0de654f5d62 100644 --- a/lib/cmetrics/src/cmt_encode_text.c +++ b/lib/cmetrics/src/cmt_encode_text.c @@ -24,10 +24,252 @@ #include #include #include +#include #include #include #include +static const char *map_type_to_otlp_key(int map_type) +{ + switch (map_type) { + case CMT_COUNTER: + return "counter"; + case CMT_GAUGE: + return "gauge"; + case CMT_UNTYPED: + return "untyped"; + case CMT_SUMMARY: + return "summary"; + case CMT_HISTOGRAM: + return "histogram"; + case CMT_EXP_HISTOGRAM: + return "exp_histogram"; + default: + return NULL; + } +} + +static struct cfl_kvlist *fetch_metadata_kvlist_key(struct cfl_kvlist *kvlist, const char *key) +{ + struct cfl_variant *entry_variant; + + if (kvlist == NULL) { + return NULL; + } + + entry_variant = cfl_kvlist_fetch(kvlist, (char *) key); + if (entry_variant == NULL || entry_variant->type != CFL_VARIANT_KVLIST) { + return NULL; + } + + return entry_variant->data.as_kvlist; +} + +static void append_variant_value(cfl_sds_t *buf, struct cfl_variant *value); + +static void append_kvlist_value(cfl_sds_t *buf, struct cfl_kvlist *kvlist) +{ + int count; + struct cfl_list *head; + struct cfl_kvpair *kvpair; + + cfl_sds_cat_safe(buf, "{", 1); + + count = 0; + cfl_list_foreach(head, &kvlist->list) { + kvpair = cfl_list_entry(head, struct cfl_kvpair, _head); + + if (count > 0) { + cfl_sds_cat_safe(buf, ", ", 2); + } + + cfl_sds_cat_safe(buf, kvpair->key, cfl_sds_len(kvpair->key)); + cfl_sds_cat_safe(buf, "=", 1); + append_variant_value(buf, kvpair->val); + + count++; + } + + cfl_sds_cat_safe(buf, "}", 1); +} + +static void append_array_value(cfl_sds_t *buf, struct cfl_array *array) +{ + size_t index; + struct cfl_variant *entry; + + cfl_sds_cat_safe(buf, "[", 1); + + for (index = 0; index < array->entry_count; index++) { + entry = cfl_array_fetch_by_index(array, index); + if (entry == NULL) { + continue; + } + + if (index > 0) { + cfl_sds_cat_safe(buf, ", ", 2); + } + + append_variant_value(buf, entry); + } + + cfl_sds_cat_safe(buf, "]", 1); +} + +static void append_bytes_value(cfl_sds_t *buf, char *bytes) +{ + size_t index; + size_t length; + char tmp[4]; + int len; + + length = cfl_sds_len(bytes); + + for (index = 0; index < length; index++) { + len = snprintf(tmp, sizeof(tmp), "%02x", (unsigned char) bytes[index]); + cfl_sds_cat_safe(buf, tmp, len); + } +} + +static void append_variant_value(cfl_sds_t *buf, struct cfl_variant *value) +{ + char tmp[128]; + int len; + + if (value == NULL) { + cfl_sds_cat_safe(buf, "null", 4); + return; + } + + if (value->type == CFL_VARIANT_STRING || value->type == CFL_VARIANT_REFERENCE) { + cfl_sds_cat_safe(buf, "\"", 1); + cfl_sds_cat_safe(buf, value->data.as_string, cfl_sds_len(value->data.as_string)); + cfl_sds_cat_safe(buf, "\"", 1); + } + else if (value->type == CFL_VARIANT_BOOL) { + cfl_sds_cat_safe(buf, value->data.as_bool ? "true" : "false", + value->data.as_bool ? 4 : 5); + } + else if (value->type == CFL_VARIANT_INT) { + len = snprintf(tmp, sizeof(tmp), "%" PRId64, value->data.as_int64); + cfl_sds_cat_safe(buf, tmp, len); + } + else if (value->type == CFL_VARIANT_UINT) { + len = snprintf(tmp, sizeof(tmp), "%" PRIu64, value->data.as_uint64); + cfl_sds_cat_safe(buf, tmp, len); + } + else if (value->type == CFL_VARIANT_DOUBLE) { + len = snprintf(tmp, sizeof(tmp), "%.17g", value->data.as_double); + cfl_sds_cat_safe(buf, tmp, len); + } + else if (value->type == CFL_VARIANT_BYTES) { + append_bytes_value(buf, value->data.as_bytes); + } + else if (value->type == CFL_VARIANT_ARRAY) { + append_array_value(buf, value->data.as_array); + } + else if (value->type == CFL_VARIANT_KVLIST) { + append_kvlist_value(buf, value->data.as_kvlist); + } + else { + cfl_sds_cat_safe(buf, "", 13); + } +} + +static struct cfl_kvlist *get_data_point_metadata_context(struct cmt *cmt, + struct cmt_map *map, + struct cmt_metric *metric) +{ + struct cfl_kvlist *otlp_root; + struct cfl_kvlist *metrics_root; + struct cfl_kvlist *type_root; + struct cfl_kvlist *metric_context; + struct cfl_kvlist *datapoints_context; + const char *type_key; + char key[128]; + + type_key = map_type_to_otlp_key(map->type); + if (type_key == NULL) { + return NULL; + } + + otlp_root = fetch_metadata_kvlist_key(cmt->external_metadata, "otlp"); + if (otlp_root == NULL) { + return NULL; + } + + metrics_root = fetch_metadata_kvlist_key(otlp_root, "metrics"); + if (metrics_root == NULL) { + return NULL; + } + + type_root = fetch_metadata_kvlist_key(metrics_root, type_key); + if (type_root == NULL) { + return NULL; + } + + metric_context = fetch_metadata_kvlist_key(type_root, map->opts->fqname); + if (metric_context == NULL) { + return NULL; + } + + datapoints_context = fetch_metadata_kvlist_key(metric_context, "datapoints"); + if (datapoints_context == NULL) { + return NULL; + } + + snprintf(key, sizeof(key) - 1, "%" PRIx64 ":%" PRIu64, + metric != NULL ? metric->hash : 0, + metric != NULL ? cmt_metric_get_timestamp(metric) : 0); + + return fetch_metadata_kvlist_key(datapoints_context, key); +} + +static void append_metric_exemplars(struct cmt *cmt, + cfl_sds_t *buf, + struct cmt_map *map, + struct cmt_metric *metric) +{ + struct cfl_kvlist *point_metadata; + struct cfl_variant *exemplars_variant; + struct cfl_array *exemplars; + struct cfl_variant *entry; + size_t index; + + point_metadata = get_data_point_metadata_context(cmt, map, metric); + if (point_metadata == NULL) { + return; + } + + exemplars_variant = cfl_kvlist_fetch(point_metadata, "exemplars"); + if (exemplars_variant == NULL || exemplars_variant->type != CFL_VARIANT_ARRAY) { + return; + } + + exemplars = exemplars_variant->data.as_array; + if (exemplars == NULL || exemplars->entry_count == 0) { + return; + } + + cfl_sds_cat_safe(buf, " exemplars=[", 13); + + for (index = 0; index < exemplars->entry_count; index++) { + entry = cfl_array_fetch_by_index(exemplars, index); + + if (entry == NULL || entry->type != CFL_VARIANT_KVLIST) { + continue; + } + + if (index > 0) { + cfl_sds_cat_safe(buf, ", ", 2); + } + + append_kvlist_value(buf, entry->data.as_kvlist); + } + + cfl_sds_cat_safe(buf, "]\n", 2); +} + static void append_histogram_metric_value(cfl_sds_t *buf, struct cmt_map *map, struct cmt_metric *metric) @@ -144,6 +386,79 @@ static void append_summary_metric_value(cfl_sds_t *buf, cfl_sds_cat_safe(buf, " }\n", 3); } +static void append_exp_histogram_metric_value(cfl_sds_t *buf, + struct cmt_metric *metric) +{ + size_t entry_buffer_length; + char entry_buffer[256]; + size_t index; + + cfl_sds_cat_safe(buf, " = { ", 5); + + entry_buffer_length = snprintf(entry_buffer, + sizeof(entry_buffer) - 1, + "scale=%d, zero_count=%" PRIu64 ", zero_threshold=%.17g, ", + metric->exp_hist_scale, + metric->exp_hist_zero_count, + metric->exp_hist_zero_threshold); + cfl_sds_cat_safe(buf, entry_buffer, entry_buffer_length); + + entry_buffer_length = snprintf(entry_buffer, + sizeof(entry_buffer) - 1, + "positive={offset=%d, bucket_counts=[", + metric->exp_hist_positive_offset); + cfl_sds_cat_safe(buf, entry_buffer, entry_buffer_length); + + for (index = 0; index < metric->exp_hist_positive_count; index++) { + entry_buffer_length = snprintf(entry_buffer, + sizeof(entry_buffer) - 1, + "%" PRIu64 "%s", + metric->exp_hist_positive_buckets[index], + (index + 1 < metric->exp_hist_positive_count) ? ", " : ""); + cfl_sds_cat_safe(buf, entry_buffer, entry_buffer_length); + } + + cfl_sds_cat_safe(buf, "]}, ", 4); + + entry_buffer_length = snprintf(entry_buffer, + sizeof(entry_buffer) - 1, + "negative={offset=%d, bucket_counts=[", + metric->exp_hist_negative_offset); + cfl_sds_cat_safe(buf, entry_buffer, entry_buffer_length); + + for (index = 0; index < metric->exp_hist_negative_count; index++) { + entry_buffer_length = snprintf(entry_buffer, + sizeof(entry_buffer) - 1, + "%" PRIu64 "%s", + metric->exp_hist_negative_buckets[index], + (index + 1 < metric->exp_hist_negative_count) ? ", " : ""); + cfl_sds_cat_safe(buf, entry_buffer, entry_buffer_length); + } + + cfl_sds_cat_safe(buf, "]}, ", 4); + + entry_buffer_length = snprintf(entry_buffer, + sizeof(entry_buffer) - 1, + "count=%" PRIu64, + metric->exp_hist_count); + cfl_sds_cat_safe(buf, entry_buffer, entry_buffer_length); + + if (metric->exp_hist_sum_set) { + entry_buffer_length = snprintf(entry_buffer, + sizeof(entry_buffer) - 1, + ", sum=%.17g", + cmt_math_uint64_to_d64(metric->exp_hist_sum)); + } + else { + entry_buffer_length = snprintf(entry_buffer, + sizeof(entry_buffer) - 1, + ", sum=unset"); + } + cfl_sds_cat_safe(buf, entry_buffer, entry_buffer_length); + + cfl_sds_cat_safe(buf, " }\n", 3); +} + static void append_metric_value(cfl_sds_t *buf, struct cmt_map *map, struct cmt_metric *metric) { @@ -154,6 +469,9 @@ static void append_metric_value(cfl_sds_t *buf, struct cmt_map *map, if (map->type == CMT_HISTOGRAM) { return append_histogram_metric_value(buf, map, metric); } + else if (map->type == CMT_EXP_HISTOGRAM) { + return append_exp_histogram_metric_value(buf, metric); + } else if (map->type == CMT_SUMMARY) { return append_summary_metric_value(buf, map, metric); } @@ -251,12 +569,14 @@ static void format_metric(struct cmt *cmt, cfl_sds_t *buf, struct cmt_map *map, cfl_sds_cat_safe(buf, "}", 1); append_metric_value(buf, map, metric); + append_metric_exemplars(cmt, buf, map, metric); } else { if (static_labels > 0) { cfl_sds_cat_safe(buf, "}", 1); } append_metric_value(buf, map, metric); + append_metric_exemplars(cmt, buf, map, metric); } } @@ -286,6 +606,7 @@ cfl_sds_t cmt_encode_text_create(struct cmt *cmt) struct cmt_untyped *untyped; struct cmt_summary *summary; struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; /* Allocate a 1KB of buffer */ buf = cfl_sds_create_size(1024); @@ -317,6 +638,12 @@ cfl_sds_t cmt_encode_text_create(struct cmt *cmt) format_metrics(cmt, &buf, histogram->map); } + /* Exponential Histograms */ + cfl_list_foreach(head, &cmt->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + format_metrics(cmt, &buf, exp_histogram->map); + } + /* Untyped */ cfl_list_foreach(head, &cmt->untypeds) { untyped = cfl_list_entry(head, struct cmt_untyped, _head); diff --git a/lib/cmetrics/src/cmt_exp_histogram.c b/lib/cmetrics/src/cmt_exp_histogram.c new file mode 100644 index 00000000000..9fea05a2bf7 --- /dev/null +++ b/lib/cmetrics/src/cmt_exp_histogram.c @@ -0,0 +1,327 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2021-2022 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include + +static struct cmt_metric *exp_histogram_get_metric(struct cmt_exp_histogram *exp_histogram, + int labels_count, char **label_vals) +{ + struct cmt_metric *metric; + + metric = cmt_map_metric_get(&exp_histogram->opts, exp_histogram->map, + labels_count, label_vals, CMT_TRUE); + if (!metric) { + cmt_log_error(exp_histogram->cmt, + "unable to retrieve metric for exponential histogram %s_%s_%s", + exp_histogram->opts.ns, exp_histogram->opts.subsystem, + exp_histogram->opts.name); + return NULL; + } + + return metric; +} + +struct cmt_exp_histogram *cmt_exp_histogram_create(struct cmt *cmt, + char *ns, char *subsystem, + char *name, char *help, + int label_count, char **label_keys) +{ + int ret; + struct cmt_exp_histogram *h; + + if (!ns) { + cmt_log_error(cmt, "null ns not allowed"); + return NULL; + } + + if (!subsystem) { + cmt_log_error(cmt, "null subsystem not allowed"); + return NULL; + } + + if (!name || strlen(name) == 0) { + cmt_log_error(cmt, "undefined name"); + return NULL; + } + + if (!help || strlen(help) == 0) { + cmt_log_error(cmt, "undefined help"); + return NULL; + } + + h = calloc(1, sizeof(struct cmt_exp_histogram)); + if (!h) { + cmt_errno(); + return NULL; + } + + cfl_list_add(&h->_head, &cmt->exp_histograms); + + ret = cmt_opts_init(&h->opts, ns, subsystem, name, help); + if (ret == -1) { + cmt_log_error(cmt, "unable to initialize options for exponential histogram"); + cmt_exp_histogram_destroy(h); + return NULL; + } + + h->map = cmt_map_create(CMT_EXP_HISTOGRAM, &h->opts, label_count, label_keys, (void *) h); + if (!h->map) { + cmt_log_error(cmt, "unable to allocate map for exponential histogram"); + cmt_exp_histogram_destroy(h); + return NULL; + } + + h->cmt = cmt; + + return h; +} + +int cmt_exp_histogram_set_default(struct cmt_exp_histogram *exp_histogram, + uint64_t timestamp, + int32_t scale, + uint64_t zero_count, + double zero_threshold, + int32_t positive_offset, + size_t positive_bucket_count, + uint64_t *positive_bucket_counts, + int32_t negative_offset, + size_t negative_bucket_count, + uint64_t *negative_bucket_counts, + int sum_set, + double sum, + uint64_t count, + int labels_count, char **label_vals) +{ + struct cmt_metric *metric; + uint64_t *new_positive_buckets; + uint64_t *new_negative_buckets; + + metric = exp_histogram_get_metric(exp_histogram, labels_count, label_vals); + if (!metric) { + return -1; + } + + new_positive_buckets = NULL; + new_negative_buckets = NULL; + + if (positive_bucket_count > 0 && positive_bucket_counts != NULL) { + new_positive_buckets = calloc(positive_bucket_count, sizeof(uint64_t)); + if (new_positive_buckets == NULL) { + return -1; + } + + memcpy(new_positive_buckets, positive_bucket_counts, + sizeof(uint64_t) * positive_bucket_count); + } + + if (negative_bucket_count > 0 && negative_bucket_counts != NULL) { + new_negative_buckets = calloc(negative_bucket_count, sizeof(uint64_t)); + if (new_negative_buckets == NULL) { + if (new_positive_buckets != NULL) { + free(new_positive_buckets); + } + return -1; + } + + memcpy(new_negative_buckets, negative_bucket_counts, + sizeof(uint64_t) * negative_bucket_count); + } + else if (negative_bucket_count > 0 && negative_bucket_counts == NULL) { + if (new_positive_buckets != NULL) { + free(new_positive_buckets); + } + return -1; + } + + if (positive_bucket_count > 0 && positive_bucket_counts == NULL) { + if (new_positive_buckets != NULL) { + free(new_positive_buckets); + } + if (new_negative_buckets != NULL) { + free(new_negative_buckets); + } + return -1; + } + + if (metric->exp_hist_positive_buckets != NULL) { + free(metric->exp_hist_positive_buckets); + } + if (metric->exp_hist_negative_buckets != NULL) { + free(metric->exp_hist_negative_buckets); + } + + metric->exp_hist_positive_buckets = new_positive_buckets; + metric->exp_hist_negative_buckets = new_negative_buckets; + metric->exp_hist_positive_count = positive_bucket_count; + metric->exp_hist_negative_count = negative_bucket_count; + + metric->exp_hist_scale = scale; + metric->exp_hist_zero_count = zero_count; + metric->exp_hist_zero_threshold = zero_threshold; + metric->exp_hist_positive_offset = positive_offset; + metric->exp_hist_negative_offset = negative_offset; + metric->exp_hist_count = count; + metric->exp_hist_sum_set = sum_set ? CMT_TRUE : CMT_FALSE; + metric->exp_hist_sum = cmt_math_d64_to_uint64(sum); + metric->timestamp = timestamp; + + return 0; +} + +int cmt_exp_histogram_destroy(struct cmt_exp_histogram *exp_histogram) +{ + cfl_list_del(&exp_histogram->_head); + cmt_opts_exit(&exp_histogram->opts); + + if (exp_histogram->map) { + cmt_map_destroy(exp_histogram->map); + } + + free(exp_histogram); + + return 0; +} + +int cmt_exp_histogram_to_explicit(struct cmt_metric *metric, + double **upper_bounds, + size_t *upper_bounds_count, + uint64_t **bucket_counts, + size_t *bucket_count) +{ + double base; + double *local_upper_bounds; + uint64_t *local_bucket_counts; + uint64_t cumulative_count; + size_t local_upper_bounds_count; + size_t local_bucket_count; + size_t index; + size_t target_index; + int64_t bucket_index; + int include_zero_threshold; + + if (metric == NULL || + upper_bounds == NULL || + upper_bounds_count == NULL || + bucket_counts == NULL || + bucket_count == NULL) { + return -1; + } + + base = pow(2.0, pow(2.0, (double) -metric->exp_hist_scale)); + if (!isfinite(base) || base <= 1.0) { + return -1; + } + + include_zero_threshold = (metric->exp_hist_zero_count > 0 || + metric->exp_hist_zero_threshold > 0.0 || + (metric->exp_hist_negative_count > 0 && metric->exp_hist_positive_count > 0) || + (metric->exp_hist_negative_count == 0 && metric->exp_hist_positive_count == 0)); + + local_upper_bounds_count = metric->exp_hist_negative_count + metric->exp_hist_positive_count; + + if (include_zero_threshold) { + local_upper_bounds_count += (metric->exp_hist_zero_threshold > 0.0) ? 3 : 1; + } + + local_bucket_count = local_upper_bounds_count + 1; + + local_upper_bounds = calloc(local_upper_bounds_count, sizeof(double)); + if (local_upper_bounds == NULL) { + return -1; + } + + local_bucket_counts = calloc(local_bucket_count, sizeof(uint64_t)); + if (local_bucket_counts == NULL) { + free(local_upper_bounds); + return -1; + } + + target_index = 0; + cumulative_count = 0; + + for (index = metric->exp_hist_negative_count ; index > 0 ; index--) { + bucket_index = (int64_t) metric->exp_hist_negative_offset + (int64_t) index - 1; + + local_upper_bounds[target_index] = -pow(base, (double) bucket_index); + if (!isfinite(local_upper_bounds[target_index])) { + free(local_bucket_counts); + free(local_upper_bounds); + return -1; + } + + cumulative_count += metric->exp_hist_negative_buckets[index - 1]; + local_bucket_counts[target_index] = cumulative_count; + target_index++; + } + + if (include_zero_threshold) { + if (metric->exp_hist_zero_threshold > 0.0) { + local_upper_bounds[target_index] = -metric->exp_hist_zero_threshold; + local_bucket_counts[target_index] = cumulative_count; + target_index++; + + cumulative_count += metric->exp_hist_zero_count; + + local_upper_bounds[target_index] = 0.0; + local_bucket_counts[target_index] = cumulative_count; + target_index++; + + local_upper_bounds[target_index] = metric->exp_hist_zero_threshold; + local_bucket_counts[target_index] = cumulative_count; + target_index++; + } + else { + cumulative_count += metric->exp_hist_zero_count; + local_upper_bounds[target_index] = 0.0; + local_bucket_counts[target_index] = cumulative_count; + target_index++; + } + } + + for (index = 0 ; index < metric->exp_hist_positive_count ; index++) { + bucket_index = (int64_t) metric->exp_hist_positive_offset + (int64_t) index + 1; + + local_upper_bounds[target_index] = pow(base, (double) bucket_index); + if (!isfinite(local_upper_bounds[target_index])) { + free(local_bucket_counts); + free(local_upper_bounds); + return -1; + } + + cumulative_count += metric->exp_hist_positive_buckets[index]; + local_bucket_counts[target_index] = cumulative_count; + target_index++; + } + + local_bucket_counts[local_bucket_count - 1] = metric->exp_hist_count; + + *upper_bounds = local_upper_bounds; + *upper_bounds_count = local_upper_bounds_count; + *bucket_counts = local_bucket_counts; + *bucket_count = local_bucket_count; + + return 0; +} diff --git a/lib/cmetrics/src/cmt_filter.c b/lib/cmetrics/src/cmt_filter.c index 82d89fe7074..6b7c220443d 100644 --- a/lib/cmetrics/src/cmt_filter.c +++ b/lib/cmetrics/src/cmt_filter.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include @@ -87,6 +88,7 @@ static int filter_context_label_key(struct cmt *dst, struct cmt *src, struct cmt_gauge *gauge; struct cmt_untyped *untyped; struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; struct cmt_summary *summary; /* Counters */ @@ -145,6 +147,20 @@ static int filter_context_label_key(struct cmt *dst, struct cmt *src, } } + /* Exponential Histogram */ + cfl_list_foreach(head, &src->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + + if (compare_label_keys(exp_histogram->map, label_key, compare_ctx, compare, flags) == CMT_FALSE) { + continue; + } + + ret = cmt_cat_exp_histogram(dst, exp_histogram, NULL); + if (ret == -1) { + return -1; + } + } + /* Summary */ cfl_list_foreach(head, &src->summaries) { summary = cfl_list_entry(head, struct cmt_summary, _head); @@ -260,6 +276,7 @@ static int filter_context_label_key_value(struct cmt *dst, struct cmt *src, struct cmt_gauge *gauge; struct cmt_untyped *untyped; struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; struct cmt_summary *summary; size_t index = 0; @@ -415,6 +432,44 @@ static int filter_context_label_key_value(struct cmt *dst, struct cmt *src, cmt_map_destroy(map); } + /* Exponential Histogram */ + cfl_list_foreach(head, &src->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + + ret = cmt_cat_copy_label_keys(exp_histogram->map, (char **) &labels); + if (ret == -1) { + return -1; + } + + map = cmt_map_create(CMT_EXP_HISTOGRAM, &exp_histogram->opts, + exp_histogram->map->label_count, + labels, (void *) exp_histogram); + free(labels); + if (!map) { + cmt_log_error(src, "unable to allocate map for exponential histogram"); + return -1; + } + + ret = cmt_cat_copy_map(&exp_histogram->opts, map, exp_histogram->map); + if (ret == -1) { + cmt_map_destroy(map); + return -1; + } + + index = filter_get_label_index(map, label_key); + if (index != -1) { + metrics_map_drop_label_value_pairs(map, index, label_value); + } + + ret = cmt_cat_exp_histogram(dst, exp_histogram, map); + if (ret == -1) { + cmt_map_destroy(map); + return -1; + } + + cmt_map_destroy(map); + } + /* Summary */ cfl_list_foreach(head, &src->summaries) { summary = cfl_list_entry(head, struct cmt_summary, _head); @@ -510,6 +565,7 @@ static int filter_context_fqname(struct cmt *dst, struct cmt *src, struct cmt_gauge *gauge; struct cmt_untyped *untyped; struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; struct cmt_summary *summary; /* Counters */ @@ -565,6 +621,19 @@ static int filter_context_fqname(struct cmt *dst, struct cmt *src, } } + /* Exponential Histogram */ + cfl_list_foreach(head, &src->exp_histograms) { + exp_histogram = cfl_list_entry(head, struct cmt_exp_histogram, _head); + if (compare_fqname(exp_histogram->map->opts, fqname, compare_ctx, compare, flags) == CMT_FALSE) { + continue; + } + + ret = cmt_cat_exp_histogram(dst, exp_histogram, NULL); + if (ret == -1) { + return -1; + } + } + /* Summary */ cfl_list_foreach(head, &src->summaries) { summary = cfl_list_entry(head, struct cmt_summary, _head); diff --git a/lib/cmetrics/src/cmt_map.c b/lib/cmetrics/src/cmt_map.c index 4639124a832..587e0cf8b34 100644 --- a/lib/cmetrics/src/cmt_map.c +++ b/lib/cmetrics/src/cmt_map.c @@ -109,7 +109,7 @@ static struct cmt_metric *map_metric_create(uint64_t hash, return NULL; } cfl_list_init(&metric->labels); - metric->val = 0.0; + cmt_metric_set_double(metric, 0, 0.0); metric->hash = hash; for (i = 0; i < labels_count; i++) { @@ -152,6 +152,12 @@ void cmt_map_metric_destroy(struct cmt_metric *metric) if (metric->hist_buckets) { free(metric->hist_buckets); } + if (metric->exp_hist_positive_buckets) { + free(metric->exp_hist_positive_buckets); + } + if (metric->exp_hist_negative_buckets) { + free(metric->exp_hist_negative_buckets); + } if (metric->sum_quantiles) { free(metric->sum_quantiles); } @@ -282,6 +288,14 @@ void cmt_map_destroy(struct cmt_map *map) free(metric->hist_buckets); } } + else if (map->type == CMT_EXP_HISTOGRAM) { + if (metric->exp_hist_positive_buckets) { + free(metric->exp_hist_positive_buckets); + } + if (metric->exp_hist_negative_buckets) { + free(metric->exp_hist_negative_buckets); + } + } else if (map->type == CMT_SUMMARY) { if (metric->sum_quantiles) { free(metric->sum_quantiles); @@ -289,6 +303,10 @@ void cmt_map_destroy(struct cmt_map *map) } } + if (map->unit != NULL) { + cfl_sds_destroy(map->unit); + } + free(map); } @@ -315,4 +333,3 @@ void destroy_label_list(struct cfl_list *label_list) free(label); } } - diff --git a/lib/cmetrics/src/cmt_metric.c b/lib/cmetrics/src/cmt_metric.c index dc58d26f8d2..7a94e22fa0b 100644 --- a/lib/cmetrics/src/cmt_metric.c +++ b/lib/cmetrics/src/cmt_metric.c @@ -22,7 +22,7 @@ #include #include -static inline int metric_exchange(struct cmt_metric *metric, uint64_t timestamp, +static inline int metric_exchange(struct cmt_metric *metric, double new_value, double old_value) { uint64_t tmp_new; @@ -38,8 +38,6 @@ static inline int metric_exchange(struct cmt_metric *metric, uint64_t timestamp, return 0; } - cmt_atomic_store(&metric->timestamp, timestamp); - return 1; } @@ -53,19 +51,58 @@ static inline void add(struct cmt_metric *metric, uint64_t timestamp, double val old = cmt_metric_get_value(metric); new = old + val; - result = metric_exchange(metric, timestamp, new, old); + result = metric_exchange(metric, new, old); } while(0 == result); + + cmt_atomic_store(&metric->val_int64, (uint64_t) ((int64_t) new)); + cmt_atomic_store(&metric->val_uint64, (uint64_t) new); + cmt_atomic_store(&metric->timestamp, timestamp); + cmt_atomic_store(&metric->value_type, CMT_METRIC_VALUE_DOUBLE); } void cmt_metric_set(struct cmt_metric *metric, uint64_t timestamp, double val) +{ + cmt_metric_set_double(metric, timestamp, val); +} + +void cmt_metric_set_double(struct cmt_metric *metric, uint64_t timestamp, double val) { uint64_t tmp; tmp = cmt_math_d64_to_uint64(val); cmt_atomic_store(&metric->val, tmp); + cmt_atomic_store(&metric->val_int64, (uint64_t) ((int64_t) val)); + cmt_atomic_store(&metric->val_uint64, (uint64_t) val); + cmt_atomic_store(&metric->timestamp, timestamp); + cmt_atomic_store(&metric->value_type, CMT_METRIC_VALUE_DOUBLE); +} + +void cmt_metric_set_int64(struct cmt_metric *metric, uint64_t timestamp, int64_t val) +{ + uint64_t tmp; + + tmp = cmt_math_d64_to_uint64((double) val); + + cmt_atomic_store(&metric->val, tmp); + cmt_atomic_store(&metric->val_int64, (uint64_t) val); + cmt_atomic_store(&metric->val_uint64, (uint64_t) val); + cmt_atomic_store(&metric->timestamp, timestamp); + cmt_atomic_store(&metric->value_type, CMT_METRIC_VALUE_INT64); +} + +void cmt_metric_set_uint64(struct cmt_metric *metric, uint64_t timestamp, uint64_t val) +{ + uint64_t tmp; + + tmp = cmt_math_d64_to_uint64((double) val); + + cmt_atomic_store(&metric->val, tmp); + cmt_atomic_store(&metric->val_int64, (uint64_t) ((int64_t) val)); + cmt_atomic_store(&metric->val_uint64, val); cmt_atomic_store(&metric->timestamp, timestamp); + cmt_atomic_store(&metric->value_type, CMT_METRIC_VALUE_UINT64); } static inline int metric_hist_exchange(struct cmt_metric *metric, @@ -132,6 +169,76 @@ double cmt_metric_get_value(struct cmt_metric *metric) return cmt_math_uint64_to_d64(val); } +int cmt_metric_get_value_type(struct cmt_metric *metric) +{ + return (int) cmt_atomic_load(&metric->value_type); +} + +int64_t cmt_metric_get_int64_value(struct cmt_metric *metric) +{ + uint64_t value_type; + + value_type = cmt_atomic_load(&metric->value_type); + + if (value_type == CMT_METRIC_VALUE_INT64) { + return (int64_t) cmt_atomic_load(&metric->val_int64); + } + + if (value_type == CMT_METRIC_VALUE_UINT64) { + return (int64_t) cmt_atomic_load(&metric->val_uint64); + } + + return (int64_t) cmt_metric_get_value(metric); +} + +uint64_t cmt_metric_get_uint64_value(struct cmt_metric *metric) +{ + uint64_t value_type; + + value_type = cmt_atomic_load(&metric->value_type); + + if (value_type == CMT_METRIC_VALUE_UINT64) { + return cmt_atomic_load(&metric->val_uint64); + } + + if (value_type == CMT_METRIC_VALUE_INT64) { + return (uint64_t) ((int64_t) cmt_atomic_load(&metric->val_int64)); + } + + return (uint64_t) cmt_metric_get_value(metric); +} + +void cmt_metric_get_value_snapshot(struct cmt_metric *metric, + int *out_type, + int64_t *out_int64, + uint64_t *out_uint64) +{ + uint64_t type_first; + uint64_t type_second; + uint64_t int_value; + uint64_t uint_value; + + do { + type_first = cmt_atomic_load(&metric->value_type); + int_value = cmt_atomic_load(&metric->val_int64); + uint_value = cmt_atomic_load(&metric->val_uint64); + type_second = cmt_atomic_load(&metric->value_type); + } + while (type_first != type_second); + + if (out_type != NULL) { + *out_type = (int) type_first; + } + + if (out_int64 != NULL) { + *out_int64 = (int64_t) int_value; + } + + if (out_uint64 != NULL) { + *out_uint64 = uint_value; + } +} + uint64_t cmt_metric_get_timestamp(struct cmt_metric *metric) { uint64_t val; diff --git a/lib/cmetrics/src/cmt_mpack_utils.c b/lib/cmetrics/src/cmt_mpack_utils.c index e303554d89a..ce639b356f7 100644 --- a/lib/cmetrics/src/cmt_mpack_utils.c +++ b/lib/cmetrics/src/cmt_mpack_utils.c @@ -47,6 +47,44 @@ int cmt_mpack_consume_double_tag(mpack_reader_t *reader, double *output_buffer) return CMT_MPACK_SUCCESS; } +int cmt_mpack_consume_int_tag(mpack_reader_t *reader, int64_t *output_buffer) +{ + uint64_t unsigned_value; + mpack_tag_t tag; + + if (NULL == output_buffer) { + return CMT_MPACK_INVALID_ARGUMENT_ERROR; + } + + if (NULL == reader) { + return CMT_MPACK_INVALID_ARGUMENT_ERROR; + } + + tag = mpack_read_tag(reader); + + if (mpack_ok != mpack_reader_error(reader)) { + return CMT_MPACK_ENGINE_ERROR; + } + + if (mpack_type_int == mpack_tag_type(&tag)) { + *output_buffer = mpack_tag_int_value(&tag); + } + else if (mpack_type_uint == mpack_tag_type(&tag)) { + unsigned_value = mpack_tag_uint_value(&tag); + + if (unsigned_value > INT64_MAX) { + return CMT_MPACK_UNEXPECTED_DATA_TYPE_ERROR; + } + + *output_buffer = (int64_t) unsigned_value; + } + else { + return CMT_MPACK_UNEXPECTED_DATA_TYPE_ERROR; + } + + return CMT_MPACK_SUCCESS; +} + int cmt_mpack_consume_uint_tag(mpack_reader_t *reader, uint64_t *output_buffer) { mpack_tag_t tag; diff --git a/lib/cmetrics/tests/CMakeLists.txt b/lib/cmetrics/tests/CMakeLists.txt index c71f8a80540..40108aad95a 100644 --- a/lib/cmetrics/tests/CMakeLists.txt +++ b/lib/cmetrics/tests/CMakeLists.txt @@ -8,10 +8,13 @@ set(UNIT_TESTS_FILES atomic_operations.c encoding.c decoding.c + opentelemetry.c cat.c issues.c null_label.c filter.c + exp_histogram.c + msgpack_abi.c ) if (CMT_BUILD_PROMETHEUS_TEXT_DECODER) diff --git a/lib/cmetrics/tests/decoding.c b/lib/cmetrics/tests/decoding.c index 6c70811698c..d73e46a12ee 100644 --- a/lib/cmetrics/tests/decoding.c +++ b/lib/cmetrics/tests/decoding.c @@ -18,169 +18,12 @@ */ #include -#include -#include -#include -#include -#include #include -#include -#include #include -#include #include #include "cmt_tests.h" -static struct cmt *generate_encoder_test_data() -{ - double quantiles[5]; - struct cmt_histogram_buckets *buckets; - double val; - struct cmt *cmt; - uint64_t ts; - struct cmt_gauge *g1; - struct cmt_counter *c1; - struct cmt_summary *s1; - struct cmt_histogram *h1; - - ts = 0; - cmt = cmt_create(); - - c1 = cmt_counter_create(cmt, "kubernetes", "network", "load_counter", "Network load counter", - 2, (char *[]) {"hostname", "app"}); - - cmt_counter_get_val(c1, 0, NULL, &val); - cmt_counter_inc(c1, ts, 0, NULL); - cmt_counter_add(c1, ts, 2, 0, NULL); - cmt_counter_get_val(c1, 0, NULL, &val); - - cmt_counter_inc(c1, ts, 2, (char *[]) {"localhost", "cmetrics"}); - cmt_counter_get_val(c1, 2, (char *[]) {"localhost", "cmetrics"}, &val); - cmt_counter_add(c1, ts, 10.55, 2, (char *[]) {"localhost", "test"}); - cmt_counter_get_val(c1, 2, (char *[]) {"localhost", "test"}, &val); - cmt_counter_set(c1, ts, 12.15, 2, (char *[]) {"localhost", "test"}); - cmt_counter_set(c1, ts, 1, 2, (char *[]) {"localhost", "test"}); - - g1 = cmt_gauge_create(cmt, "kubernetes", "network", "load_gauge", "Network load gauge", 0, NULL); - - cmt_gauge_get_val(g1, 0, NULL, &val); - cmt_gauge_set(g1, ts, 2.0, 0, NULL); - cmt_gauge_get_val(g1, 0, NULL, &val); - cmt_gauge_inc(g1, ts, 0, NULL); - cmt_gauge_get_val(g1, 0, NULL, &val); - cmt_gauge_sub(g1, ts, 2, 0, NULL); - cmt_gauge_get_val(g1, 0, NULL, &val); - cmt_gauge_dec(g1, ts, 0, NULL); - cmt_gauge_get_val(g1, 0, NULL, &val); - cmt_gauge_inc(g1, ts, 0, NULL); - - buckets = cmt_histogram_buckets_create(3, 0.05, 5.0, 10.0); - - h1 = cmt_histogram_create(cmt, - "k8s", "network", "load_histogram", "Network load histogram", - buckets, - 1, (char *[]) {"my_label"}); - - cmt_histogram_observe(h1, ts, 0.001, 0, NULL); - cmt_histogram_observe(h1, ts, 0.020, 0, NULL); - cmt_histogram_observe(h1, ts, 5.0, 0, NULL); - cmt_histogram_observe(h1, ts, 8.0, 0, NULL); - cmt_histogram_observe(h1, ts, 1000, 0, NULL); - - cmt_histogram_observe(h1, ts, 0.001, 1, (char *[]) {"my_val"}); - cmt_histogram_observe(h1, ts, 0.020, 1, (char *[]) {"my_val"}); - cmt_histogram_observe(h1, ts, 5.0, 1, (char *[]) {"my_val"}); - cmt_histogram_observe(h1, ts, 8.0, 1, (char *[]) {"my_val"}); - cmt_histogram_observe(h1, ts, 1000, 1, (char *[]) {"my_val"});; - - quantiles[0] = 0.1; - quantiles[1] = 0.2; - quantiles[2] = 0.3; - quantiles[3] = 0.4; - quantiles[4] = 0.5; - - s1 = cmt_summary_create(cmt, - "k8s", "disk", "load_summary", "Disk load summary", - 5, quantiles, - 1, (char *[]) {"my_label"}); - - quantiles[0] = 1.1; - quantiles[1] = 2.2; - quantiles[2] = 3.3; - quantiles[3] = 4.4; - quantiles[4] = 5.5; - - cmt_summary_set_default(s1, ts, quantiles, 51.612894511314444, 10, 0, NULL); - - quantiles[0] = 11.11; - quantiles[1] = 0; - quantiles[2] = 33.33; - quantiles[3] = 44.44; - quantiles[4] = 55.55; - - cmt_summary_set_default(s1, ts, quantiles, 51.612894511314444, 10, 1, (char *[]) {"my_val"}); - - return cmt; -} - -void test_opentelemetry() -{ - cfl_sds_t reference_prometheus_context; - cfl_sds_t opentelemetry_context; - struct cfl_list decoded_context_list; - cfl_sds_t prometheus_context; - struct cmt *decoded_context; - size_t offset; - int result; - struct cmt *cmt; - - offset = 0; - - cmt_initialize(); - - cmt = generate_encoder_test_data(); - TEST_CHECK(cmt != NULL); - - reference_prometheus_context = cmt_encode_prometheus_create(cmt, CMT_TRUE); - TEST_CHECK(reference_prometheus_context != NULL); - - if (reference_prometheus_context != NULL) { - opentelemetry_context = cmt_encode_opentelemetry_create(cmt); - TEST_CHECK(opentelemetry_context != NULL); - - if (opentelemetry_context != NULL) { - result = cmt_decode_opentelemetry_create(&decoded_context_list, - opentelemetry_context, - cfl_sds_len(opentelemetry_context), - &offset); - - if (TEST_CHECK(result == 0)) { - decoded_context = cfl_list_entry_first(&decoded_context_list, struct cmt, _head); - - if (TEST_CHECK(result == 0)) { - prometheus_context = cmt_encode_prometheus_create(decoded_context, - CMT_TRUE); - TEST_CHECK(prometheus_context != NULL); - - if (prometheus_context != NULL) { - TEST_CHECK(strcmp(prometheus_context, - reference_prometheus_context) == 0); - - cmt_encode_prometheus_destroy(prometheus_context); - } - } - - cmt_decode_opentelemetry_destroy(&decoded_context_list); - } - } - - cmt_encode_opentelemetry_destroy(opentelemetry_context); - cmt_encode_prometheus_destroy(reference_prometheus_context); - } - - cmt_destroy(cmt); -} void test_prometheus_remote_write() { @@ -230,7 +73,6 @@ void test_statsd() TEST_LIST = { - {"opentelemetry", test_opentelemetry}, {"prometheus_remote_write", test_prometheus_remote_write}, {"statsd", test_statsd}, { 0 } diff --git a/lib/cmetrics/tests/encoding.c b/lib/cmetrics/tests/encoding.c index b962a468bd3..f0553528159 100644 --- a/lib/cmetrics/tests/encoding.c +++ b/lib/cmetrics/tests/encoding.c @@ -1008,7 +1008,7 @@ void test_splunk_hec_histogram() "{\"host\":\"localhost\",\"time\":1435658235.000000123,\"event\":\"metric\",\"index\":\"fluent-bit-metrics\",\"source\":\"fluent-bit-cmetrics\",\"sourcetype\":\"cmetrics\",\"fields\":{\"metric_name:network.load_bucket\":5.0,\"le\":\"5.0\",\"metric_type\":\"Histogram\"}}" "{\"host\":\"localhost\",\"time\":1435658235.000000123,\"event\":\"metric\",\"index\":\"fluent-bit-metrics\",\"source\":\"fluent-bit-cmetrics\",\"sourcetype\":\"cmetrics\",\"fields\":{\"metric_name:network.load_bucket\":10.0,\"le\":\"10.0\",\"metric_type\":\"Histogram\"}}" "{\"host\":\"localhost\",\"time\":1435658235.000000123,\"event\":\"metric\",\"index\":\"fluent-bit-metrics\",\"source\":\"fluent-bit-cmetrics\",\"sourcetype\":\"cmetrics\",\"fields\":{\"metric_name:network.load_bucket\":10.0,\"le\":\"+Inf\",\"metric_type\":\"Histogram\"}}" - "{\"host\":\"localhost\",\"time\":1435658235.000000123,\"event\":\"metric\",\"index\":\"fluent-bit-metrics\",\"source\":\"fluent-bit-cmetrics\",\"sourcetype\":\"cmetrics\",\"fields\":{\"metric_name:network.load_sum\":45.0,\"metric_type\":\"Histogram\"}}" + "{\"host\":\"localhost\",\"time\":1435658235.000000123,\"event\":\"metric\",\"index\":\"fluent-bit-metrics\",\"source\":\"fluent-bit-cmetrics\",\"sourcetype\":\"cmetrics\",\"fields\":{\"metric_name:network.load_sum\":45.9,\"metric_type\":\"Histogram\"}}" "{\"host\":\"localhost\",\"time\":1435658235.000000123,\"event\":\"metric\",\"index\":\"fluent-bit-metrics\",\"source\":\"fluent-bit-cmetrics\",\"sourcetype\":\"cmetrics\",\"fields\":{\"metric_name:network.load_count\":10.0,\"metric_type\":\"Histogram\"}}"; char *out2 = "{\"host\":\"localhost\",\"time\":1435658235.000000123,\"event\":\"metric\",\"index\":\"fluent-bit-metrics\",\"source\":\"fluent-bit-cmetrics\",\"sourcetype\":\"cmetrics\",\"fields\":{\"metric_name:network.load_bucket\":1.0,\"le\":\"0.005\",\"static\":\"test\",\"metric_type\":\"Histogram\"}}" @@ -1023,7 +1023,7 @@ void test_splunk_hec_histogram() "{\"host\":\"localhost\",\"time\":1435658235.000000123,\"event\":\"metric\",\"index\":\"fluent-bit-metrics\",\"source\":\"fluent-bit-cmetrics\",\"sourcetype\":\"cmetrics\",\"fields\":{\"metric_name:network.load_bucket\":5.0,\"le\":\"5.0\",\"static\":\"test\",\"metric_type\":\"Histogram\"}}" "{\"host\":\"localhost\",\"time\":1435658235.000000123,\"event\":\"metric\",\"index\":\"fluent-bit-metrics\",\"source\":\"fluent-bit-cmetrics\",\"sourcetype\":\"cmetrics\",\"fields\":{\"metric_name:network.load_bucket\":10.0,\"le\":\"10.0\",\"static\":\"test\",\"metric_type\":\"Histogram\"}}" "{\"host\":\"localhost\",\"time\":1435658235.000000123,\"event\":\"metric\",\"index\":\"fluent-bit-metrics\",\"source\":\"fluent-bit-cmetrics\",\"sourcetype\":\"cmetrics\",\"fields\":{\"metric_name:network.load_bucket\":10.0,\"le\":\"+Inf\",\"static\":\"test\",\"metric_type\":\"Histogram\"}}" - "{\"host\":\"localhost\",\"time\":1435658235.000000123,\"event\":\"metric\",\"index\":\"fluent-bit-metrics\",\"source\":\"fluent-bit-cmetrics\",\"sourcetype\":\"cmetrics\",\"fields\":{\"metric_name:network.load_sum\":45.0,\"static\":\"test\",\"metric_type\":\"Histogram\"}}" + "{\"host\":\"localhost\",\"time\":1435658235.000000123,\"event\":\"metric\",\"index\":\"fluent-bit-metrics\",\"source\":\"fluent-bit-cmetrics\",\"sourcetype\":\"cmetrics\",\"fields\":{\"metric_name:network.load_sum\":45.9,\"static\":\"test\",\"metric_type\":\"Histogram\"}}" "{\"host\":\"localhost\",\"time\":1435658235.000000123,\"event\":\"metric\",\"index\":\"fluent-bit-metrics\",\"source\":\"fluent-bit-cmetrics\",\"sourcetype\":\"cmetrics\",\"fields\":{\"metric_name:network.load_count\":10.0,\"static\":\"test\",\"metric_type\":\"Histogram\"}}"; cmt_initialize(); diff --git a/lib/cmetrics/tests/exp_histogram.c b/lib/cmetrics/tests/exp_histogram.c new file mode 100644 index 00000000000..afec47fb860 --- /dev/null +++ b/lib/cmetrics/tests/exp_histogram.c @@ -0,0 +1,801 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2026 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "cmt_tests.h" + +static void print_splunk_payload(cfl_sds_t payload) +{ + size_t index; + + for (index = 0; index < cfl_sds_len(payload); index++) { + putchar(payload[index]); + + if (payload[index] == '}' && + index + 1 < cfl_sds_len(payload) && + payload[index + 1] == '{') { + putchar('\n'); + } + } + + putchar('\n'); +} + +static struct cmt_exp_histogram *create_test_metric(struct cmt *cmt, uint64_t timestamp) +{ + int result; + uint64_t positive[3] = {3, 5, 7}; + uint64_t negative[2] = {2, 1}; + struct cmt_exp_histogram *exp_histogram; + + exp_histogram = cmt_exp_histogram_create(cmt, + "cm", "native", "exp_hist", "native exp histogram", + 1, (char *[]) {"endpoint"}); + TEST_CHECK(exp_histogram != NULL); + + if (exp_histogram == NULL) { + return NULL; + } + + result = cmt_exp_histogram_set_default(exp_histogram, + timestamp, + 2, + 11, + 0.0, + -2, + 3, + positive, + -1, + 2, + negative, + CMT_TRUE, + 42.25, + 29, + 1, + (char *[]) {"api"}); + TEST_CHECK(result == 0); + + if (result != 0) { + cmt_exp_histogram_destroy(exp_histogram); + return NULL; + } + + return exp_histogram; +} + +static struct cmt_exp_histogram *create_test_metric_with_zero_threshold( + struct cmt *cmt, + uint64_t timestamp, + double zero_threshold, + uint64_t zero_count, + double sum, + uint64_t count) +{ + int result; + uint64_t positive[3] = {3, 5, 7}; + uint64_t negative[2] = {2, 1}; + struct cmt_exp_histogram *exp_histogram; + + exp_histogram = cmt_exp_histogram_create(cmt, + "cm", "native", "exp_hist", "native exp histogram", + 1, (char *[]) {"endpoint"}); + TEST_CHECK(exp_histogram != NULL); + + if (exp_histogram == NULL) { + return NULL; + } + + result = cmt_exp_histogram_set_default(exp_histogram, + timestamp, + 2, + zero_count, + zero_threshold, + -2, + 3, + positive, + -1, + 2, + negative, + CMT_TRUE, + sum, + count, + 1, + (char *[]) {"api"}); + TEST_CHECK(result == 0); + + if (result != 0) { + cmt_exp_histogram_destroy(exp_histogram); + return NULL; + } + + return exp_histogram; +} + +static struct cmt_exp_histogram *create_test_metric_custom( + struct cmt *cmt, + uint64_t timestamp, + int32_t scale, + uint64_t zero_count, + double zero_threshold, + int32_t positive_offset, + size_t positive_count, + uint64_t *positive_buckets, + int32_t negative_offset, + size_t negative_count, + uint64_t *negative_buckets, + int sum_set, + double sum, + uint64_t count) +{ + int result; + struct cmt_exp_histogram *exp_histogram; + + exp_histogram = cmt_exp_histogram_create(cmt, + "cm", "native", "exp_hist", "native exp histogram", + 1, (char *[]) {"endpoint"}); + TEST_CHECK(exp_histogram != NULL); + + if (exp_histogram == NULL) { + return NULL; + } + + result = cmt_exp_histogram_set_default(exp_histogram, + timestamp, + scale, + zero_count, + zero_threshold, + positive_offset, + positive_count, + positive_buckets, + negative_offset, + negative_count, + negative_buckets, + sum_set, + sum, + count, + 1, + (char *[]) {"api"}); + TEST_CHECK(result == 0); + + if (result != 0) { + cmt_exp_histogram_destroy(exp_histogram); + return NULL; + } + + return exp_histogram; +} + +static int get_prometheus_bucket_value(cfl_sds_t encoded_prometheus, + const char *le, + double *out_value) +{ + char needle[128]; + char *cursor; + + snprintf(needle, sizeof(needle) - 1, + "cm_native_exp_hist_bucket{le=\"%s\",endpoint=\"api\"} ", + le); + + cursor = strstr(encoded_prometheus, needle); + if (cursor == NULL) { + return -1; + } + + cursor += strlen(needle); + *out_value = strtod(cursor, NULL); + + return 0; +} + +static int remote_write_contains_metric_name(Prometheus__WriteRequest *request, + const char *metric_name) +{ + size_t index; + size_t label_index; + + for (index = 0; index < request->n_timeseries; index++) { + if (request->timeseries[index] == NULL) { + continue; + } + + for (label_index = 0; label_index < request->timeseries[index]->n_labels; label_index++) { + if (request->timeseries[index]->labels[label_index] == NULL) { + continue; + } + + if (request->timeseries[index]->labels[label_index]->name != NULL && + request->timeseries[index]->labels[label_index]->value != NULL && + strcmp(request->timeseries[index]->labels[label_index]->name, "__name__") == 0 && + strcmp(request->timeseries[index]->labels[label_index]->value, metric_name) == 0) { + return CMT_TRUE; + } + } + } + + return CMT_FALSE; +} + +static int assert_prometheus_bucket_monotonicity(cfl_sds_t encoded_prometheus, + double *out_last_finite, + double *out_plus_inf) +{ + char *cursor; + char *line_end; + char *value_cursor; + char *parsed_end; + char le_buffer[64]; + size_t le_length; + double previous_value; + double current_value; + int found_any; + + previous_value = -1.0; + *out_last_finite = -1.0; + *out_plus_inf = -1.0; + found_any = CMT_FALSE; + + cursor = encoded_prometheus; + + while (cursor != NULL) { + cursor = strstr(cursor, "cm_native_exp_hist_bucket{le=\""); + if (cursor == NULL) { + break; + } + + cursor += strlen("cm_native_exp_hist_bucket{le=\""); + line_end = strstr(cursor, "\""); + if (line_end == NULL) { + return -1; + } + + le_length = line_end - cursor; + if (le_length >= sizeof(le_buffer)) { + return -1; + } + + memcpy(le_buffer, cursor, le_length); + le_buffer[le_length] = '\0'; + + value_cursor = strstr(line_end, "} "); + if (value_cursor == NULL) { + return -1; + } + value_cursor += 2; + + current_value = strtod(value_cursor, &parsed_end); + if (parsed_end == value_cursor) { + return -1; + } + + if (found_any && current_value < previous_value) { + return -1; + } + + if (strcmp(le_buffer, "+Inf") == 0) { + *out_plus_inf = current_value; + } + else { + *out_last_finite = current_value; + } + + previous_value = current_value; + found_any = CMT_TRUE; + cursor = parsed_end; + } + + if (!found_any || *out_plus_inf < 0.0 || *out_last_finite < 0.0) { + return -1; + } + + if (*out_last_finite > *out_plus_inf) { + return -1; + } + + return 0; +} + +void test_exp_histogram_msgpack_roundtrip() +{ + int result; + size_t offset; + char *packed_buffer; + size_t packed_size; + struct cmt *input_context; + struct cmt *output_context; + struct cmt_exp_histogram *exp_histogram; + struct cmt_metric *metric; + + cmt_initialize(); + + input_context = cmt_create(); + TEST_CHECK(input_context != NULL); + + exp_histogram = create_test_metric(input_context, 123); + TEST_CHECK(exp_histogram != NULL); + + result = cmt_encode_msgpack_create(input_context, &packed_buffer, &packed_size); + TEST_CHECK(result == CMT_DECODE_MSGPACK_SUCCESS); + + offset = 0; + output_context = NULL; + result = cmt_decode_msgpack_create(&output_context, packed_buffer, packed_size, &offset); + TEST_CHECK(result == CMT_DECODE_MSGPACK_SUCCESS); + TEST_CHECK(output_context != NULL); + TEST_CHECK(cfl_list_size(&output_context->exp_histograms) == 1); + + exp_histogram = cfl_list_entry_first(&output_context->exp_histograms, + struct cmt_exp_histogram, _head); + metric = cmt_map_metric_get(&exp_histogram->opts, exp_histogram->map, + 1, (char *[]) {"api"}, CMT_FALSE); + TEST_CHECK(metric != NULL); + + if (metric != NULL) { + printf("\n========== EXP HIST MSGPACK ROUNDTRIP ==========\n"); + printf("scale=%d zero_count=%" PRIu64 " count=%" PRIu64 " sum=%.17g\n\n", + metric->exp_hist_scale, + metric->exp_hist_zero_count, + metric->exp_hist_count, + cmt_math_uint64_to_d64(metric->exp_hist_sum)); + + TEST_CHECK(metric->exp_hist_scale == 2); + TEST_CHECK(metric->exp_hist_zero_count == 11); + TEST_CHECK(metric->exp_hist_positive_offset == -2); + TEST_CHECK(metric->exp_hist_negative_offset == -1); + TEST_CHECK(metric->exp_hist_positive_count == 3); + TEST_CHECK(metric->exp_hist_negative_count == 2); + TEST_CHECK(metric->exp_hist_count == 29); + TEST_CHECK(metric->exp_hist_sum_set == CMT_TRUE); + TEST_CHECK(fabs(cmt_math_uint64_to_d64(metric->exp_hist_sum) - 42.25) < 0.00001); + TEST_CHECK(metric->exp_hist_positive_buckets != NULL); + TEST_CHECK(metric->exp_hist_negative_buckets != NULL); + + if (metric->exp_hist_positive_buckets != NULL && + metric->exp_hist_negative_buckets != NULL) { + TEST_CHECK(metric->exp_hist_positive_buckets[0] == 3); + TEST_CHECK(metric->exp_hist_positive_buckets[1] == 5); + TEST_CHECK(metric->exp_hist_positive_buckets[2] == 7); + TEST_CHECK(metric->exp_hist_negative_buckets[0] == 2); + TEST_CHECK(metric->exp_hist_negative_buckets[1] == 1); + } + } + + cmt_destroy(input_context); + cmt_decode_msgpack_destroy(output_context); + cmt_encode_msgpack_destroy(packed_buffer); +} + +void test_exp_histogram_encoder_smoke() +{ + int result; + uint64_t timestamp; + size_t packed_size; + char *packed_buffer; + cfl_sds_t encoded_text; + cfl_sds_t encoded_prometheus; + cfl_sds_t encoded_influx; + cfl_sds_t encoded_splunk_hec; + cfl_sds_t encoded_remote_write = NULL; + cfl_sds_t encoded_opentelemetry = NULL; + char *encoded_cloudwatch_emf = NULL; + size_t encoded_cloudwatch_emf_size; + struct cmt *context; + + cmt_initialize(); + timestamp = cfl_time_now(); + + context = cmt_create(); + TEST_CHECK(context != NULL); + + TEST_CHECK(create_test_metric(context, timestamp) != NULL); + + result = cmt_encode_msgpack_create(context, &packed_buffer, &packed_size); + TEST_CHECK(result == 0); + cmt_encode_msgpack_destroy(packed_buffer); + + encoded_text = cmt_encode_text_create(context); + TEST_CHECK(encoded_text != NULL); + if (encoded_text != NULL) { + printf("\n========== EXP HIST TEXT ==========\n%s\n", encoded_text); + TEST_CHECK(strstr(encoded_text, "scale=2") != NULL); + TEST_CHECK(strstr(encoded_text, "zero_count=11") != NULL); + TEST_CHECK(strstr(encoded_text, "positive={offset=-2, bucket_counts=[3, 5, 7]}") != NULL); + TEST_CHECK(strstr(encoded_text, "negative={offset=-1, bucket_counts=[2, 1]}") != NULL); + TEST_CHECK(strstr(encoded_text, "count=29") != NULL); + TEST_CHECK(strstr(encoded_text, "sum=42.25") != NULL); + } + cmt_encode_text_destroy(encoded_text); + + encoded_prometheus = cmt_encode_prometheus_create(context, CMT_TRUE); + TEST_CHECK(encoded_prometheus != NULL); + if (encoded_prometheus != NULL) { + double last_finite_bucket; + double plus_inf_bucket; + + printf("\n========== EXP HIST PROMETHEUS ==========\n%s\n", encoded_prometheus); + + result = assert_prometheus_bucket_monotonicity(encoded_prometheus, + &last_finite_bucket, + &plus_inf_bucket); + TEST_CHECK(result == 0); + if (result == 0) { + TEST_CHECK(plus_inf_bucket == 29.0); + TEST_CHECK(last_finite_bucket <= plus_inf_bucket); + } + } + cmt_encode_prometheus_destroy(encoded_prometheus); + + encoded_influx = cmt_encode_influx_create(context); + TEST_CHECK(encoded_influx != NULL); + if (encoded_influx != NULL) { + printf("\n========== EXP HIST INFLUX ==========\n%s\n", encoded_influx); + } + cmt_encode_influx_destroy(encoded_influx); + + encoded_splunk_hec = cmt_encode_splunk_hec_create(context, + "localhost", "test-index", + NULL, NULL); + TEST_CHECK(encoded_splunk_hec != NULL); + if (encoded_splunk_hec != NULL) { + printf("\n========== EXP HIST SPLUNK HEC ==========\n"); + print_splunk_payload(encoded_splunk_hec); + printf("\n"); + } + cmt_encode_splunk_hec_destroy(encoded_splunk_hec); + + result = cmt_encode_cloudwatch_emf_create(context, + &encoded_cloudwatch_emf, + &encoded_cloudwatch_emf_size, + CMT_TRUE); + TEST_CHECK(result == 0); + if (result == 0) { + printf("========== EXP HIST CLOUDWATCH EMF ==========\n"); + printf("payload_size=%zu\n\n", encoded_cloudwatch_emf_size); + } + if (encoded_cloudwatch_emf != NULL) { + cmt_encode_cloudwatch_emf_destroy(encoded_cloudwatch_emf); + } + + encoded_remote_write = cmt_encode_prometheus_remote_write_create(context); + TEST_CHECK(encoded_remote_write != NULL); + if (encoded_remote_write != NULL) { + printf("========== EXP HIST REMOTE WRITE ==========\n"); + printf("payload_size=%zu\n\n", cfl_sds_len(encoded_remote_write)); + } + if (encoded_remote_write != NULL) { + cmt_encode_prometheus_remote_write_destroy(encoded_remote_write); + } + + encoded_opentelemetry = cmt_encode_opentelemetry_create(context); + TEST_CHECK(encoded_opentelemetry != NULL); + if (encoded_opentelemetry != NULL) { + printf("========== EXP HIST OPENTELEMETRY ==========\n"); + printf("payload_size=%zu\n\n", cfl_sds_len(encoded_opentelemetry)); + } + if (encoded_opentelemetry != NULL) { + cmt_encode_opentelemetry_destroy(encoded_opentelemetry); + } + + cmt_destroy(context); +} + +void test_exp_histogram_nonzero_zero_threshold() +{ + int result; + double bucket_value; + cfl_sds_t encoded_text; + cfl_sds_t encoded_prometheus; + cfl_sds_t encoded_influx; + cfl_sds_t encoded_splunk_hec; + struct cmt *context; + + cmt_initialize(); + + context = cmt_create(); + TEST_CHECK(context != NULL); + + TEST_CHECK(create_test_metric_with_zero_threshold(context, + cfl_time_now(), + 0.5, + 4, + 42.25, + 22) != NULL); + + encoded_text = cmt_encode_text_create(context); + TEST_CHECK(encoded_text != NULL); + if (encoded_text != NULL) { + printf("\n========== EXP HIST NON-ZERO ZERO_THRESHOLD TEXT ==========\n%s\n", + encoded_text); + TEST_CHECK(strstr(encoded_text, "zero_threshold=0.5") != NULL); + } + cmt_encode_text_destroy(encoded_text); + + encoded_prometheus = cmt_encode_prometheus_create(context, CMT_TRUE); + TEST_CHECK(encoded_prometheus != NULL); + if (encoded_prometheus != NULL) { + double last_finite_bucket; + double plus_inf_bucket; + + printf("\n========== EXP HIST NON-ZERO ZERO_THRESHOLD PROMETHEUS ==========\n%s\n", + encoded_prometheus); + + result = assert_prometheus_bucket_monotonicity(encoded_prometheus, + &last_finite_bucket, + &plus_inf_bucket); + TEST_CHECK(result == 0); + if (result == 0) { + TEST_CHECK(plus_inf_bucket == 22.0); + TEST_CHECK(last_finite_bucket <= plus_inf_bucket); + } + + result = get_prometheus_bucket_value(encoded_prometheus, "0.0", &bucket_value); + TEST_CHECK(result == 0); + if (result == 0) { + TEST_CHECK(bucket_value == 7.0); + } + + result = get_prometheus_bucket_value(encoded_prometheus, "+Inf", &bucket_value); + TEST_CHECK(result == 0); + if (result == 0) { + TEST_CHECK(bucket_value == 22.0); + } + + TEST_CHECK(strstr(encoded_prometheus, "cm_native_exp_hist_sum{endpoint=\"api\"} 42.25 ") != NULL); + } + cmt_encode_prometheus_destroy(encoded_prometheus); + + encoded_influx = cmt_encode_influx_create(context); + TEST_CHECK(encoded_influx != NULL); + if (encoded_influx != NULL) { + printf("\n========== EXP HIST NON-ZERO ZERO_THRESHOLD INFLUX ==========\n%s\n", + encoded_influx); + TEST_CHECK(strstr(encoded_influx, "sum=42.25") != NULL); + TEST_CHECK(strstr(encoded_influx, "+Inf=22") != NULL); + } + cmt_encode_influx_destroy(encoded_influx); + + encoded_splunk_hec = cmt_encode_splunk_hec_create(context, + "localhost", "test-index", + NULL, NULL); + TEST_CHECK(encoded_splunk_hec != NULL); + if (encoded_splunk_hec != NULL) { + printf("\n========== EXP HIST NON-ZERO ZERO_THRESHOLD SPLUNK HEC ==========\n"); + print_splunk_payload(encoded_splunk_hec); + printf("\n"); + TEST_CHECK(strstr(encoded_splunk_hec, "\"metric_name:native.exp_hist_sum\":42.25") != NULL); + TEST_CHECK(strstr(encoded_splunk_hec, "\"metric_name:native.exp_hist_count\":22.0") != NULL); + } + cmt_encode_splunk_hec_destroy(encoded_splunk_hec); + + cmt_destroy(context); +} + +void test_exp_histogram_cat_filter_smoke() +{ + int result; + struct cmt *source; + struct cmt *cat_target; + struct cmt *filter_target; + + cmt_initialize(); + + source = cmt_create(); + cat_target = cmt_create(); + filter_target = cmt_create(); + + TEST_CHECK(source != NULL); + TEST_CHECK(cat_target != NULL); + TEST_CHECK(filter_target != NULL); + TEST_CHECK(create_test_metric(source, cfl_time_now()) != NULL); + + result = cmt_cat(cat_target, source); + TEST_CHECK(result == 0); + TEST_CHECK(cfl_list_size(&cat_target->exp_histograms) == 1); + printf("\n========== EXP HIST CAT ==========\n"); + printf("exp_histograms=%d\n\n", cfl_list_size(&cat_target->exp_histograms)); + + result = cmt_filter(filter_target, source, "cm_native_exp", NULL, + NULL, NULL, CMT_FILTER_PREFIX); + TEST_CHECK(result == 0); + TEST_CHECK(cfl_list_size(&filter_target->exp_histograms) == 1); + printf("========== EXP HIST FILTER ==========\n"); + printf("exp_histograms=%d\n\n", cfl_list_size(&filter_target->exp_histograms)); + + cmt_destroy(source); + cmt_destroy(cat_target); + cmt_destroy(filter_target); +} + +void test_exp_histogram_cat_sparse_merge() +{ + int result; + uint64_t positive_a[3] = {3, 5, 7}; + uint64_t negative_a[2] = {2, 1}; + uint64_t positive_b[2] = {10, 11}; + uint64_t negative_b[3] = {4, 5, 6}; + struct cmt *source_a; + struct cmt *source_b; + struct cmt *target; + struct cmt_exp_histogram *exp_histogram; + struct cmt_metric *metric; + + cmt_initialize(); + + source_a = cmt_create(); + source_b = cmt_create(); + target = cmt_create(); + + TEST_CHECK(source_a != NULL); + TEST_CHECK(source_b != NULL); + TEST_CHECK(target != NULL); + + TEST_CHECK(create_test_metric_custom(source_a, cfl_time_now(), + 2, 4, 0.0, + -2, 3, positive_a, + -1, 2, negative_a, + CMT_TRUE, 42.25, 22) != NULL); + TEST_CHECK(create_test_metric_custom(source_b, cfl_time_now(), + 2, 1, 0.0, + -1, 2, positive_b, + -3, 3, negative_b, + CMT_TRUE, 10.5, 31) != NULL); + + result = cmt_cat(target, source_a); + TEST_CHECK(result == 0); + result = cmt_cat(target, source_b); + TEST_CHECK(result == 0); + + TEST_CHECK(cfl_list_size(&target->exp_histograms) == 1); + + exp_histogram = cfl_list_entry_first(&target->exp_histograms, + struct cmt_exp_histogram, _head); + metric = cmt_map_metric_get(&exp_histogram->opts, exp_histogram->map, + 1, (char *[]) {"api"}, CMT_FALSE); + TEST_CHECK(metric != NULL); + + if (metric != NULL) { + TEST_CHECK(metric->exp_hist_scale == 2); + TEST_CHECK(metric->exp_hist_zero_threshold == 0.0); + TEST_CHECK(metric->exp_hist_zero_count == 5); + TEST_CHECK(metric->exp_hist_count == 53); + TEST_CHECK(metric->exp_hist_sum_set == CMT_TRUE); + TEST_CHECK(fabs(cmt_math_uint64_to_d64(metric->exp_hist_sum) - 52.75) < 0.00001); + + TEST_CHECK(metric->exp_hist_positive_offset == -2); + TEST_CHECK(metric->exp_hist_positive_count == 3); + TEST_CHECK(metric->exp_hist_positive_buckets != NULL); + if (metric->exp_hist_positive_buckets != NULL) { + TEST_CHECK(metric->exp_hist_positive_buckets[0] == 3); + TEST_CHECK(metric->exp_hist_positive_buckets[1] == 15); + TEST_CHECK(metric->exp_hist_positive_buckets[2] == 18); + } + + TEST_CHECK(metric->exp_hist_negative_offset == -3); + TEST_CHECK(metric->exp_hist_negative_count == 4); + TEST_CHECK(metric->exp_hist_negative_buckets != NULL); + if (metric->exp_hist_negative_buckets != NULL) { + TEST_CHECK(metric->exp_hist_negative_buckets[0] == 4); + TEST_CHECK(metric->exp_hist_negative_buckets[1] == 5); + TEST_CHECK(metric->exp_hist_negative_buckets[2] == 8); + TEST_CHECK(metric->exp_hist_negative_buckets[3] == 1); + } + } + + cmt_destroy(source_a); + cmt_destroy(source_b); + cmt_destroy(target); +} + +void test_exp_histogram_prometheus_no_sum() +{ + uint64_t positive[3] = {3, 5, 7}; + uint64_t negative[2] = {2, 1}; + cfl_sds_t encoded_prometheus; + struct cmt *context; + + cmt_initialize(); + + context = cmt_create(); + TEST_CHECK(context != NULL); + + TEST_CHECK(create_test_metric_custom(context, cfl_time_now(), + 2, 11, 0.0, + -2, 3, positive, + -1, 2, negative, + CMT_FALSE, 123.75, 29) != NULL); + + encoded_prometheus = cmt_encode_prometheus_create(context, CMT_TRUE); + TEST_CHECK(encoded_prometheus != NULL); + if (encoded_prometheus != NULL) { + TEST_CHECK(strstr(encoded_prometheus, "cm_native_exp_hist_count{endpoint=\"api\"} 29 ") != NULL); + TEST_CHECK(strstr(encoded_prometheus, "cm_native_exp_hist_sum{endpoint=\"api\"}") == NULL); + } + cmt_encode_prometheus_destroy(encoded_prometheus); + + cmt_destroy(context); +} + +void test_exp_histogram_remote_write_no_sum() +{ + uint64_t positive[3] = {3, 5, 7}; + uint64_t negative[2] = {2, 1}; + cfl_sds_t encoded_remote_write; + struct cmt *context; + Prometheus__WriteRequest *request; + + cmt_initialize(); + + context = cmt_create(); + TEST_CHECK(context != NULL); + if (context == NULL) { + return; + } + + TEST_CHECK(create_test_metric_custom(context, cfl_time_now(), + 2, 11, 0.0, + -2, 3, positive, + -1, 2, negative, + CMT_FALSE, 123.75, 29) != NULL); + + encoded_remote_write = cmt_encode_prometheus_remote_write_create(context); + TEST_CHECK(encoded_remote_write != NULL); + if (encoded_remote_write != NULL) { + request = prometheus__write_request__unpack(NULL, + cfl_sds_len(encoded_remote_write), + (uint8_t *) encoded_remote_write); + TEST_CHECK(request != NULL); + if (request != NULL) { + TEST_CHECK(remote_write_contains_metric_name(request, "cm_native_exp_hist_count") == CMT_TRUE); + TEST_CHECK(remote_write_contains_metric_name(request, "cm_native_exp_hist_sum") == CMT_FALSE); + TEST_CHECK(remote_write_contains_metric_name(request, "cm_native_exp_hist_bucket") == CMT_TRUE); + prometheus__write_request__free_unpacked(request, NULL); + } + } + + cmt_encode_prometheus_remote_write_destroy(encoded_remote_write); + cmt_destroy(context); +} + +TEST_LIST = { + {"exp_histogram_msgpack_roundtrip", test_exp_histogram_msgpack_roundtrip}, + {"exp_histogram_encoder_smoke", test_exp_histogram_encoder_smoke}, + {"exp_histogram_nonzero_zero_threshold", test_exp_histogram_nonzero_zero_threshold}, + {"exp_histogram_cat_filter_smoke", test_exp_histogram_cat_filter_smoke}, + {"exp_histogram_cat_sparse_merge", test_exp_histogram_cat_sparse_merge}, + {"exp_histogram_prometheus_no_sum", test_exp_histogram_prometheus_no_sum}, + {"exp_histogram_remote_write_no_sum", test_exp_histogram_remote_write_no_sum}, + { 0 } +}; diff --git a/lib/cmetrics/tests/msgpack_abi.c b/lib/cmetrics/tests/msgpack_abi.c new file mode 100644 index 00000000000..c9d5d13e318 --- /dev/null +++ b/lib/cmetrics/tests/msgpack_abi.c @@ -0,0 +1,356 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2026 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmt_tests.h" + +static int contains_bytes(const char *buffer, size_t buffer_length, + const char *needle, size_t needle_length) +{ + size_t index; + + if (buffer == NULL || needle == NULL || needle_length == 0 || + buffer_length < needle_length) { + return CMT_FALSE; + } + + for (index = 0; index <= buffer_length - needle_length; index++) { + if (memcmp(&buffer[index], needle, needle_length) == 0) { + return CMT_TRUE; + } + } + + return CMT_FALSE; +} + +static int patch_nth_key_array_size(char *buffer, size_t buffer_size, + const char *key, int occurrence, + uint8_t new_array_tag) +{ + size_t index; + size_t key_len; + int found_count; + + if (buffer == NULL || key == NULL || occurrence <= 0) { + return -1; + } + + key_len = strlen(key); + found_count = 0; + + for (index = 0; index + key_len < buffer_size; index++) { + if (memcmp(&buffer[index], key, key_len) == 0) { + int is_key_token = CMT_FALSE; + + if (index >= 1 && + (unsigned char) buffer[index - 1] == (0xa0 | key_len)) { + is_key_token = CMT_TRUE; + } + else if (index >= 2 && + (unsigned char) buffer[index - 2] == 0xd9 && + (unsigned char) buffer[index - 1] == key_len) { + is_key_token = CMT_TRUE; + } + else if (index >= 3 && + (unsigned char) buffer[index - 3] == 0xda && + (((unsigned char) buffer[index - 2] << 8) | + (unsigned char) buffer[index - 1]) == key_len) { + is_key_token = CMT_TRUE; + } + + if (!is_key_token) { + continue; + } + + found_count++; + if (found_count == occurrence) { + if (index + key_len >= buffer_size) { + return -1; + } + + if (((unsigned char) buffer[index + key_len] & 0xf0) != 0x90) { + return -1; + } + + buffer[index + key_len] = (char) new_array_tag; + return 0; + } + } + } + + return -1; +} + +void test_msgpack_abi_legacy_value_only_decode() +{ + int ret; + size_t offset; + char *payload; + size_t payload_size; + struct cmt *input_cmt; + struct cmt *output_cmt; + struct cmt_gauge *input_gauge; + struct cmt_gauge *output_gauge; + + cmt_initialize(); + + input_cmt = cmt_create(); + TEST_CHECK(input_cmt != NULL); + if (input_cmt == NULL) { + return; + } + + input_gauge = cmt_gauge_create(input_cmt, "ns", "sub", "legacy_value", "legacy", 0, NULL); + TEST_CHECK(input_gauge != NULL); + if (input_gauge == NULL) { + cmt_destroy(input_cmt); + return; + } + + cmt_gauge_set(input_gauge, 111, 42.25, 0, NULL); + + payload = NULL; + payload_size = 0; + ret = cmt_encode_msgpack_create(input_cmt, &payload, &payload_size); + TEST_CHECK(ret == 0); + if (ret != 0) { + cmt_destroy(input_cmt); + return; + } + + TEST_CHECK(contains_bytes(payload, payload_size, "value_int64", 11) == CMT_FALSE); + TEST_CHECK(contains_bytes(payload, payload_size, "value_uint64", 12) == CMT_FALSE); + + offset = 0; + output_cmt = NULL; + ret = cmt_decode_msgpack_create(&output_cmt, payload, payload_size, &offset); + TEST_CHECK(ret == 0); + TEST_CHECK(output_cmt != NULL); + + if (ret == 0 && output_cmt != NULL) { + output_gauge = cfl_list_entry_first(&output_cmt->gauges, struct cmt_gauge, _head); + TEST_CHECK(output_gauge != NULL); + if (output_gauge != NULL) { + TEST_CHECK(output_gauge->map->metric_static_set == CMT_TRUE); + TEST_CHECK(cmt_metric_get_value_type(&output_gauge->map->metric) == CMT_METRIC_VALUE_DOUBLE); + TEST_CHECK(cmt_metric_get_value(&output_gauge->map->metric) == 42.25); + } + } + + cmt_destroy(input_cmt); + if (output_cmt != NULL) { + cmt_decode_msgpack_destroy(output_cmt); + } + cmt_encode_msgpack_destroy(payload); +} + +void test_msgpack_abi_typed_int64_decode() +{ + int ret; + size_t offset; + char *payload; + size_t payload_size; + struct cmt *input_cmt; + struct cmt *output_cmt; + struct cmt_gauge *input_gauge; + struct cmt_gauge *output_gauge; + int64_t expected_value; + + cmt_initialize(); + + input_cmt = cmt_create(); + TEST_CHECK(input_cmt != NULL); + if (input_cmt == NULL) { + return; + } + + input_gauge = cmt_gauge_create(input_cmt, "ns", "sub", "typed_int", "typed", 0, NULL); + TEST_CHECK(input_gauge != NULL); + if (input_gauge == NULL) { + cmt_destroy(input_cmt); + return; + } + + expected_value = 9007199254740993LL; + cmt_gauge_set(input_gauge, 123, 0, 0, NULL); + cmt_metric_set_int64(&input_gauge->map->metric, 123, expected_value); + input_gauge->map->metric_static_set = CMT_TRUE; + + payload = NULL; + payload_size = 0; + ret = cmt_encode_msgpack_create(input_cmt, &payload, &payload_size); + TEST_CHECK(ret == 0); + if (ret != 0) { + cmt_destroy(input_cmt); + return; + } + + TEST_CHECK(contains_bytes(payload, payload_size, "value_type", 10) == CMT_TRUE); + TEST_CHECK(contains_bytes(payload, payload_size, "value_int64", 11) == CMT_TRUE); + + offset = 0; + output_cmt = NULL; + ret = cmt_decode_msgpack_create(&output_cmt, payload, payload_size, &offset); + TEST_CHECK(ret == 0); + TEST_CHECK(output_cmt != NULL); + + if (ret == 0 && output_cmt != NULL) { + output_gauge = cfl_list_entry_first(&output_cmt->gauges, struct cmt_gauge, _head); + TEST_CHECK(output_gauge != NULL); + if (output_gauge != NULL) { + TEST_CHECK(output_gauge->map->metric_static_set == CMT_TRUE); + TEST_CHECK(cmt_metric_get_value_type(&output_gauge->map->metric) == CMT_METRIC_VALUE_INT64); + TEST_CHECK(cmt_metric_get_int64_value(&output_gauge->map->metric) == expected_value); + } + } + + cmt_destroy(input_cmt); + if (output_cmt != NULL) { + cmt_decode_msgpack_destroy(output_cmt); + } + cmt_encode_msgpack_destroy(payload); +} + +void test_msgpack_abi_summary_quantiles_reject_mismatch() +{ + int ret; + size_t offset; + char *payload; + size_t payload_size; + double quantiles[2]; + struct cmt *cmt; + struct cmt_summary *summary; + struct cmt *decoded_cmt; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + if (cmt == NULL) { + return; + } + + quantiles[0] = 0.5; + quantiles[1] = 0.9; + + summary = cmt_summary_create(cmt, "ns", "sub", "sum", "summary", 2, quantiles, 0, NULL); + TEST_CHECK(summary != NULL); + if (summary == NULL) { + cmt_destroy(cmt); + return; + } + + cmt_summary_set_default(summary, 123, quantiles, 1.4, 2, 0, NULL); + + ret = cmt_encode_msgpack_create(cmt, &payload, &payload_size); + TEST_CHECK(ret == 0); + if (ret != 0) { + cmt_destroy(cmt); + return; + } + + ret = patch_nth_key_array_size(payload, payload_size, "quantiles", 2, 0x93); + if (ret != 0) { + ret = patch_nth_key_array_size(payload, payload_size, "quantiles", 1, 0x93); + } + TEST_CHECK(ret == 0); + + offset = 0; + decoded_cmt = NULL; + ret = cmt_decode_msgpack_create(&decoded_cmt, payload, payload_size, &offset); + TEST_CHECK(ret != CMT_DECODE_MSGPACK_SUCCESS); + TEST_CHECK(decoded_cmt == NULL); + + cmt_encode_msgpack_destroy(payload); + cmt_destroy(cmt); +} + +void test_msgpack_abi_histogram_buckets_reject_mismatch() +{ + int ret; + size_t offset; + char *payload; + size_t payload_size; + struct cmt *cmt; + struct cmt_histogram_buckets *buckets; + struct cmt_histogram *histogram; + struct cmt *decoded_cmt; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + if (cmt == NULL) { + return; + } + + buckets = cmt_histogram_buckets_create(2, 1.0, 2.0); + TEST_CHECK(buckets != NULL); + if (buckets == NULL) { + cmt_destroy(cmt); + return; + } + + histogram = cmt_histogram_create(cmt, "ns", "sub", "hist", "hist", buckets, 0, NULL); + TEST_CHECK(histogram != NULL); + if (histogram == NULL) { + cmt_destroy(cmt); + return; + } + + cmt_histogram_observe(histogram, 100, 0.5, 0, NULL); + cmt_histogram_observe(histogram, 100, 1.5, 0, NULL); + cmt_histogram_observe(histogram, 100, 3.0, 0, NULL); + + ret = cmt_encode_msgpack_create(cmt, &payload, &payload_size); + TEST_CHECK(ret == 0); + if (ret != 0) { + cmt_destroy(cmt); + return; + } + + ret = patch_nth_key_array_size(payload, payload_size, "buckets", 2, 0x94); + TEST_CHECK(ret == 0); + + offset = 0; + decoded_cmt = NULL; + ret = cmt_decode_msgpack_create(&decoded_cmt, payload, payload_size, &offset); + TEST_CHECK(ret != CMT_DECODE_MSGPACK_SUCCESS); + TEST_CHECK(decoded_cmt == NULL); + + cmt_encode_msgpack_destroy(payload); + cmt_destroy(cmt); +} + +TEST_LIST = { + {"msgpack_abi_legacy_value_only_decode", test_msgpack_abi_legacy_value_only_decode}, + {"msgpack_abi_typed_int64_decode", test_msgpack_abi_typed_int64_decode}, + {"msgpack_abi_summary_quantiles_reject_mismatch", test_msgpack_abi_summary_quantiles_reject_mismatch}, + {"msgpack_abi_histogram_buckets_reject_mismatch", test_msgpack_abi_histogram_buckets_reject_mismatch}, + {0} +}; diff --git a/lib/cmetrics/tests/opentelemetry.c b/lib/cmetrics/tests/opentelemetry.c new file mode 100644 index 00000000000..4770f6a26b2 --- /dev/null +++ b/lib/cmetrics/tests/opentelemetry.c @@ -0,0 +1,1210 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* CMetrics + * ======== + * Copyright 2026 The CMetrics Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cmt_tests.h" + +static struct cmt *generate_api_test_data() +{ + double quantiles[5]; + struct cmt_histogram_buckets *buckets; + double val; + uint64_t ts; + uint64_t exp_positive[3] = {3, 5, 7}; + uint64_t exp_negative[2] = {2, 1}; + struct cmt *cmt; + struct cmt_counter *counter; + struct cmt_gauge *gauge; + struct cmt_summary *summary; + struct cmt_histogram *histogram; + struct cmt_untyped *untyped; + struct cmt_exp_histogram *exp_histogram; + + ts = 123456789; + cmt = cmt_create(); + if (cmt == NULL) { + return NULL; + } + + counter = cmt_counter_create(cmt, "kubernetes", "network", "load_counter", "Network load counter", + 2, (char *[]) {"hostname", "app"}); + if (counter == NULL) { + cmt_destroy(cmt); + return NULL; + } + + cmt_counter_get_val(counter, 0, NULL, &val); + cmt_counter_inc(counter, ts, 0, NULL); + cmt_counter_add(counter, ts, 2, 0, NULL); + cmt_counter_inc(counter, ts, 2, (char *[]) {"localhost", "cmetrics"}); + cmt_counter_add(counter, ts, 10.55, 2, (char *[]) {"localhost", "test"}); + cmt_counter_set(counter, ts, 12.15, 2, (char *[]) {"localhost", "test"}); + + gauge = cmt_gauge_create(cmt, "kubernetes", "network", "load_gauge", "Network load gauge", 0, NULL); + if (gauge == NULL) { + cmt_destroy(cmt); + return NULL; + } + + cmt_gauge_set(gauge, ts, 2.0, 0, NULL); + cmt_gauge_inc(gauge, ts, 0, NULL); + cmt_gauge_sub(gauge, ts, 5.0, 0, NULL); + + untyped = cmt_untyped_create(cmt, "kubernetes", "network", "load_untyped", "Network load untyped", 0, NULL); + if (untyped == NULL) { + cmt_destroy(cmt); + return NULL; + } + cmt_untyped_set(untyped, ts, -7.0, 0, NULL); + + buckets = cmt_histogram_buckets_create(3, 0.05, 5.0, 10.0); + histogram = cmt_histogram_create(cmt, + "k8s", "network", "load_histogram", "Network load histogram", + buckets, + 1, (char *[]) {"my_label"}); + if (histogram == NULL) { + cmt_destroy(cmt); + return NULL; + } + + cmt_histogram_observe(histogram, ts, 0.001, 0, NULL); + cmt_histogram_observe(histogram, ts, 8.0, 0, NULL); + cmt_histogram_observe(histogram, ts, 1000, 1, (char *[]) {"my_val"}); + + quantiles[0] = 0.1; + quantiles[1] = 0.2; + quantiles[2] = 0.3; + quantiles[3] = 0.4; + quantiles[4] = 0.5; + + summary = cmt_summary_create(cmt, + "k8s", "disk", "load_summary", "Disk load summary", + 5, quantiles, + 1, (char *[]) {"my_label"}); + if (summary == NULL) { + cmt_destroy(cmt); + return NULL; + } + + quantiles[0] = 11.11; + quantiles[1] = 22.22; + quantiles[2] = 33.33; + quantiles[3] = 44.44; + quantiles[4] = 55.55; + cmt_summary_set_default(summary, ts, quantiles, 51.612894511314444, 10, 1, (char *[]) {"my_val"}); + + exp_histogram = cmt_exp_histogram_create(cmt, "cm", "native", "exp_hist", "native exp histogram", + 1, (char *[]) {"endpoint"}); + if (exp_histogram == NULL) { + cmt_destroy(cmt); + return NULL; + } + + cmt_exp_histogram_set_default(exp_histogram, + ts, + 2, + 11, + 0.5, + -2, + 3, + exp_positive, + -1, + 2, + exp_negative, + CMT_TRUE, + 42.25, + 29, + 1, + (char *[]) {"api"}); + + return cmt; +} + +static int compare_text_lines(const void *a, const void *b) +{ + const char *line_a; + const char *line_b; + + line_a = *(const char * const *) a; + line_b = *(const char * const *) b; + + return strcmp(line_a, line_b); +} + +static int are_texts_equivalent_ignoring_line_order(const char *left, const char *right) +{ + char *left_copy; + char *right_copy; + char *saveptr; + char *line; + char **left_lines; + char **right_lines; + size_t left_count; + size_t right_count; + size_t index; + size_t max_lines; + + if (left == NULL || right == NULL) { + return CMT_FALSE; + } + + left_copy = strdup(left); + right_copy = strdup(right); + if (left_copy == NULL || right_copy == NULL) { + free(left_copy); + free(right_copy); + return CMT_FALSE; + } + + max_lines = 1; + for (index = 0; left[index] != '\0'; index++) { + if (left[index] == '\n') { + max_lines++; + } + } + for (index = 0; right[index] != '\0'; index++) { + if (right[index] == '\n') { + max_lines++; + } + } + + left_lines = calloc(max_lines, sizeof(char *)); + right_lines = calloc(max_lines, sizeof(char *)); + if (left_lines == NULL || right_lines == NULL) { + free(left_lines); + free(right_lines); + free(left_copy); + free(right_copy); + return CMT_FALSE; + } + + left_count = 0; + saveptr = NULL; + line = strtok_r(left_copy, "\n", &saveptr); + while (line != NULL) { + left_lines[left_count++] = line; + line = strtok_r(NULL, "\n", &saveptr); + } + + right_count = 0; + saveptr = NULL; + line = strtok_r(right_copy, "\n", &saveptr); + while (line != NULL) { + right_lines[right_count++] = line; + line = strtok_r(NULL, "\n", &saveptr); + } + + if (left_count != right_count) { + free(left_lines); + free(right_lines); + free(left_copy); + free(right_copy); + return CMT_FALSE; + } + + qsort(left_lines, left_count, sizeof(char *), compare_text_lines); + qsort(right_lines, right_count, sizeof(char *), compare_text_lines); + + for (index = 0; index < left_count; index++) { + if (strcmp(left_lines[index], right_lines[index]) != 0) { + free(left_lines); + free(right_lines); + free(left_copy); + free(right_copy); + return CMT_FALSE; + } + } + + free(left_lines); + free(right_lines); + free(left_copy); + free(right_copy); + + return CMT_TRUE; +} + +static cfl_sds_t generate_exponential_histogram_otlp_payload() +{ + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest request; + Opentelemetry__Proto__Metrics__V1__ResourceMetrics resource_metrics; + Opentelemetry__Proto__Metrics__V1__ScopeMetrics scope_metrics; + Opentelemetry__Proto__Metrics__V1__Metric metric; + Opentelemetry__Proto__Metrics__V1__ExponentialHistogram exponential_histogram; + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint data_point; + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets positive_buckets; + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint__Buckets negative_buckets; + Opentelemetry__Proto__Metrics__V1__Exemplar exemplar; + Opentelemetry__Proto__Common__V1__KeyValue metric_metadata_kv; + Opentelemetry__Proto__Common__V1__AnyValue metric_metadata_value; + Opentelemetry__Proto__Metrics__V1__ResourceMetrics *resource_metrics_list[1]; + Opentelemetry__Proto__Metrics__V1__ScopeMetrics *scope_metrics_list[1]; + Opentelemetry__Proto__Metrics__V1__Metric *metric_list[1]; + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint *data_point_list[1]; + Opentelemetry__Proto__Metrics__V1__Exemplar *exemplar_list[1]; + Opentelemetry__Proto__Common__V1__KeyValue *metric_metadata_list[1]; + uint64_t positive_bucket_counts[2]; + uint64_t negative_bucket_counts[1]; + uint8_t span_id[8]; + uint8_t trace_id[16]; + size_t payload_size; + unsigned char *packed_payload; + cfl_sds_t payload; + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__init(&request); + opentelemetry__proto__metrics__v1__resource_metrics__init(&resource_metrics); + opentelemetry__proto__metrics__v1__scope_metrics__init(&scope_metrics); + opentelemetry__proto__metrics__v1__metric__init(&metric); + opentelemetry__proto__metrics__v1__exponential_histogram__init(&exponential_histogram); + opentelemetry__proto__metrics__v1__exponential_histogram_data_point__init(&data_point); + opentelemetry__proto__metrics__v1__exponential_histogram_data_point__buckets__init(&positive_buckets); + opentelemetry__proto__metrics__v1__exponential_histogram_data_point__buckets__init(&negative_buckets); + opentelemetry__proto__metrics__v1__exemplar__init(&exemplar); + opentelemetry__proto__common__v1__key_value__init(&metric_metadata_kv); + opentelemetry__proto__common__v1__any_value__init(&metric_metadata_value); + + positive_bucket_counts[0] = 3; + positive_bucket_counts[1] = 2; + negative_bucket_counts[0] = 1; + + positive_buckets.offset = 0; + positive_buckets.n_bucket_counts = 2; + positive_buckets.bucket_counts = positive_bucket_counts; + + negative_buckets.offset = 0; + negative_buckets.n_bucket_counts = 1; + negative_buckets.bucket_counts = negative_bucket_counts; + + memset(span_id, 0xAB, sizeof(span_id)); + memset(trace_id, 0xCD, sizeof(trace_id)); + + exemplar.time_unix_nano = 5; + exemplar.span_id.data = span_id; + exemplar.span_id.len = sizeof(span_id); + exemplar.trace_id.data = trace_id; + exemplar.trace_id.len = sizeof(trace_id); + exemplar.value_case = OPENTELEMETRY__PROTO__METRICS__V1__EXEMPLAR__VALUE_AS_DOUBLE; + exemplar.as_double = 3.25; + exemplar_list[0] = &exemplar; + + data_point.time_unix_nano = 1; + data_point.start_time_unix_nano = 123; + data_point.count = 7; + data_point.has_sum = CMT_TRUE; + data_point.sum = 8.0; + data_point.scale = 0; + data_point.zero_count = 1; + data_point.zero_threshold = 0.0; + data_point.flags = OPENTELEMETRY__PROTO__METRICS__V1__DATA_POINT_FLAGS__DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK; + data_point.has_min = CMT_TRUE; + data_point.min = -2.5; + data_point.has_max = CMT_TRUE; + data_point.max = 4.5; + data_point.positive = &positive_buckets; + data_point.negative = &negative_buckets; + data_point.exemplars = exemplar_list; + data_point.n_exemplars = 1; + + metric_metadata_value.value_case = OPENTELEMETRY__PROTO__COMMON__V1__ANY_VALUE__VALUE_STRING_VALUE; + metric_metadata_value.string_value = "decode-roundtrip"; + metric_metadata_kv.key = "origin"; + metric_metadata_kv.value = &metric_metadata_value; + metric_metadata_list[0] = &metric_metadata_kv; + + data_point_list[0] = &data_point; + exponential_histogram.n_data_points = 1; + exponential_histogram.data_points = data_point_list; + exponential_histogram.aggregation_temporality = + OPENTELEMETRY__PROTO__METRICS__V1__AGGREGATION_TEMPORALITY__AGGREGATION_TEMPORALITY_CUMULATIVE; + + metric.name = "exp_hist"; + metric.n_metadata = 1; + metric.metadata = metric_metadata_list; + metric.data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_EXPONENTIAL_HISTOGRAM; + metric.exponential_histogram = &exponential_histogram; + + metric_list[0] = &metric; + scope_metrics.n_metrics = 1; + scope_metrics.metrics = metric_list; + + scope_metrics_list[0] = &scope_metrics; + resource_metrics.n_scope_metrics = 1; + resource_metrics.scope_metrics = scope_metrics_list; + + resource_metrics_list[0] = &resource_metrics; + request.n_resource_metrics = 1; + request.resource_metrics = resource_metrics_list; + + payload_size = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__get_packed_size(&request); + packed_payload = calloc(1, payload_size); + if (packed_payload == NULL) { + return NULL; + } + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__pack(&request, + packed_payload); + + payload = cfl_sds_create_len((char *) packed_payload, payload_size); + free(packed_payload); + + return payload; +} + +static cfl_sds_t generate_gauge_int_otlp_payload_with_unit() +{ + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest request; + Opentelemetry__Proto__Metrics__V1__ResourceMetrics resource_metrics; + Opentelemetry__Proto__Metrics__V1__ScopeMetrics scope_metrics; + Opentelemetry__Proto__Metrics__V1__Metric metric; + Opentelemetry__Proto__Metrics__V1__Gauge gauge; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint data_point; + Opentelemetry__Proto__Metrics__V1__ResourceMetrics *resource_metrics_list[1]; + Opentelemetry__Proto__Metrics__V1__ScopeMetrics *scope_metrics_list[1]; + Opentelemetry__Proto__Metrics__V1__Metric *metric_list[1]; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint *data_point_list[1]; + size_t payload_size; + unsigned char *packed_payload; + cfl_sds_t payload; + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__init(&request); + opentelemetry__proto__metrics__v1__resource_metrics__init(&resource_metrics); + opentelemetry__proto__metrics__v1__scope_metrics__init(&scope_metrics); + opentelemetry__proto__metrics__v1__metric__init(&metric); + opentelemetry__proto__metrics__v1__gauge__init(&gauge); + opentelemetry__proto__metrics__v1__number_data_point__init(&data_point); + + data_point.time_unix_nano = 123; + data_point.start_time_unix_nano = 0; + data_point.value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT; + data_point.as_int = -7; + + data_point_list[0] = &data_point; + gauge.n_data_points = 1; + gauge.data_points = data_point_list; + + metric.name = "g_int"; + metric.unit = "bytes"; + metric.data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_GAUGE; + metric.gauge = &gauge; + + metric_list[0] = &metric; + scope_metrics.n_metrics = 1; + scope_metrics.metrics = metric_list; + + scope_metrics_list[0] = &scope_metrics; + resource_metrics.n_scope_metrics = 1; + resource_metrics.scope_metrics = scope_metrics_list; + + resource_metrics_list[0] = &resource_metrics; + request.n_resource_metrics = 1; + request.resource_metrics = resource_metrics_list; + + payload_size = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__get_packed_size(&request); + packed_payload = calloc(1, payload_size); + if (packed_payload == NULL) { + return NULL; + } + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__pack(&request, + packed_payload); + + payload = cfl_sds_create_len((char *) packed_payload, payload_size); + free(packed_payload); + + return payload; +} + +static cfl_sds_t generate_sum_non_monotonic_int_otlp_payload() +{ + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest request; + Opentelemetry__Proto__Metrics__V1__ResourceMetrics resource_metrics; + Opentelemetry__Proto__Metrics__V1__ScopeMetrics scope_metrics; + Opentelemetry__Proto__Metrics__V1__Metric metric; + Opentelemetry__Proto__Metrics__V1__Sum sum; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint data_point; + Opentelemetry__Proto__Metrics__V1__ResourceMetrics *resource_metrics_list[1]; + Opentelemetry__Proto__Metrics__V1__ScopeMetrics *scope_metrics_list[1]; + Opentelemetry__Proto__Metrics__V1__Metric *metric_list[1]; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint *data_point_list[1]; + size_t payload_size; + unsigned char *packed_payload; + cfl_sds_t payload; + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__init(&request); + opentelemetry__proto__metrics__v1__resource_metrics__init(&resource_metrics); + opentelemetry__proto__metrics__v1__scope_metrics__init(&scope_metrics); + opentelemetry__proto__metrics__v1__metric__init(&metric); + opentelemetry__proto__metrics__v1__sum__init(&sum); + opentelemetry__proto__metrics__v1__number_data_point__init(&data_point); + + data_point.time_unix_nano = 321; + data_point.start_time_unix_nano = 100; + data_point.value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT; + data_point.as_int = -7; + + data_point_list[0] = &data_point; + sum.n_data_points = 1; + sum.data_points = data_point_list; + sum.aggregation_temporality = + OPENTELEMETRY__PROTO__METRICS__V1__AGGREGATION_TEMPORALITY__AGGREGATION_TEMPORALITY_CUMULATIVE; + sum.is_monotonic = CMT_FALSE; + + metric.name = "sum_non_monotonic_int"; + metric.data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_SUM; + metric.sum = ∑ + + metric_list[0] = &metric; + scope_metrics.n_metrics = 1; + scope_metrics.metrics = metric_list; + + scope_metrics_list[0] = &scope_metrics; + resource_metrics.n_scope_metrics = 1; + resource_metrics.scope_metrics = scope_metrics_list; + + resource_metrics_list[0] = &resource_metrics; + request.n_resource_metrics = 1; + request.resource_metrics = resource_metrics_list; + + payload_size = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__get_packed_size(&request); + packed_payload = calloc(1, payload_size); + if (packed_payload == NULL) { + return NULL; + } + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__pack(&request, + packed_payload); + + payload = cfl_sds_create_len((char *) packed_payload, payload_size); + free(packed_payload); + + return payload; +} + +static cfl_sds_t generate_gauge_large_int_otlp_payload() +{ + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest request; + Opentelemetry__Proto__Metrics__V1__ResourceMetrics resource_metrics; + Opentelemetry__Proto__Metrics__V1__ScopeMetrics scope_metrics; + Opentelemetry__Proto__Metrics__V1__Metric metric; + Opentelemetry__Proto__Metrics__V1__Gauge gauge; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint data_point; + Opentelemetry__Proto__Metrics__V1__ResourceMetrics *resource_metrics_list[1]; + Opentelemetry__Proto__Metrics__V1__ScopeMetrics *scope_metrics_list[1]; + Opentelemetry__Proto__Metrics__V1__Metric *metric_list[1]; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint *data_point_list[1]; + size_t payload_size; + unsigned char *packed_payload; + cfl_sds_t payload; + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__init(&request); + opentelemetry__proto__metrics__v1__resource_metrics__init(&resource_metrics); + opentelemetry__proto__metrics__v1__scope_metrics__init(&scope_metrics); + opentelemetry__proto__metrics__v1__metric__init(&metric); + opentelemetry__proto__metrics__v1__gauge__init(&gauge); + opentelemetry__proto__metrics__v1__number_data_point__init(&data_point); + + data_point.time_unix_nano = 456; + data_point.start_time_unix_nano = 0; + data_point.value_case = OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT; + data_point.as_int = 9007199254740993LL; + + data_point_list[0] = &data_point; + gauge.n_data_points = 1; + gauge.data_points = data_point_list; + + metric.name = "g_int_large"; + metric.data_case = OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_GAUGE; + metric.gauge = &gauge; + + metric_list[0] = &metric; + scope_metrics.n_metrics = 1; + scope_metrics.metrics = metric_list; + + scope_metrics_list[0] = &scope_metrics; + resource_metrics.n_scope_metrics = 1; + resource_metrics.scope_metrics = scope_metrics_list; + + resource_metrics_list[0] = &resource_metrics; + request.n_resource_metrics = 1; + request.resource_metrics = resource_metrics_list; + + payload_size = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__get_packed_size(&request); + packed_payload = calloc(1, payload_size); + if (packed_payload == NULL) { + return NULL; + } + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__pack(&request, + packed_payload); + + payload = cfl_sds_create_len((char *) packed_payload, payload_size); + free(packed_payload); + + return payload; +} + +void test_opentelemetry_encode_multi_resource_scope_containers() +{ + struct cmt *cmt; + struct cmt_gauge *gauge; + struct cfl_array *resource_metrics_list; + struct cfl_array *scope_metrics_list; + struct cfl_kvlist *resource_entry; + struct cfl_kvlist *scope_entry; + struct cfl_kvlist *rm_root; + struct cfl_kvlist *rm_meta; + struct cfl_kvlist *sm_root; + struct cfl_kvlist *sm_meta; + cfl_sds_t payload; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *service_request; + + cmt_initialize(); + + cmt = cmt_create(); + TEST_CHECK(cmt != NULL); + if (cmt == NULL) { + return; + } + + gauge = cmt_gauge_create(cmt, "ns", "sub", "multi_container_gauge", "g", 0, NULL); + TEST_CHECK(gauge != NULL); + if (gauge == NULL) { + cmt_destroy(cmt); + return; + } + cmt_gauge_set(gauge, 123, 1.5, 0, NULL); + + resource_metrics_list = cfl_array_create(2); + TEST_CHECK(resource_metrics_list != NULL); + if (resource_metrics_list == NULL) { + cmt_destroy(cmt); + return; + } + + resource_entry = cfl_kvlist_create(); + rm_root = cfl_kvlist_create(); + rm_meta = cfl_kvlist_create(); + cfl_kvlist_insert_string(rm_meta, "schema_url", "rm-schema-1"); + cfl_kvlist_insert_kvlist(rm_root, "metadata", rm_meta); + cfl_kvlist_insert_kvlist(resource_entry, "resource_metrics", rm_root); + + scope_metrics_list = cfl_array_create(2); + scope_entry = cfl_kvlist_create(); + sm_root = cfl_kvlist_create(); + sm_meta = cfl_kvlist_create(); + cfl_kvlist_insert_string(sm_meta, "schema_url", "sm-schema-1"); + cfl_kvlist_insert_kvlist(sm_root, "metadata", sm_meta); + cfl_kvlist_insert_kvlist(scope_entry, "scope_metrics", sm_root); + cfl_array_append_kvlist(scope_metrics_list, scope_entry); + + scope_entry = cfl_kvlist_create(); + sm_root = cfl_kvlist_create(); + sm_meta = cfl_kvlist_create(); + cfl_kvlist_insert_string(sm_meta, "schema_url", "sm-schema-2"); + cfl_kvlist_insert_kvlist(sm_root, "metadata", sm_meta); + cfl_kvlist_insert_kvlist(scope_entry, "scope_metrics", sm_root); + cfl_array_append_kvlist(scope_metrics_list, scope_entry); + + cfl_kvlist_insert_array(resource_entry, "scope_metrics_list", scope_metrics_list); + cfl_array_append_kvlist(resource_metrics_list, resource_entry); + + resource_entry = cfl_kvlist_create(); + rm_root = cfl_kvlist_create(); + rm_meta = cfl_kvlist_create(); + cfl_kvlist_insert_string(rm_meta, "schema_url", "rm-schema-2"); + cfl_kvlist_insert_kvlist(rm_root, "metadata", rm_meta); + cfl_kvlist_insert_kvlist(resource_entry, "resource_metrics", rm_root); + + scope_metrics_list = cfl_array_create(1); + scope_entry = cfl_kvlist_create(); + sm_root = cfl_kvlist_create(); + sm_meta = cfl_kvlist_create(); + cfl_kvlist_insert_string(sm_meta, "schema_url", "sm-schema-3"); + cfl_kvlist_insert_kvlist(sm_root, "metadata", sm_meta); + cfl_kvlist_insert_kvlist(scope_entry, "scope_metrics", sm_root); + cfl_array_append_kvlist(scope_metrics_list, scope_entry); + cfl_kvlist_insert_array(resource_entry, "scope_metrics_list", scope_metrics_list); + cfl_array_append_kvlist(resource_metrics_list, resource_entry); + + cfl_kvlist_insert_array(cmt->external_metadata, "resource_metrics_list", resource_metrics_list); + + payload = cmt_encode_opentelemetry_create(cmt); + TEST_CHECK(payload != NULL); + if (payload != NULL) { + service_request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(payload), (uint8_t *) payload); + TEST_CHECK(service_request != NULL); + if (service_request != NULL) { + TEST_CHECK(service_request->n_resource_metrics == 2); + if (service_request->n_resource_metrics == 2) { + TEST_CHECK(service_request->resource_metrics[0]->n_scope_metrics == 2); + TEST_CHECK(service_request->resource_metrics[1]->n_scope_metrics == 1); + TEST_CHECK(strcmp(service_request->resource_metrics[0]->schema_url, "rm-schema-1") == 0); + TEST_CHECK(strcmp(service_request->resource_metrics[1]->schema_url, "rm-schema-2") == 0); + TEST_CHECK(strcmp(service_request->resource_metrics[0]->scope_metrics[0]->schema_url, "sm-schema-1") == 0); + TEST_CHECK(strcmp(service_request->resource_metrics[0]->scope_metrics[1]->schema_url, "sm-schema-2") == 0); + TEST_CHECK(strcmp(service_request->resource_metrics[1]->scope_metrics[0]->schema_url, "sm-schema-3") == 0); + TEST_CHECK(service_request->resource_metrics[0]->scope_metrics[0]->n_metrics == 1); + TEST_CHECK(service_request->resource_metrics[0]->scope_metrics[1]->n_metrics == 0); + TEST_CHECK(service_request->resource_metrics[1]->scope_metrics[0]->n_metrics == 0); + } + + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(service_request, NULL); + } + + cmt_encode_opentelemetry_destroy(payload); + } + + cmt_destroy(cmt); +} + +void test_opentelemetry_api_full_roundtrip_with_msgpack() +{ + int ret; + size_t offset; + size_t msgpack_offset; + cfl_sds_t otlp_payload_1; + cfl_sds_t otlp_payload_2; + cfl_sds_t reference_text; + cfl_sds_t result_text; + char *msgpack_buffer; + size_t msgpack_size; + struct cfl_list decoded_list_1; + struct cfl_list decoded_list_2; + struct cmt *api_context; + struct cmt *decoded_context_1; + struct cmt *decoded_context_2; + struct cmt *msgpack_context; + + cmt_initialize(); + + api_context = generate_api_test_data(); + TEST_CHECK(api_context != NULL); + if (api_context == NULL) { + return; + } + + reference_text = cmt_encode_text_create(api_context); + TEST_CHECK(reference_text != NULL); + + otlp_payload_1 = cmt_encode_opentelemetry_create(api_context); + TEST_CHECK(otlp_payload_1 != NULL); + if (otlp_payload_1 == NULL) { + cmt_encode_text_destroy(reference_text); + cmt_destroy(api_context); + return; + } + + offset = 0; + ret = cmt_decode_opentelemetry_create(&decoded_list_1, otlp_payload_1, cfl_sds_len(otlp_payload_1), &offset); + TEST_CHECK(ret == CMT_DECODE_OPENTELEMETRY_SUCCESS); + if (ret != CMT_DECODE_OPENTELEMETRY_SUCCESS) { + cmt_encode_opentelemetry_destroy(otlp_payload_1); + cmt_encode_text_destroy(reference_text); + cmt_destroy(api_context); + return; + } + + decoded_context_1 = cfl_list_entry_first(&decoded_list_1, struct cmt, _head); + TEST_CHECK(decoded_context_1 != NULL); + if (decoded_context_1 == NULL) { + cmt_decode_opentelemetry_destroy(&decoded_list_1); + cmt_encode_opentelemetry_destroy(otlp_payload_1); + cmt_encode_text_destroy(reference_text); + cmt_destroy(api_context); + return; + } + + ret = cmt_encode_msgpack_create(decoded_context_1, &msgpack_buffer, &msgpack_size); + TEST_CHECK(ret == 0); + if (ret != 0) { + cmt_decode_opentelemetry_destroy(&decoded_list_1); + cmt_encode_opentelemetry_destroy(otlp_payload_1); + cmt_encode_text_destroy(reference_text); + cmt_destroy(api_context); + return; + } + + msgpack_offset = 0; + msgpack_context = NULL; + ret = cmt_decode_msgpack_create(&msgpack_context, msgpack_buffer, msgpack_size, &msgpack_offset); + TEST_CHECK(ret == 0); + if (ret != 0 || msgpack_context == NULL) { + cmt_encode_msgpack_destroy(msgpack_buffer); + cmt_decode_opentelemetry_destroy(&decoded_list_1); + cmt_encode_opentelemetry_destroy(otlp_payload_1); + cmt_encode_text_destroy(reference_text); + cmt_destroy(api_context); + return; + } + + otlp_payload_2 = cmt_encode_opentelemetry_create(msgpack_context); + TEST_CHECK(otlp_payload_2 != NULL); + if (otlp_payload_2 == NULL) { + cmt_decode_msgpack_destroy(msgpack_context); + cmt_encode_msgpack_destroy(msgpack_buffer); + cmt_decode_opentelemetry_destroy(&decoded_list_1); + cmt_encode_opentelemetry_destroy(otlp_payload_1); + cmt_encode_text_destroy(reference_text); + cmt_destroy(api_context); + return; + } + + offset = 0; + ret = cmt_decode_opentelemetry_create(&decoded_list_2, otlp_payload_2, cfl_sds_len(otlp_payload_2), &offset); + TEST_CHECK(ret == CMT_DECODE_OPENTELEMETRY_SUCCESS); + if (ret == CMT_DECODE_OPENTELEMETRY_SUCCESS) { + decoded_context_2 = cfl_list_entry_first(&decoded_list_2, struct cmt, _head); + TEST_CHECK(decoded_context_2 != NULL); + + if (decoded_context_2 != NULL) { + result_text = cmt_encode_text_create(decoded_context_2); + TEST_CHECK(result_text != NULL); + + if (result_text != NULL && reference_text != NULL) { + TEST_CHECK(are_texts_equivalent_ignoring_line_order(reference_text, + result_text) == CMT_TRUE); + cmt_encode_text_destroy(result_text); + } + } + + cmt_decode_opentelemetry_destroy(&decoded_list_2); + } + + cmt_encode_opentelemetry_destroy(otlp_payload_2); + cmt_decode_msgpack_destroy(msgpack_context); + cmt_encode_msgpack_destroy(msgpack_buffer); + cmt_decode_opentelemetry_destroy(&decoded_list_1); + cmt_encode_opentelemetry_destroy(otlp_payload_1); + cmt_encode_text_destroy(reference_text); + cmt_destroy(api_context); +} + +void test_opentelemetry_gauge_int_and_unit_decode() +{ + cfl_sds_t payload; + struct cfl_list decoded_context_list; + struct cmt *decoded_context; + struct cmt_gauge *gauge; + double val; + size_t offset; + int result; + + cmt_initialize(); + + payload = generate_gauge_int_otlp_payload_with_unit(); + TEST_CHECK(payload != NULL); + + if (payload == NULL) { + return; + } + + offset = 0; + result = cmt_decode_opentelemetry_create(&decoded_context_list, + payload, + cfl_sds_len(payload), + &offset); + TEST_CHECK(result == CMT_DECODE_OPENTELEMETRY_SUCCESS); + + if (result != CMT_DECODE_OPENTELEMETRY_SUCCESS) { + cfl_sds_destroy(payload); + return; + } + + decoded_context = cfl_list_entry_first(&decoded_context_list, struct cmt, _head); + TEST_CHECK(decoded_context != NULL); + + if (decoded_context != NULL) { + TEST_CHECK(cfl_list_size(&decoded_context->gauges) == 1); + gauge = cfl_list_entry_first(&decoded_context->gauges, struct cmt_gauge, _head); + TEST_CHECK(gauge != NULL); + + if (gauge != NULL) { + TEST_CHECK(gauge->map != NULL); + if (gauge->map != NULL) { + TEST_CHECK(gauge->map->unit != NULL); + if (gauge->map->unit != NULL) { + TEST_CHECK(strcmp(gauge->map->unit, "bytes") == 0); + } + } + + result = cmt_gauge_get_val(gauge, 0, NULL, &val); + TEST_CHECK(result == 0); + if (result == 0) { + TEST_CHECK(val == -7.0); + } + } + } + + cmt_decode_opentelemetry_destroy(&decoded_context_list); + cfl_sds_destroy(payload); +} + +void test_opentelemetry_exponential_histogram() +{ + cfl_sds_t payload; + cfl_sds_t first_prometheus_context; + cfl_sds_t first_text_context; + cfl_sds_t second_payload; + cfl_sds_t second_prometheus_context; + struct cfl_list first_decoded_context_list; + struct cfl_list second_decoded_context_list; + struct cmt *first_decoded_context; + struct cmt *second_decoded_context; + size_t offset; + int result; + + cmt_initialize(); + + payload = generate_exponential_histogram_otlp_payload(); + TEST_CHECK(payload != NULL); + + if (payload == NULL) { + return; + } + + offset = 0; + result = cmt_decode_opentelemetry_create(&first_decoded_context_list, + payload, + cfl_sds_len(payload), + &offset); + TEST_CHECK(result == CMT_DECODE_OPENTELEMETRY_SUCCESS); + + if (result != CMT_DECODE_OPENTELEMETRY_SUCCESS) { + cfl_sds_destroy(payload); + return; + } + + first_decoded_context = cfl_list_entry_first(&first_decoded_context_list, struct cmt, _head); + TEST_CHECK(first_decoded_context != NULL); + + if (first_decoded_context == NULL) { + cmt_decode_opentelemetry_destroy(&first_decoded_context_list); + cfl_sds_destroy(payload); + return; + } + + first_prometheus_context = cmt_encode_prometheus_create(first_decoded_context, CMT_TRUE); + TEST_CHECK(first_prometheus_context != NULL); + + if (first_prometheus_context != NULL) { + TEST_CHECK(strstr(first_prometheus_context, "exp_hist_bucket{le=\"-1.0\"} 1 0") != NULL); + TEST_CHECK(strstr(first_prometheus_context, "exp_hist_bucket{le=\"0.0\"} 2 0") != NULL); + TEST_CHECK(strstr(first_prometheus_context, "exp_hist_bucket{le=\"2.0\"} 5 0") != NULL); + TEST_CHECK(strstr(first_prometheus_context, "exp_hist_bucket{le=\"4.0\"} 7 0") != NULL); + TEST_CHECK(strstr(first_prometheus_context, "exp_hist_bucket{le=\"+Inf\"} 7 0") != NULL); + TEST_CHECK(strstr(first_prometheus_context, "exp_hist_sum 8 0") != NULL); + TEST_CHECK(strstr(first_prometheus_context, "exp_hist_count 7 0") != NULL); + } + + first_text_context = cmt_encode_text_create(first_decoded_context); + TEST_CHECK(first_text_context != NULL); + if (first_text_context != NULL) { + TEST_CHECK(strstr(first_text_context, "exemplars=[") != NULL); + TEST_CHECK(strstr(first_text_context, "as_double=3.25") != NULL); + TEST_CHECK(strstr(first_text_context, "time_unix_nano=5") != NULL); + cmt_encode_text_destroy(first_text_context); + } + + second_payload = cmt_encode_opentelemetry_create(first_decoded_context); + TEST_CHECK(second_payload != NULL); + + if (second_payload != NULL) { + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *service_request; + Opentelemetry__Proto__Metrics__V1__Metric *roundtrip_metric; + Opentelemetry__Proto__Metrics__V1__ExponentialHistogramDataPoint *roundtrip_dp; + + offset = 0; + result = cmt_decode_opentelemetry_create(&second_decoded_context_list, + second_payload, + cfl_sds_len(second_payload), + &offset); + TEST_CHECK(result == CMT_DECODE_OPENTELEMETRY_SUCCESS); + + if (result == CMT_DECODE_OPENTELEMETRY_SUCCESS) { + second_decoded_context = cfl_list_entry_first(&second_decoded_context_list, struct cmt, _head); + TEST_CHECK(second_decoded_context != NULL); + + if (second_decoded_context != NULL) { + second_prometheus_context = cmt_encode_prometheus_create(second_decoded_context, CMT_TRUE); + TEST_CHECK(second_prometheus_context != NULL); + + if (second_prometheus_context != NULL && first_prometheus_context != NULL) { + TEST_CHECK(strcmp(first_prometheus_context, second_prometheus_context) == 0); + cmt_encode_prometheus_destroy(second_prometheus_context); + } + } + + cmt_decode_opentelemetry_destroy(&second_decoded_context_list); + } + + service_request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(second_payload), (uint8_t *) second_payload); + TEST_CHECK(service_request != NULL); + + if (service_request != NULL && + service_request->n_resource_metrics == 1 && + service_request->resource_metrics[0]->n_scope_metrics == 1 && + service_request->resource_metrics[0]->scope_metrics[0]->n_metrics == 1) { + roundtrip_metric = service_request->resource_metrics[0]->scope_metrics[0]->metrics[0]; + TEST_CHECK(roundtrip_metric->n_metadata == 1); + if (roundtrip_metric->n_metadata == 1) { + TEST_CHECK(strcmp(roundtrip_metric->metadata[0]->key, "origin") == 0); + } + + TEST_CHECK(roundtrip_metric->data_case == + OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_EXPONENTIAL_HISTOGRAM); + if (roundtrip_metric->data_case == + OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_EXPONENTIAL_HISTOGRAM) { + roundtrip_dp = roundtrip_metric->exponential_histogram->data_points[0]; + TEST_CHECK(roundtrip_dp->start_time_unix_nano == 123); + TEST_CHECK(roundtrip_dp->flags == + OPENTELEMETRY__PROTO__METRICS__V1__DATA_POINT_FLAGS__DATA_POINT_FLAGS_NO_RECORDED_VALUE_MASK); + TEST_CHECK(roundtrip_dp->has_min == CMT_TRUE); + TEST_CHECK(roundtrip_dp->has_max == CMT_TRUE); + TEST_CHECK(roundtrip_dp->min == -2.5); + TEST_CHECK(roundtrip_dp->max == 4.5); + TEST_CHECK(roundtrip_dp->n_exemplars == 1); + } + } + + if (service_request != NULL) { + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(service_request, NULL); + } + + cmt_encode_opentelemetry_destroy(second_payload); + } + + if (first_prometheus_context != NULL) { + cmt_encode_prometheus_destroy(first_prometheus_context); + } + + cmt_decode_opentelemetry_destroy(&first_decoded_context_list); + cfl_sds_destroy(payload); +} + +void test_opentelemetry_sum_non_monotonic_int_roundtrip() +{ + cfl_sds_t payload; + cfl_sds_t encoded_payload; + struct cfl_list decoded_context_list; + struct cmt *decoded_context; + struct cmt_counter *counter; + double value; + size_t offset; + int result; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *service_request; + Opentelemetry__Proto__Metrics__V1__Metric *roundtrip_metric; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint *roundtrip_dp; + + cmt_initialize(); + + payload = generate_sum_non_monotonic_int_otlp_payload(); + TEST_CHECK(payload != NULL); + if (payload == NULL) { + return; + } + + offset = 0; + result = cmt_decode_opentelemetry_create(&decoded_context_list, + payload, + cfl_sds_len(payload), + &offset); + TEST_CHECK(result == CMT_DECODE_OPENTELEMETRY_SUCCESS); + if (result != CMT_DECODE_OPENTELEMETRY_SUCCESS) { + cfl_sds_destroy(payload); + return; + } + + decoded_context = cfl_list_entry_first(&decoded_context_list, struct cmt, _head); + TEST_CHECK(decoded_context != NULL); + + if (decoded_context != NULL) { + TEST_CHECK(cfl_list_size(&decoded_context->counters) == 1); + counter = cfl_list_entry_first(&decoded_context->counters, struct cmt_counter, _head); + TEST_CHECK(counter != NULL); + + if (counter != NULL) { + TEST_CHECK(counter->allow_reset == CMT_TRUE); + result = cmt_counter_get_val(counter, 0, NULL, &value); + TEST_CHECK(result == 0); + if (result == 0) { + TEST_CHECK(value == -7.0); + } + } + + encoded_payload = cmt_encode_opentelemetry_create(decoded_context); + TEST_CHECK(encoded_payload != NULL); + if (encoded_payload != NULL) { + service_request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(encoded_payload), (uint8_t *) encoded_payload); + TEST_CHECK(service_request != NULL); + + if (service_request != NULL && + service_request->n_resource_metrics == 1 && + service_request->resource_metrics[0]->n_scope_metrics == 1 && + service_request->resource_metrics[0]->scope_metrics[0]->n_metrics == 1) { + roundtrip_metric = service_request->resource_metrics[0]->scope_metrics[0]->metrics[0]; + TEST_CHECK(roundtrip_metric->data_case == + OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_SUM); + if (roundtrip_metric->data_case == + OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_SUM) { + TEST_CHECK(roundtrip_metric->sum->is_monotonic == CMT_FALSE); + TEST_CHECK(roundtrip_metric->sum->aggregation_temporality == + OPENTELEMETRY__PROTO__METRICS__V1__AGGREGATION_TEMPORALITY__AGGREGATION_TEMPORALITY_CUMULATIVE); + TEST_CHECK(roundtrip_metric->sum->n_data_points == 1); + roundtrip_dp = roundtrip_metric->sum->data_points[0]; + TEST_CHECK(roundtrip_dp->value_case == + OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT); + TEST_CHECK(roundtrip_dp->as_int == -7); + } + } + + if (service_request != NULL) { + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(service_request, NULL); + } + + cmt_encode_opentelemetry_destroy(encoded_payload); + } + } + + cmt_decode_opentelemetry_destroy(&decoded_context_list); + cfl_sds_destroy(payload); +} + +void test_opentelemetry_large_int_roundtrip_with_msgpack() +{ + cfl_sds_t payload; + cfl_sds_t encoded_payload; + char *msgpack_buffer; + size_t msgpack_size; + size_t offset; + size_t msgpack_offset; + int result; + struct cfl_list decoded_context_list; + struct cmt *decoded_context; + struct cmt *msgpack_context; + Opentelemetry__Proto__Collector__Metrics__V1__ExportMetricsServiceRequest *service_request; + Opentelemetry__Proto__Metrics__V1__Metric *roundtrip_metric; + Opentelemetry__Proto__Metrics__V1__NumberDataPoint *roundtrip_dp; + + cmt_initialize(); + + payload = generate_gauge_large_int_otlp_payload(); + TEST_CHECK(payload != NULL); + if (payload == NULL) { + return; + } + + offset = 0; + result = cmt_decode_opentelemetry_create(&decoded_context_list, + payload, + cfl_sds_len(payload), + &offset); + TEST_CHECK(result == CMT_DECODE_OPENTELEMETRY_SUCCESS); + if (result != CMT_DECODE_OPENTELEMETRY_SUCCESS) { + cfl_sds_destroy(payload); + return; + } + + decoded_context = cfl_list_entry_first(&decoded_context_list, struct cmt, _head); + TEST_CHECK(decoded_context != NULL); + if (decoded_context == NULL) { + cmt_decode_opentelemetry_destroy(&decoded_context_list); + cfl_sds_destroy(payload); + return; + } + + result = cmt_encode_msgpack_create(decoded_context, &msgpack_buffer, &msgpack_size); + TEST_CHECK(result == 0); + if (result != 0) { + cmt_decode_opentelemetry_destroy(&decoded_context_list); + cfl_sds_destroy(payload); + return; + } + + msgpack_offset = 0; + msgpack_context = NULL; + result = cmt_decode_msgpack_create(&msgpack_context, msgpack_buffer, msgpack_size, &msgpack_offset); + TEST_CHECK(result == 0); + if (result != 0 || msgpack_context == NULL) { + cmt_encode_msgpack_destroy(msgpack_buffer); + cmt_decode_opentelemetry_destroy(&decoded_context_list); + cfl_sds_destroy(payload); + return; + } + + encoded_payload = cmt_encode_opentelemetry_create(msgpack_context); + TEST_CHECK(encoded_payload != NULL); + if (encoded_payload != NULL) { + service_request = opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__unpack( + NULL, cfl_sds_len(encoded_payload), (uint8_t *) encoded_payload); + TEST_CHECK(service_request != NULL); + + if (service_request != NULL && + service_request->n_resource_metrics == 1 && + service_request->resource_metrics[0]->n_scope_metrics == 1 && + service_request->resource_metrics[0]->scope_metrics[0]->n_metrics == 1) { + roundtrip_metric = service_request->resource_metrics[0]->scope_metrics[0]->metrics[0]; + TEST_CHECK(roundtrip_metric->data_case == + OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_GAUGE); + if (roundtrip_metric->data_case == + OPENTELEMETRY__PROTO__METRICS__V1__METRIC__DATA_GAUGE && + roundtrip_metric->gauge->n_data_points == 1) { + roundtrip_dp = roundtrip_metric->gauge->data_points[0]; + TEST_CHECK(roundtrip_dp->value_case == + OPENTELEMETRY__PROTO__METRICS__V1__NUMBER_DATA_POINT__VALUE_AS_INT); + TEST_CHECK(roundtrip_dp->as_int == 9007199254740993LL); + } + } + + if (service_request != NULL) { + opentelemetry__proto__collector__metrics__v1__export_metrics_service_request__free_unpacked(service_request, NULL); + } + cmt_encode_opentelemetry_destroy(encoded_payload); + } + + cmt_decode_msgpack_destroy(msgpack_context); + cmt_encode_msgpack_destroy(msgpack_buffer); + cmt_decode_opentelemetry_destroy(&decoded_context_list); + cfl_sds_destroy(payload); +} + +TEST_LIST = { + {"opentelemetry_api_full_roundtrip_with_msgpack", test_opentelemetry_api_full_roundtrip_with_msgpack}, + {"opentelemetry_encode_multi_resource_scope_containers", test_opentelemetry_encode_multi_resource_scope_containers}, + {"opentelemetry_exponential_histogram", test_opentelemetry_exponential_histogram}, + {"opentelemetry_gauge_int_and_unit_decode", test_opentelemetry_gauge_int_and_unit_decode}, + {"opentelemetry_sum_non_monotonic_int_roundtrip", test_opentelemetry_sum_non_monotonic_int_roundtrip}, + {"opentelemetry_large_int_roundtrip_with_msgpack", test_opentelemetry_large_int_roundtrip_with_msgpack}, + { 0 } +}; diff --git a/plugins/in_opentelemetry/opentelemetry_logs.c b/plugins/in_opentelemetry/opentelemetry_logs.c index abc495e8b7f..d38795cebbe 100644 --- a/plugins/in_opentelemetry/opentelemetry_logs.c +++ b/plugins/in_opentelemetry/opentelemetry_logs.c @@ -641,16 +641,14 @@ int opentelemetry_process_logs(struct flb_opentelemetry *ctx, /* Detect the type of payload */ if (content_type) { - if (strcasecmp(content_type, "application/json") == 0) { - if (buf[0] != '{') { + if (opentelemetry_is_json_content_type(content_type) == FLB_TRUE) { + if (opentelemetry_payload_starts_with_json_object(buf, size) != FLB_TRUE) { flb_plg_error(ctx->ins, "Invalid JSON payload"); return -1; } is_proto = FLB_FALSE; } - else if (strcasecmp(content_type, "application/protobuf") == 0 || - strcasecmp(content_type, "application/grpc") == 0 || - strcasecmp(content_type, "application/x-protobuf") == 0) { + else if (opentelemetry_is_protobuf_content_type(content_type) == FLB_TRUE) { is_proto = FLB_TRUE; } else { diff --git a/plugins/in_opentelemetry/opentelemetry_prot.c b/plugins/in_opentelemetry/opentelemetry_prot.c index a974049d3a2..33c3e2efe8b 100644 --- a/plugins/in_opentelemetry/opentelemetry_prot.c +++ b/plugins/in_opentelemetry/opentelemetry_prot.c @@ -27,6 +27,7 @@ #include #include #include +#include #include @@ -46,25 +47,6 @@ #define HTTP_CONTENT_JSON 0 -static int is_grpc_content_type(const char *content_type) -{ - if (content_type == NULL) { - return FLB_FALSE; - } - - if (strncasecmp(content_type, "application/grpc", 16) != 0) { - return FLB_FALSE; - } - - if (content_type[16] == '\0' || - content_type[16] == '+' || - content_type[16] == ';') { - return FLB_TRUE; - } - - return FLB_FALSE; -} - static int is_profiles_export_path(const char *path) { if (path == NULL) { @@ -141,6 +123,7 @@ static int send_response(struct http_conn *conn, int http_status, char *message) static int process_payload_metrics(struct flb_opentelemetry *ctx, struct http_conn *conn, flb_sds_t tag, size_t tag_len, + flb_sds_t content_type, struct mk_http_session *session, struct mk_http_request *request) { @@ -150,12 +133,24 @@ static int process_payload_metrics(struct flb_opentelemetry *ctx, struct http_co size_t offset; int result; + (void) conn; + (void) session; + offset = 0; - result = cmt_decode_opentelemetry_create(&decoded_contexts, - request->data.data, - request->data.len, - &offset); + if (content_type != NULL && + opentelemetry_is_json_content_type(content_type) == FLB_TRUE) { + result = flb_opentelemetry_metrics_json_to_cmt(&decoded_contexts, + request->data.data, + request->data.len); + } + else { + result = cmt_decode_opentelemetry_create(&decoded_contexts, + request->data.data, + request->data.len, + &offset); + } + if (result != CMT_DECODE_OPENTELEMETRY_SUCCESS) { flb_plg_error(ctx->ins, "could not decode metrics payload"); return -1; @@ -605,7 +600,8 @@ int opentelemetry_prot_handle(struct flb_opentelemetry *ctx, struct http_conn *c } if (strcmp(uri, "/v1/metrics") == 0) { - ret = process_payload_metrics(ctx, conn, tag, tag_len, session, request); + ret = process_payload_metrics(ctx, conn, tag, tag_len, content_type, + session, request); } else if (strcmp(uri, "/v1/traces") == 0) { ret = opentelemetry_process_traces(ctx, content_type, tag, tag_len, @@ -938,14 +934,13 @@ static int process_payload_metrics_ng(struct flb_opentelemetry *ctx, offset = 0; /* note: if the content type is gRPC, it was already decoded */ - if (strcasecmp(request->content_type, "application/json") == 0) { - flb_plg_error(ctx->ins, "Unsupported metrics with content type %s", - request->content_type); + if (opentelemetry_is_json_content_type(request->content_type) == FLB_TRUE) { + result = flb_opentelemetry_metrics_json_to_cmt(&decoded_contexts, + payload, + payload_size); } - else if (is_grpc_content_type(request->content_type) == FLB_TRUE || - strcasecmp(request->content_type, "application/x-protobuf") == 0 || - strcasecmp(request->content_type, "application/json") == 0) { - + else if (opentelemetry_is_protobuf_content_type(request->content_type) == + FLB_TRUE) { result = cmt_decode_opentelemetry_create(&decoded_contexts, payload, payload_size, @@ -1056,15 +1051,14 @@ static int process_payload_profiles_ng(struct flb_opentelemetry *ctx, flb_error("[otel] content type missing"); return -1; } - else if (strcasecmp(request->content_type, "application/json") == 0) { + else if (opentelemetry_is_json_content_type(request->content_type) == FLB_TRUE) { flb_error("[otel] unsuported profiles encoding type : %s", request->content_type); return -1; } - else if (is_grpc_content_type(request->content_type) == FLB_TRUE || - strcasecmp(request->content_type, "application/protobuf") == 0 || - strcasecmp(request->content_type, "application/x-protobuf") == 0) { + else if (opentelemetry_is_protobuf_content_type(request->content_type) == + FLB_TRUE) { profiles_context = NULL; offset = 0; @@ -1236,7 +1230,8 @@ int opentelemetry_prot_handle_ng(struct flb_http_request *request, } /* Check if the payload is gRPC compressed */ - if (grpc_request && is_grpc_content_type(request->content_type) == FLB_TRUE) { + if (grpc_request && + opentelemetry_is_grpc_content_type(request->content_type) == FLB_TRUE) { next_grpc_message: diff --git a/plugins/in_opentelemetry/opentelemetry_traces.c b/plugins/in_opentelemetry/opentelemetry_traces.c index edc0ec230da..eca51c0e0c5 100644 --- a/plugins/in_opentelemetry/opentelemetry_traces.c +++ b/plugins/in_opentelemetry/opentelemetry_traces.c @@ -27,6 +27,7 @@ #include "opentelemetry.h" #include "opentelemetry_traces.h" +#include "opentelemetry_utils.h" int opentelemetry_traces_process_protobuf(struct flb_opentelemetry *ctx, flb_sds_t tag, @@ -157,17 +158,15 @@ int opentelemetry_process_traces(struct flb_opentelemetry *ctx, /* Detect the type of payload */ if (content_type) { - if (strcasecmp(content_type, "application/json") == 0) { - if (buf[0] != '{') { + if (opentelemetry_is_json_content_type(content_type) == FLB_TRUE) { + if (opentelemetry_payload_starts_with_json_object(buf, size) != FLB_TRUE) { flb_plg_error(ctx->ins, "Invalid JSON payload"); return -1; } is_proto = FLB_FALSE; } - else if (strcasecmp(content_type, "application/protobuf") == 0 || - strcasecmp(content_type, "application/grpc") == 0 || - strcasecmp(content_type, "application/x-protobuf") == 0) { + else if (opentelemetry_is_protobuf_content_type(content_type) == FLB_TRUE) { is_proto = FLB_TRUE; } else { diff --git a/plugins/in_opentelemetry/opentelemetry_utils.c b/plugins/in_opentelemetry/opentelemetry_utils.c index 16002818c72..190d8ef072d 100644 --- a/plugins/in_opentelemetry/opentelemetry_utils.c +++ b/plugins/in_opentelemetry/opentelemetry_utils.c @@ -154,6 +154,119 @@ int json_payload_get_wrapped_value(msgpack_object *wrapper, return 0; } +static int opentelemetry_content_type_matches(const char *content_type, + const char *expected_content_type) +{ + size_t content_length; + size_t expected_length; + char trailing_character; + + if (content_type == NULL || expected_content_type == NULL) { + return FLB_FALSE; + } + + expected_length = strlen(expected_content_type); + content_length = strlen(content_type); + + if (content_length < expected_length) { + return FLB_FALSE; + } + + if (strncasecmp(content_type, expected_content_type, expected_length) != 0) { + return FLB_FALSE; + } + + trailing_character = content_type[expected_length]; + + if (trailing_character == '\0' || + trailing_character == ';' || + trailing_character == ' ' || + trailing_character == '\t') { + return FLB_TRUE; + } + + return FLB_FALSE; +} + +int opentelemetry_is_grpc_content_type(const char *content_type) +{ + size_t content_length; + + if (content_type == NULL) { + return FLB_FALSE; + } + + content_length = strlen(content_type); + if (content_length < 16) { + return FLB_FALSE; + } + + if (strncasecmp(content_type, "application/grpc", 16) != 0) { + return FLB_FALSE; + } + + if (content_type[16] == '\0' || + content_type[16] == '+' || + content_type[16] == ';' || + content_type[16] == ' ' || + content_type[16] == '\t') { + return FLB_TRUE; + } + + return FLB_FALSE; +} + +int opentelemetry_is_json_content_type(const char *content_type) +{ + return opentelemetry_content_type_matches(content_type, "application/json"); +} + +int opentelemetry_is_protobuf_content_type(const char *content_type) +{ + if (opentelemetry_content_type_matches(content_type, "application/protobuf") == + FLB_TRUE) { + return FLB_TRUE; + } + + if (opentelemetry_content_type_matches(content_type, + "application/x-protobuf") == FLB_TRUE) { + return FLB_TRUE; + } + + if (opentelemetry_is_grpc_content_type(content_type) == FLB_TRUE) { + return FLB_TRUE; + } + + return FLB_FALSE; +} + +int opentelemetry_payload_starts_with_json_object(const void *payload, + size_t payload_size) +{ + size_t index; + const unsigned char *buffer; + + if (payload == NULL || payload_size == 0) { + return FLB_FALSE; + } + + buffer = (const unsigned char *) payload; + + for (index = 0; index < payload_size; index++) { + if (isspace(buffer[index])) { + continue; + } + + if (buffer[index] == '{') { + return FLB_TRUE; + } + + return FLB_FALSE; + } + + return FLB_FALSE; +} + static int hex_to_int(char ch) { if (ch >= '0' && ch <= '9') { diff --git a/plugins/in_opentelemetry/opentelemetry_utils.h b/plugins/in_opentelemetry/opentelemetry_utils.h index 35fc586db1a..3aee0855ebe 100644 --- a/plugins/in_opentelemetry/opentelemetry_utils.h +++ b/plugins/in_opentelemetry/opentelemetry_utils.h @@ -31,6 +31,11 @@ int json_payload_get_wrapped_value(msgpack_object *wrapper, msgpack_object **value, int *type); +int opentelemetry_is_grpc_content_type(const char *content_type); +int opentelemetry_is_json_content_type(const char *content_type); +int opentelemetry_is_protobuf_content_type(const char *content_type); +int opentelemetry_payload_starts_with_json_object(const void *payload, size_t payload_size); + int hex_to_id(char *str, int len, unsigned char *out_buf, int out_size); uint64_t convert_string_number_to_u64(char *str, size_t len); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8ba8440ce0c..b67f59aad52 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -116,6 +116,7 @@ endif() set(src ${src} opentelemetry/flb_opentelemetry_logs.c + opentelemetry/flb_opentelemetry_metrics.c opentelemetry/flb_opentelemetry_traces.c opentelemetry/flb_opentelemetry_utils.c ) diff --git a/src/opentelemetry/flb_opentelemetry_metrics.c b/src/opentelemetry/flb_opentelemetry_metrics.c new file mode 100644 index 00000000000..3633c573ca3 --- /dev/null +++ b/src/opentelemetry/flb_opentelemetry_metrics.c @@ -0,0 +1,3745 @@ +/* -*- Mode: C; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ + +/* Fluent Bit + * ========== + * Copyright (C) 2015-2026 The Fluent Bit Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include + +#define OTEL_METRICS_JSON_DECODER_ERROR CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR + +static void destroy_context_list(struct cfl_list *context_list) +{ + struct cfl_list *iterator; + struct cfl_list *tmp; + struct cmt *context; + + if (context_list == NULL) { + return; + } + + cfl_list_foreach_safe(iterator, tmp, context_list) { + context = cfl_list_entry(iterator, struct cmt, _head); + cfl_list_del(&context->_head); + cmt_destroy(context); + } +} + +static int parse_u64_value(msgpack_object *obj, uint64_t *value) +{ + size_t index; + + if (obj == NULL || value == NULL) { + return -1; + } + + if (obj->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + *value = (uint64_t) obj->via.u64; + return 0; + } + else if (obj->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + if (obj->via.i64 < 0) { + return -1; + } + *value = (uint64_t) obj->via.i64; + return 0; + } + else if (obj->type == MSGPACK_OBJECT_STR) { + if (obj->via.str.size == 0 || + obj->via.str.ptr[0] == '-' || + obj->via.str.size > 31) { + return -1; + } + + for (index = 0 ; index < obj->via.str.size ; index++) { + if (!isdigit((unsigned char) obj->via.str.ptr[index])) { + return -1; + } + } + + *value = flb_otel_utils_convert_string_number_to_u64( + (char *) obj->via.str.ptr, obj->via.str.size); + return 0; + } + + return -1; +} + +static int parse_i64_value(msgpack_object *obj, int64_t *value) +{ + char *end; + flb_sds_t string_value; + uint64_t temp_u64; + int64_t temp_i64; + + if (obj == NULL || value == NULL) { + return -1; + } + + if (obj->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + temp_u64 = obj->via.u64; + + if (temp_u64 > INT64_MAX) { + return -1; + } + + *value = (int64_t) temp_u64; + return 0; + } + else if (obj->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + *value = obj->via.i64; + return 0; + } + else if (obj->type == MSGPACK_OBJECT_STR) { + string_value = flb_sds_create_len(obj->via.str.ptr, obj->via.str.size); + if (string_value == NULL) { + return -1; + } + + errno = 0; + end = NULL; + temp_i64 = strtoll(string_value, &end, 10); + + if (errno == ERANGE || + end == string_value || + (end != NULL && *end != '\0')) { + flb_sds_destroy(string_value); + return -1; + } + + flb_sds_destroy(string_value); + + *value = temp_i64; + return 0; + } + + return -1; +} + +static int parse_double_value(msgpack_object *obj, double *value) +{ + char *end; + flb_sds_t string_value; + + if (obj->type == MSGPACK_OBJECT_FLOAT32) { + *value = (double) obj->via.f64; + return 0; + } + else if (obj->type == MSGPACK_OBJECT_FLOAT64) { + *value = obj->via.f64; + return 0; + } + else if (obj->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + *value = (double) obj->via.u64; + return 0; + } + else if (obj->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + *value = (double) obj->via.i64; + return 0; + } + else if (obj->type == MSGPACK_OBJECT_STR) { + string_value = flb_sds_create_len(obj->via.str.ptr, obj->via.str.size); + if (string_value == NULL) { + return -1; + } + + end = NULL; + *value = strtod(string_value, &end); + + if (end == string_value || (end != NULL && *end != '\0')) { + flb_sds_destroy(string_value); + return -1; + } + + flb_sds_destroy(string_value); + + return 0; + } + + return -1; +} + +static int parse_i32_value(msgpack_object *obj, int32_t *value) +{ + char *end; + flb_sds_t string_value; + uint64_t temp_u64; + int64_t temp_i64; + + if (obj->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + temp_u64 = obj->via.u64; + + if (temp_u64 > INT32_MAX) { + return -1; + } + + *value = (int32_t) temp_u64; + return 0; + } + else if (obj->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + temp_i64 = obj->via.i64; + + if (temp_i64 < INT32_MIN || temp_i64 > INT32_MAX) { + return -1; + } + + *value = (int32_t) temp_i64; + return 0; + } + else if (obj->type == MSGPACK_OBJECT_STR) { + string_value = flb_sds_create_len(obj->via.str.ptr, obj->via.str.size); + if (string_value == NULL) { + return -1; + } + + end = NULL; + temp_i64 = strtoll(string_value, &end, 10); + + if (end == string_value || (end != NULL && *end != '\0')) { + flb_sds_destroy(string_value); + return -1; + } + + flb_sds_destroy(string_value); + + if (temp_i64 < INT32_MIN || temp_i64 > INT32_MAX) { + return -1; + } + + *value = (int32_t) temp_i64; + return 0; + } + + return -1; +} + +static int get_metric_help_string(msgpack_object_map *metric_map, + flb_sds_t *out_help) +{ + int help_index; + msgpack_object *help_object; + + *out_help = NULL; + + help_index = flb_otel_utils_find_map_entry_by_key(metric_map, + "description", + 0, + FLB_TRUE); + if (help_index < 0) { + *out_help = flb_sds_create("-"); + } + else { + help_object = &metric_map->ptr[help_index].val; + + if (help_object->type != MSGPACK_OBJECT_STR) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + if (help_object->via.str.size == 0) { + *out_help = flb_sds_create("-"); + } + else { + *out_help = flb_sds_create_len(help_object->via.str.ptr, + help_object->via.str.size); + } + } + + if (*out_help == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + return 0; +} + +static int set_metric_unit(struct cmt_map *map, msgpack_object_map *metric_map) +{ + int unit_index; + msgpack_object *unit_object; + + unit_index = flb_otel_utils_find_map_entry_by_key(metric_map, + "unit", + 0, + FLB_TRUE); + if (unit_index < 0) { + return 0; + } + + unit_object = &metric_map->ptr[unit_index].val; + if (unit_object->type != MSGPACK_OBJECT_STR) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + if (map->unit != NULL) { + cfl_sds_destroy(map->unit); + map->unit = NULL; + } + + if (unit_object->via.str.size == 0) { + return 0; + } + + map->unit = cfl_sds_create_len(unit_object->via.str.ptr, + unit_object->via.str.size); + if (map->unit == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + return 0; +} + +static char *map_type_to_key(int map_type) +{ + switch (map_type) { + case CMT_COUNTER: + return "counter"; + case CMT_GAUGE: + return "gauge"; + case CMT_UNTYPED: + return "untyped"; + case CMT_SUMMARY: + return "summary"; + case CMT_HISTOGRAM: + return "histogram"; + case CMT_EXP_HISTOGRAM: + return "exp_histogram"; + default: + return "unknown"; + } +} + +static struct cfl_kvlist *get_or_create_external_metadata_kvlist( + struct cfl_kvlist *root, char *key) +{ + struct cfl_variant *entry_variant; + struct cfl_kvlist *entry_kvlist; + int result; + + entry_variant = cfl_kvlist_fetch(root, key); + + if (entry_variant == NULL) { + entry_kvlist = cfl_kvlist_create(); + + if (entry_kvlist == NULL) { + return NULL; + } + + result = cfl_kvlist_insert_kvlist(root, key, entry_kvlist); + if (result != 0) { + cfl_kvlist_destroy(entry_kvlist); + return NULL; + } + } + else { + entry_kvlist = entry_variant->data.as_kvlist; + } + + return entry_kvlist; +} + +static uint64_t compute_metric_hash(struct cmt_map *map, struct cmt_metric *sample) +{ + struct cfl_list *head; + struct cmt_map_label *label_value; + cfl_hash_state_t hash_state; + + if (sample == NULL || map == NULL) { + return 0; + } + + if (cfl_list_size(&sample->labels) == 0) { + return 0; + } + + cfl_hash_64bits_reset(&hash_state); + cfl_hash_64bits_update(&hash_state, + map->opts->fqname, + cfl_sds_len(map->opts->fqname)); + + cfl_list_foreach(head, &sample->labels) { + label_value = cfl_list_entry(head, struct cmt_map_label, _head); + cfl_hash_64bits_update(&hash_state, + label_value->name, + cfl_sds_len(label_value->name)); + } + + return cfl_hash_64bits_digest(&hash_state); +} + +static struct cfl_kvlist *get_or_create_metric_metadata_context(struct cmt *cmt, + struct cmt_map *map) +{ + struct cfl_kvlist *otlp_root; + struct cfl_kvlist *metrics_root; + struct cfl_kvlist *type_root; + + if (cmt == NULL || map == NULL || map->opts == NULL || map->opts->fqname == NULL) { + return NULL; + } + + otlp_root = get_or_create_external_metadata_kvlist(cmt->external_metadata, "otlp"); + if (otlp_root == NULL) { + return NULL; + } + + metrics_root = get_or_create_external_metadata_kvlist(otlp_root, "metrics"); + if (metrics_root == NULL) { + return NULL; + } + + type_root = get_or_create_external_metadata_kvlist(metrics_root, + map_type_to_key(map->type)); + if (type_root == NULL) { + return NULL; + } + + return get_or_create_external_metadata_kvlist(type_root, map->opts->fqname); +} + +static struct cfl_kvlist *get_or_create_data_point_metadata_context( + struct cmt *cmt, + struct cmt_map *map, + struct cmt_metric *sample, + uint64_t timestamp) +{ + char key[128]; + struct cfl_kvlist *metric_context; + struct cfl_kvlist *datapoints_context; + + if (sample != NULL && sample->hash == 0 && cfl_list_size(&sample->labels) > 0) { + sample->hash = compute_metric_hash(map, sample); + } + + metric_context = get_or_create_metric_metadata_context(cmt, map); + if (metric_context == NULL) { + return NULL; + } + + datapoints_context = get_or_create_external_metadata_kvlist(metric_context, + "datapoints"); + if (datapoints_context == NULL) { + return NULL; + } + + snprintf(key, sizeof(key) - 1, "%llx:%llu", + (unsigned long long) (sample != NULL ? sample->hash : 0), + (unsigned long long) timestamp); + + return get_or_create_external_metadata_kvlist(datapoints_context, key); +} + +static int object_to_sds(msgpack_object *obj, flb_sds_t *out) +{ + flb_sds_t value; + + value = NULL; + + if (obj->type == MSGPACK_OBJECT_STR) { + value = flb_sds_create_len(obj->via.str.ptr, obj->via.str.size); + } + else if (obj->type == MSGPACK_OBJECT_BOOLEAN) { + value = flb_sds_create_size(4); + if (value != NULL) { + flb_sds_printf(&value, "%d", obj->via.boolean ? 1 : 0); + } + } + else if (obj->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + value = flb_sds_create_size(32); + if (value != NULL) { + flb_sds_printf(&value, "%llu", + (unsigned long long) obj->via.u64); + } + } + else if (obj->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + value = flb_sds_create_size(32); + if (value != NULL) { + flb_sds_printf(&value, "%lld", + (long long) obj->via.i64); + } + } + else if (obj->type == MSGPACK_OBJECT_FLOAT32 || + obj->type == MSGPACK_OBJECT_FLOAT64) { + value = flb_sds_create_size(64); + if (value != NULL) { + flb_sds_printf(&value, "%.17g", obj->via.f64); + } + } + else if (obj->type == MSGPACK_OBJECT_NIL) { + value = flb_sds_create(""); + } + + if (value == NULL) { + return -1; + } + + *out = value; + + return 0; +} + +static int otel_any_value_to_string(msgpack_object *wrapper, flb_sds_t *out) +{ + msgpack_object *value; + int type; + int result; + + result = flb_otel_utils_json_payload_get_wrapped_value(wrapper, &value, &type); + if (result != 0 || value == NULL) { + return -1; + } + + return object_to_sds(value, out); +} + +static void destroy_label_arrays(int count, + flb_sds_t *keys, + flb_sds_t *values) +{ + int index; + + if (keys != NULL) { + for (index = 0 ; index < count ; index++) { + if (keys[index] != NULL) { + flb_sds_destroy(keys[index]); + } + } + flb_free(keys); + } + + if (values != NULL) { + for (index = 0 ; index < count ; index++) { + if (values[index] != NULL) { + flb_sds_destroy(values[index]); + } + } + flb_free(values); + } +} + +static int parse_datapoint_labels(msgpack_object *attributes_object, + int *out_count, + flb_sds_t **out_keys, + flb_sds_t **out_values) +{ + int attribute_index; + int key_index; + int value_index; + flb_sds_t *keys; + flb_sds_t *values; + msgpack_object *entry; + msgpack_object_map *entry_map; + msgpack_object *key_object; + msgpack_object *value_object; + msgpack_object_array *attributes; + int result; + + *out_count = 0; + *out_keys = NULL; + *out_values = NULL; + + if (attributes_object == NULL) { + return 0; + } + + if (attributes_object->type != MSGPACK_OBJECT_ARRAY) { + return -1; + } + + attributes = &attributes_object->via.array; + if (attributes->size == 0) { + return 0; + } + + keys = flb_calloc(attributes->size, sizeof(flb_sds_t)); + if (keys == NULL) { + flb_errno(); + return -1; + } + + values = flb_calloc(attributes->size, sizeof(flb_sds_t)); + if (values == NULL) { + flb_errno(); + destroy_label_arrays((int) attributes->size, keys, NULL); + return -1; + } + + for (attribute_index = 0 ; attribute_index < attributes->size ; attribute_index++) { + entry = &attributes->ptr[attribute_index]; + if (entry->type != MSGPACK_OBJECT_MAP) { + continue; + } + + entry_map = &entry->via.map; + key_index = flb_otel_utils_find_map_entry_by_key(entry_map, "key", 0, FLB_TRUE); + value_index = flb_otel_utils_find_map_entry_by_key(entry_map, "value", 0, FLB_TRUE); + + if (key_index < 0 || value_index < 0) { + continue; + } + + key_object = &entry_map->ptr[key_index].val; + value_object = &entry_map->ptr[value_index].val; + + if (key_object->type != MSGPACK_OBJECT_STR) { + continue; + } + + keys[*out_count] = flb_sds_create_len(key_object->via.str.ptr, + key_object->via.str.size); + if (keys[*out_count] == NULL) { + destroy_label_arrays((int) attributes->size, keys, values); + return -1; + } + + result = otel_any_value_to_string(value_object, &values[*out_count]); + if (result != 0) { + values[*out_count] = flb_sds_create(""); + } + + if (values[*out_count] == NULL) { + destroy_label_arrays((int) attributes->size, keys, values); + return -1; + } + + (*out_count)++; + } + + *out_keys = keys; + *out_values = values; + + return 0; +} + +static int check_label_layout(int expected_count, + flb_sds_t *expected_keys, + int point_label_count, + flb_sds_t *point_keys, + flb_sds_t *point_values) +{ + int expected_index; + int point_index; + int *used_indexes; + flb_sds_t *ordered_values; + + if (expected_count != point_label_count) { + return -1; + } + + used_indexes = flb_calloc(point_label_count, sizeof(int)); + if (used_indexes == NULL) { + flb_errno(); + return -1; + } + + ordered_values = flb_calloc(point_label_count, sizeof(flb_sds_t)); + if (ordered_values == NULL) { + flb_errno(); + flb_free(used_indexes); + return -1; + } + + for (expected_index = 0 ; expected_index < expected_count ; expected_index++) { + point_index = -1; + + for (point_index = 0 ; point_index < point_label_count ; point_index++) { + if (used_indexes[point_index] != FLB_FALSE) { + continue; + } + + if (strcmp(expected_keys[expected_index], point_keys[point_index]) == 0) { + used_indexes[point_index] = FLB_TRUE; + ordered_values[expected_index] = point_values[point_index]; + point_values[point_index] = NULL; + break; + } + } + + if (point_index >= point_label_count) { + flb_free(ordered_values); + flb_free(used_indexes); + return -1; + } + } + + for (point_index = 0 ; point_index < point_label_count ; point_index++) { + if (point_values[point_index] != NULL) { + flb_sds_destroy(point_values[point_index]); + } + point_values[point_index] = ordered_values[point_index]; + } + + destroy_label_arrays(point_label_count, point_keys, NULL); + flb_free(ordered_values); + flb_free(used_indexes); + + return 0; +} + +static int parse_number_datapoint_value(msgpack_object_map *point_map, + int *number_value_case, + int64_t *integer_value, + double *double_value) +{ + int result; + msgpack_object *obj; + + *number_value_case = CMT_METRIC_VALUE_DOUBLE; + *integer_value = 0; + *double_value = 0; + + result = flb_otel_utils_find_map_entry_by_key(point_map, "asDouble", 0, FLB_TRUE); + if (result >= 0) { + obj = &point_map->ptr[result].val; + return parse_double_value(obj, double_value); + } + + result = flb_otel_utils_find_map_entry_by_key(point_map, "asInt", 0, FLB_TRUE); + if (result >= 0) { + obj = &point_map->ptr[result].val; + *number_value_case = CMT_METRIC_VALUE_INT64; + return parse_i64_value(obj, integer_value); + } + + return -1; +} + +static int parse_datapoint_timestamp(msgpack_object_map *point_map, + uint64_t *timestamp) +{ + int result; + msgpack_object *obj; + + *timestamp = 0; + + result = flb_otel_utils_find_map_entry_by_key(point_map, "timeUnixNano", 0, FLB_TRUE); + if (result < 0) { + return 0; + } + + obj = &point_map->ptr[result].val; + return parse_u64_value(obj, timestamp); +} + +static int clone_exemplars_to_kvlist(struct cfl_kvlist *target, + msgpack_object *exemplars_object) +{ + int index; + int time_unix_nano_index; + int span_id_index; + int trace_id_index; + int as_double_index; + int as_int_index; + int filtered_attributes_index; + int result; + unsigned char id_buffer[32]; + size_t decoded_length; + uint64_t time_unix_nano; + int64_t as_int_value; + double as_double_value; + msgpack_object *exemplar_object; + msgpack_object *field_object; + msgpack_object_map *exemplar_map; + msgpack_object_array *exemplar_array; + struct cfl_array *array; + struct cfl_kvlist *entry; + struct cfl_kvlist *filtered_attributes; + + if (target == NULL || exemplars_object == NULL) { + return 0; + } + + if (exemplars_object->type != MSGPACK_OBJECT_ARRAY) { + return -1; + } + + exemplar_array = &exemplars_object->via.array; + if (exemplar_array->size == 0) { + return 0; + } + + array = cfl_array_create(exemplar_array->size); + if (array == NULL) { + return -1; + } + + for (index = 0 ; index < exemplar_array->size ; index++) { + exemplar_object = &exemplar_array->ptr[index]; + if (exemplar_object->type != MSGPACK_OBJECT_MAP) { + cfl_array_destroy(array); + return -1; + } + + entry = cfl_kvlist_create(); + if (entry == NULL) { + cfl_array_destroy(array); + return -1; + } + + exemplar_map = &exemplar_object->via.map; + + time_unix_nano_index = flb_otel_utils_find_map_entry_by_key(exemplar_map, + "timeUnixNano", + 0, + FLB_TRUE); + if (time_unix_nano_index >= 0) { + field_object = &exemplar_map->ptr[time_unix_nano_index].val; + result = parse_u64_value(field_object, &time_unix_nano); + if (result != 0 || + cfl_kvlist_insert_uint64(entry, + "time_unix_nano", + time_unix_nano) != 0) { + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + } + + span_id_index = flb_otel_utils_find_map_entry_by_key(exemplar_map, + "spanId", + 0, + FLB_TRUE); + if (span_id_index >= 0) { + field_object = &exemplar_map->ptr[span_id_index].val; + if (field_object->type == MSGPACK_OBJECT_STR) { + if (field_object->via.str.size == 16 && + flb_otel_utils_hex_to_id(field_object->via.str.ptr, + field_object->via.str.size, + id_buffer, + 8) == 0) { + result = cfl_kvlist_insert_bytes(entry, + "span_id", + (char *) id_buffer, + 8, + CFL_FALSE); + } + else { + decoded_length = 0; + if (flb_base64_decode(id_buffer, + sizeof(id_buffer), + &decoded_length, + (unsigned char *) field_object->via.str.ptr, + field_object->via.str.size) == 0 && + decoded_length > 0 && + decoded_length <= sizeof(id_buffer)) { + result = cfl_kvlist_insert_bytes(entry, + "span_id", + (char *) id_buffer, + decoded_length, + CFL_FALSE); + } + else { + result = cfl_kvlist_insert_bytes(entry, + "span_id", + (char *) field_object->via.str.ptr, + field_object->via.str.size, + CFL_FALSE); + } + } + + if (result != 0) { + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + } + } + + trace_id_index = flb_otel_utils_find_map_entry_by_key(exemplar_map, + "traceId", + 0, + FLB_TRUE); + if (trace_id_index >= 0) { + field_object = &exemplar_map->ptr[trace_id_index].val; + if (field_object->type == MSGPACK_OBJECT_STR) { + if (field_object->via.str.size == 32 && + flb_otel_utils_hex_to_id(field_object->via.str.ptr, + field_object->via.str.size, + id_buffer, + 16) == 0) { + result = cfl_kvlist_insert_bytes(entry, + "trace_id", + (char *) id_buffer, + 16, + CFL_FALSE); + } + else { + decoded_length = 0; + if (flb_base64_decode(id_buffer, + sizeof(id_buffer), + &decoded_length, + (unsigned char *) field_object->via.str.ptr, + field_object->via.str.size) == 0 && + decoded_length > 0 && + decoded_length <= sizeof(id_buffer)) { + result = cfl_kvlist_insert_bytes(entry, + "trace_id", + (char *) id_buffer, + decoded_length, + CFL_FALSE); + } + else { + result = cfl_kvlist_insert_bytes(entry, + "trace_id", + (char *) field_object->via.str.ptr, + field_object->via.str.size, + CFL_FALSE); + } + } + + if (result != 0) { + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + } + } + + as_double_index = flb_otel_utils_find_map_entry_by_key(exemplar_map, + "asDouble", + 0, + FLB_TRUE); + if (as_double_index >= 0) { + field_object = &exemplar_map->ptr[as_double_index].val; + if (parse_double_value(field_object, &as_double_value) == 0) { + if (cfl_kvlist_insert_double(entry, + "as_double", + as_double_value) != 0) { + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + } + } + else { + as_int_index = flb_otel_utils_find_map_entry_by_key(exemplar_map, + "asInt", + 0, + FLB_TRUE); + if (as_int_index >= 0) { + flb_sds_t string_value; + + field_object = &exemplar_map->ptr[as_int_index].val; + if (field_object->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + as_int_value = field_object->via.i64; + } + else if (field_object->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + as_int_value = field_object->via.u64; + } + else if (field_object->type == MSGPACK_OBJECT_STR) { + string_value = flb_sds_create_len(field_object->via.str.ptr, + field_object->via.str.size); + if (string_value == NULL) { + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + + as_int_value = strtoll(string_value, NULL, 10); + flb_sds_destroy(string_value); + } + else { + as_int_value = 0; + } + + if (cfl_kvlist_insert_int64(entry, + "as_int", + as_int_value) != 0) { + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + } + } + + filtered_attributes_index = flb_otel_utils_find_map_entry_by_key( + exemplar_map, + "filteredAttributes", + 0, + FLB_TRUE); + if (filtered_attributes_index >= 0) { + filtered_attributes = cfl_kvlist_create(); + if (filtered_attributes == NULL) { + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + + field_object = &exemplar_map->ptr[filtered_attributes_index].val; + if (flb_otel_utils_clone_kvlist_from_otlp_json_array( + filtered_attributes, + field_object) != 0) { + cfl_kvlist_destroy(filtered_attributes); + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + + if (cfl_kvlist_insert_kvlist(entry, + "filtered_attributes", + filtered_attributes) != 0) { + cfl_kvlist_destroy(filtered_attributes); + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + } + + if (cfl_array_append_kvlist(array, entry) != 0) { + cfl_kvlist_destroy(entry); + cfl_array_destroy(array); + return -1; + } + } + + if (cfl_kvlist_insert_array(target, "exemplars", array) != 0) { + cfl_array_destroy(array); + return -1; + } + + return 0; +} + +static int parse_optional_datapoint_u64(msgpack_object_map *point_map, + char *key, + uint64_t *out_value) +{ + int result; + msgpack_object *value_object; + + result = flb_otel_utils_find_map_entry_by_key(point_map, + key, + 0, + FLB_TRUE); + if (result < 0) { + return -1; + } + + value_object = &point_map->ptr[result].val; + if (parse_u64_value(value_object, out_value) != 0) { + return -1; + } + + return 0; +} + +static void append_common_datapoint_metadata(struct cfl_kvlist *point_metadata, + msgpack_object_map *point_map) +{ + int result; + uint64_t start_time_unix_nano; + uint64_t flags; + msgpack_object *exemplars_object; + + if (point_metadata == NULL || point_map == NULL) { + return; + } + + result = parse_optional_datapoint_u64(point_map, + "startTimeUnixNano", + &start_time_unix_nano); + if (result == 0) { + cfl_kvlist_insert_uint64(point_metadata, + "start_time_unix_nano", + start_time_unix_nano); + } + + result = parse_optional_datapoint_u64(point_map, "flags", &flags); + if (result == 0) { + cfl_kvlist_insert_uint64(point_metadata, "flags", flags); + } + + result = flb_otel_utils_find_map_entry_by_key(point_map, + "exemplars", + 0, + FLB_TRUE); + if (result >= 0) { + exemplars_object = &point_map->ptr[result].val; + clone_exemplars_to_kvlist(point_metadata, exemplars_object); + } +} + +static int clone_metric_metadata(struct cmt *context, + struct cmt_map *map, + msgpack_object_map *metric_map) +{ + int metadata_index; + msgpack_object *metadata_object; + struct cfl_kvlist *metric_context; + struct cfl_kvlist *metadata_context; + + metadata_index = flb_otel_utils_find_map_entry_by_key(metric_map, + "metadata", + 0, + FLB_TRUE); + if (metadata_index < 0) { + return 0; + } + + metadata_object = &metric_map->ptr[metadata_index].val; + if (metadata_object->type != MSGPACK_OBJECT_ARRAY) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + metric_context = get_or_create_metric_metadata_context(context, map); + if (metric_context == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + metadata_context = get_or_create_external_metadata_kvlist(metric_context, "metadata"); + if (metadata_context == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + if (flb_otel_utils_clone_kvlist_from_otlp_json_array( + metadata_context, + metadata_object) != 0) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + return 0; +} + +static int clone_scope_metadata_and_attributes(struct cfl_kvlist *external_metadata, + msgpack_object *scope_object) +{ + int result; + int index; + uint64_t dropped_attributes_count; + msgpack_object *field_object; + msgpack_object_map *scope_map; + struct cfl_kvlist *root; + struct cfl_kvlist *metadata; + struct cfl_kvlist *attributes; + + root = get_or_create_external_metadata_kvlist(external_metadata, "scope"); + if (root == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + metadata = get_or_create_external_metadata_kvlist(root, "metadata"); + if (metadata == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + attributes = get_or_create_external_metadata_kvlist(root, "attributes"); + if (attributes == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + if (scope_object == NULL) { + return 0; + } + + if (scope_object->type != MSGPACK_OBJECT_MAP) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + scope_map = &scope_object->via.map; + + index = flb_otel_utils_find_map_entry_by_key(scope_map, "name", 0, FLB_TRUE); + if (index >= 0) { + field_object = &scope_map->ptr[index].val; + if (field_object->type != MSGPACK_OBJECT_STR || + cfl_kvlist_insert_string_s(metadata, + "name", + 4, + (char *) field_object->via.str.ptr, + field_object->via.str.size, + CFL_FALSE) != 0) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + } + + index = flb_otel_utils_find_map_entry_by_key(scope_map, "version", 0, FLB_TRUE); + if (index >= 0) { + field_object = &scope_map->ptr[index].val; + if (field_object->type != MSGPACK_OBJECT_STR || + cfl_kvlist_insert_string_s(metadata, + "version", + 7, + (char *) field_object->via.str.ptr, + field_object->via.str.size, + CFL_FALSE) != 0) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + } + + index = flb_otel_utils_find_map_entry_by_key(scope_map, + "droppedAttributesCount", + 0, + FLB_TRUE); + if (index >= 0) { + field_object = &scope_map->ptr[index].val; + + result = parse_u64_value(field_object, &dropped_attributes_count); + if (result != 0 || + cfl_kvlist_insert_int64(metadata, + "dropped_attributes_count", + (int64_t) dropped_attributes_count) != 0) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + } + + index = flb_otel_utils_find_map_entry_by_key(scope_map, "attributes", 0, FLB_TRUE); + if (index >= 0) { + field_object = &scope_map->ptr[index].val; + if (field_object->type != MSGPACK_OBJECT_ARRAY) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + if (flb_otel_utils_clone_kvlist_from_otlp_json_array(attributes, + field_object) != 0) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + } + + return 0; +} + +static int clone_scope_metrics_metadata(struct cfl_kvlist *external_metadata, + msgpack_object_map *scope_metrics_map) +{ + int index; + msgpack_object *schema_url; + struct cfl_kvlist *root; + struct cfl_kvlist *metadata; + + root = get_or_create_external_metadata_kvlist(external_metadata, "scope_metrics"); + if (root == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + metadata = get_or_create_external_metadata_kvlist(root, "metadata"); + if (metadata == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + index = flb_otel_utils_find_map_entry_by_key(scope_metrics_map, + "schemaUrl", + 0, + FLB_TRUE); + if (index < 0) { + return 0; + } + + schema_url = &scope_metrics_map->ptr[index].val; + if (schema_url->type != MSGPACK_OBJECT_STR || + cfl_kvlist_insert_string_s(metadata, + "schema_url", + 10, + (char *) schema_url->via.str.ptr, + schema_url->via.str.size, + CFL_FALSE) != 0) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + return 0; +} + +static int clone_resource_metadata_and_attributes(struct cfl_kvlist *external_metadata, + msgpack_object *resource_object) +{ + int result; + int index; + uint64_t dropped_attributes_count; + msgpack_object *field_object; + msgpack_object_map *resource_map; + struct cfl_kvlist *root; + struct cfl_kvlist *metadata; + struct cfl_kvlist *attributes; + + root = get_or_create_external_metadata_kvlist(external_metadata, "resource"); + if (root == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + metadata = get_or_create_external_metadata_kvlist(root, "metadata"); + if (metadata == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + attributes = get_or_create_external_metadata_kvlist(root, "attributes"); + if (attributes == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + if (resource_object == NULL) { + return 0; + } + + if (resource_object->type != MSGPACK_OBJECT_MAP) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + resource_map = &resource_object->via.map; + + index = flb_otel_utils_find_map_entry_by_key(resource_map, + "droppedAttributesCount", + 0, + FLB_TRUE); + if (index >= 0) { + field_object = &resource_map->ptr[index].val; + + result = parse_u64_value(field_object, &dropped_attributes_count); + if (result != 0 || + cfl_kvlist_insert_int64(metadata, + "dropped_attributes_count", + (int64_t) dropped_attributes_count) != 0) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + } + + index = flb_otel_utils_find_map_entry_by_key(resource_map, + "attributes", + 0, + FLB_TRUE); + if (index >= 0) { + field_object = &resource_map->ptr[index].val; + if (field_object->type != MSGPACK_OBJECT_ARRAY) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + if (flb_otel_utils_clone_kvlist_from_otlp_json_array(attributes, + field_object) != 0) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + } + + return 0; +} + +static int clone_resource_metrics_metadata(struct cfl_kvlist *external_metadata, + msgpack_object_map *resource_metrics_map) +{ + int index; + msgpack_object *schema_url; + struct cfl_kvlist *root; + struct cfl_kvlist *metadata; + + root = get_or_create_external_metadata_kvlist(external_metadata, "resource_metrics"); + if (root == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + metadata = get_or_create_external_metadata_kvlist(root, "metadata"); + if (metadata == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + index = flb_otel_utils_find_map_entry_by_key(resource_metrics_map, + "schemaUrl", + 0, + FLB_TRUE); + if (index < 0) { + return 0; + } + + schema_url = &resource_metrics_map->ptr[index].val; + if (schema_url->type != MSGPACK_OBJECT_STR || + cfl_kvlist_insert_string_s(metadata, + "schema_url", + 10, + (char *) schema_url->via.str.ptr, + schema_url->via.str.size, + CFL_FALSE) != 0) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + return 0; +} + +static int process_metric_gauge_data_points(struct cmt *context, + msgpack_object_map *metric_map, + msgpack_object *name_object, + msgpack_object_array *data_points) +{ + struct cmt_gauge *gauge; + msgpack_object *point; + msgpack_object *attributes_obj; + msgpack_object_map *point_map; + flb_sds_t *metric_label_keys; + int metric_label_count; + int point_label_count; + flb_sds_t *point_label_keys; + flb_sds_t *point_label_values; + uint64_t timestamp; + int64_t int_value; + double value; + int index; + int result; + int number_value_case; + int metric_initialized; + flb_sds_t metric_name; + flb_sds_t metric_help; + struct cmt_metric *sample; + struct cfl_kvlist *point_metadata; + + metric_label_keys = NULL; + metric_label_count = 0; + point_label_keys = NULL; + point_label_values = NULL; + gauge = NULL; + metric_name = NULL; + metric_help = NULL; + sample = NULL; + point_metadata = NULL; + metric_initialized = FLB_FALSE; + + for (index = 0 ; index < data_points->size ; index++) { + point = &data_points->ptr[index]; + if (point->type != MSGPACK_OBJECT_MAP) { + continue; + } + + point_map = &point->via.map; + + result = parse_number_datapoint_value(point_map, + &number_value_case, + &int_value, + &value); + if (result != 0) { + continue; + } + + result = parse_datapoint_timestamp(point_map, ×tamp); + if (result != 0) { + continue; + } + + result = flb_otel_utils_find_map_entry_by_key(point_map, "attributes", 0, FLB_TRUE); + if (result >= 0) { + attributes_obj = &point_map->ptr[result].val; + } + else { + attributes_obj = NULL; + } + + point_label_count = 0; + point_label_keys = NULL; + point_label_values = NULL; + + result = parse_datapoint_labels(attributes_obj, + &point_label_count, + &point_label_keys, + &point_label_values); + if (result != 0) { + destroy_label_arrays(point_label_count, point_label_keys, point_label_values); + continue; + } + + if (!metric_initialized) { + metric_label_count = point_label_count; + metric_label_keys = point_label_keys; + point_label_keys = NULL; + + metric_name = flb_sds_create_len(name_object->via.str.ptr, + name_object->via.str.size); + if (metric_name == NULL) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + result = get_metric_help_string(metric_map, &metric_help); + if (result != 0) { + flb_sds_destroy(metric_name); + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + return result; + } + + gauge = cmt_gauge_create(context, "", "", + metric_name, + metric_help, metric_label_count, + (char **) metric_label_keys); + flb_sds_destroy(metric_name); + flb_sds_destroy(metric_help); + metric_help = NULL; + + if (gauge == NULL) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + result = set_metric_unit(gauge->map, metric_map); + if (result != 0) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + cmt_gauge_destroy(gauge); + return result; + } + + result = clone_metric_metadata(context, gauge->map, metric_map); + if (result != 0) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + cmt_gauge_destroy(gauge); + return result; + } + + metric_initialized = FLB_TRUE; + } + else { + result = check_label_layout(metric_label_count, + metric_label_keys, + point_label_count, + point_label_keys, + point_label_values); + if (result != 0) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + continue; + } + } + + sample = cmt_map_metric_get(&gauge->opts, + gauge->map, + point_label_count, + (char **) point_label_values, + CMT_TRUE); + if (sample != NULL) { + if (number_value_case == CMT_METRIC_VALUE_INT64) { + cmt_metric_set_int64(sample, timestamp, int_value); + } + else { + cmt_metric_set_double(sample, timestamp, value); + } + + point_metadata = get_or_create_data_point_metadata_context(context, + gauge->map, + sample, + timestamp); + if (point_metadata != NULL) { + append_common_datapoint_metadata(point_metadata, point_map); + + if (number_value_case == CMT_METRIC_VALUE_INT64) { + cfl_kvlist_insert_string(point_metadata, + "number_value_case", + "int"); + } + else { + cfl_kvlist_insert_string(point_metadata, + "number_value_case", + "double"); + } + } + } + + destroy_label_arrays(point_label_count, NULL, point_label_values); + } + + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + + return 0; +} + +static int process_metric_sum_data_points(struct cmt *context, + msgpack_object_map *metric_map, + msgpack_object *name_object, + int allow_reset, + int aggregation_type, + msgpack_object_array *data_points) +{ + struct cmt_counter *counter; + msgpack_object *point; + msgpack_object *attributes_obj; + msgpack_object_map *point_map; + flb_sds_t *metric_label_keys; + int metric_label_count; + int point_label_count; + flb_sds_t *point_label_keys; + flb_sds_t *point_label_values; + uint64_t timestamp; + int64_t int_value; + double value; + int index; + int result; + int number_value_case; + int metric_initialized; + flb_sds_t metric_name; + flb_sds_t metric_help; + struct cmt_metric *sample; + struct cfl_kvlist *point_metadata; + + metric_label_keys = NULL; + metric_label_count = 0; + point_label_keys = NULL; + point_label_values = NULL; + counter = NULL; + metric_name = NULL; + metric_help = NULL; + sample = NULL; + point_metadata = NULL; + metric_initialized = FLB_FALSE; + + for (index = 0 ; index < data_points->size ; index++) { + point = &data_points->ptr[index]; + if (point->type != MSGPACK_OBJECT_MAP) { + continue; + } + + point_map = &point->via.map; + + result = parse_number_datapoint_value(point_map, + &number_value_case, + &int_value, + &value); + if (result != 0) { + continue; + } + + result = parse_datapoint_timestamp(point_map, ×tamp); + if (result != 0) { + continue; + } + + result = flb_otel_utils_find_map_entry_by_key(point_map, "attributes", 0, FLB_TRUE); + if (result >= 0) { + attributes_obj = &point_map->ptr[result].val; + } + else { + attributes_obj = NULL; + } + + point_label_count = 0; + point_label_keys = NULL; + point_label_values = NULL; + + result = parse_datapoint_labels(attributes_obj, + &point_label_count, + &point_label_keys, + &point_label_values); + if (result != 0) { + destroy_label_arrays(point_label_count, point_label_keys, point_label_values); + continue; + } + + if (!metric_initialized) { + metric_label_count = point_label_count; + metric_label_keys = point_label_keys; + point_label_keys = NULL; + + metric_name = flb_sds_create_len(name_object->via.str.ptr, + name_object->via.str.size); + if (metric_name == NULL) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + result = get_metric_help_string(metric_map, &metric_help); + if (result != 0) { + flb_sds_destroy(metric_name); + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + return result; + } + + counter = cmt_counter_create(context, "", "", + metric_name, + metric_help, metric_label_count, + (char **) metric_label_keys); + flb_sds_destroy(metric_name); + flb_sds_destroy(metric_help); + metric_help = NULL; + + if (counter == NULL) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + result = set_metric_unit(counter->map, metric_map); + if (result != 0) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + cmt_counter_destroy(counter); + return result; + } + + result = clone_metric_metadata(context, counter->map, metric_map); + if (result != 0) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + cmt_counter_destroy(counter); + return result; + } + + if (allow_reset) { + cmt_counter_allow_reset(counter); + } + + counter->aggregation_type = aggregation_type; + metric_initialized = FLB_TRUE; + } + else { + result = check_label_layout(metric_label_count, + metric_label_keys, + point_label_count, + point_label_keys, + point_label_values); + if (result != 0) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + continue; + } + } + + sample = cmt_map_metric_get(&counter->opts, + counter->map, + point_label_count, + (char **) point_label_values, + CMT_TRUE); + if (sample != NULL) { + if (number_value_case == CMT_METRIC_VALUE_INT64) { + if (allow_reset == FLB_FALSE && int_value < 0) { + cmt_metric_set_double(sample, timestamp, 0.0); + } + else { + cmt_metric_set_int64(sample, timestamp, int_value); + } + } + else { + cmt_metric_set_double(sample, timestamp, value); + } + + point_metadata = get_or_create_data_point_metadata_context(context, + counter->map, + sample, + timestamp); + if (point_metadata != NULL) { + append_common_datapoint_metadata(point_metadata, point_map); + + if (number_value_case == CMT_METRIC_VALUE_INT64) { + cfl_kvlist_insert_string(point_metadata, + "number_value_case", + "int"); + } + else { + cfl_kvlist_insert_string(point_metadata, + "number_value_case", + "double"); + } + } + } + + destroy_label_arrays(point_label_count, NULL, point_label_values); + } + + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + + return 0; +} + +static int parse_histogram_bounds(msgpack_object *explicit_bounds_object, + double **out_bounds, + size_t *out_count) +{ + size_t index; + msgpack_object *bound_object; + msgpack_object_array *bounds_array; + double *bounds; + + *out_bounds = NULL; + *out_count = 0; + + if (explicit_bounds_object == NULL) { + return -1; + } + + if (explicit_bounds_object->type != MSGPACK_OBJECT_ARRAY) { + return -1; + } + + bounds_array = &explicit_bounds_object->via.array; + + if (bounds_array->size == 0) { + return -1; + } + + bounds = flb_calloc(bounds_array->size, sizeof(double)); + if (bounds == NULL) { + flb_errno(); + return -1; + } + + for (index = 0 ; index < bounds_array->size ; index++) { + bound_object = &bounds_array->ptr[index]; + + if (parse_double_value(bound_object, &bounds[index]) != 0) { + flb_free(bounds); + return -1; + } + } + + *out_bounds = bounds; + *out_count = bounds_array->size; + + return 0; +} + +static int parse_histogram_bucket_counts(msgpack_object *bucket_counts_object, + uint64_t **out_bucket_counts, + size_t *out_count) +{ + size_t index; + msgpack_object *count_object; + msgpack_object_array *counts_array; + uint64_t *bucket_counts; + + *out_bucket_counts = NULL; + *out_count = 0; + + if (bucket_counts_object == NULL) { + return -1; + } + + if (bucket_counts_object->type != MSGPACK_OBJECT_ARRAY) { + return -1; + } + + counts_array = &bucket_counts_object->via.array; + if (counts_array->size == 0) { + return -1; + } + + bucket_counts = flb_calloc(counts_array->size, sizeof(uint64_t)); + if (bucket_counts == NULL) { + flb_errno(); + return -1; + } + + for (index = 0 ; index < counts_array->size ; index++) { + count_object = &counts_array->ptr[index]; + + if (parse_u64_value(count_object, &bucket_counts[index]) != 0) { + flb_free(bucket_counts); + return -1; + } + } + + *out_bucket_counts = bucket_counts; + *out_count = counts_array->size; + + return 0; +} + +static int parse_u64_array(msgpack_object *array_object, + uint64_t **out_values, + size_t *out_count) +{ + size_t index; + msgpack_object *item_object; + msgpack_object_array *items_array; + uint64_t *values; + + *out_values = NULL; + *out_count = 0; + + if (array_object == NULL) { + return 0; + } + + if (array_object->type != MSGPACK_OBJECT_ARRAY) { + return -1; + } + + items_array = &array_object->via.array; + + if (items_array->size == 0) { + return 0; + } + + values = flb_calloc(items_array->size, sizeof(uint64_t)); + if (values == NULL) { + flb_errno(); + return -1; + } + + for (index = 0 ; index < items_array->size ; index++) { + item_object = &items_array->ptr[index]; + + if (parse_u64_value(item_object, &values[index]) != 0) { + flb_free(values); + return -1; + } + } + + *out_values = values; + *out_count = items_array->size; + + return 0; +} + +static int parse_exponential_histogram_buckets(msgpack_object *bucket_object, + int32_t *out_offset, + uint64_t **out_counts, + size_t *out_count) +{ + int offset_index; + int bucket_counts_index; + int result; + msgpack_object *offset_object; + msgpack_object *bucket_counts_object; + msgpack_object_map *bucket_map; + + *out_offset = 0; + *out_counts = NULL; + *out_count = 0; + + if (bucket_object == NULL) { + return 0; + } + + if (bucket_object->type != MSGPACK_OBJECT_MAP) { + return -1; + } + + bucket_map = &bucket_object->via.map; + + offset_index = flb_otel_utils_find_map_entry_by_key(bucket_map, + "offset", + 0, + FLB_TRUE); + if (offset_index >= 0) { + offset_object = &bucket_map->ptr[offset_index].val; + + if (parse_i32_value(offset_object, out_offset) != 0) { + return -1; + } + } + + bucket_counts_index = flb_otel_utils_find_map_entry_by_key(bucket_map, + "bucketCounts", + 0, + FLB_TRUE); + if (bucket_counts_index < 0) { + return 0; + } + + bucket_counts_object = &bucket_map->ptr[bucket_counts_index].val; + result = parse_u64_array(bucket_counts_object, out_counts, out_count); + + return result; +} + +static int parse_summary_quantile_values(msgpack_object *quantile_values_object, + double **out_quantiles, + double **out_values, + size_t *out_count) +{ + size_t index; + int quantile_index; + int value_index; + double *quantiles; + double *values; + msgpack_object *entry_object; + msgpack_object_map *entry_map; + msgpack_object_array *quantile_values_array; + + *out_quantiles = NULL; + *out_values = NULL; + *out_count = 0; + + if (quantile_values_object == NULL) { + return 0; + } + + if (quantile_values_object->type != MSGPACK_OBJECT_ARRAY) { + return -1; + } + + quantile_values_array = &quantile_values_object->via.array; + if (quantile_values_array->size == 0) { + return 0; + } + + quantiles = flb_calloc(quantile_values_array->size, sizeof(double)); + values = flb_calloc(quantile_values_array->size, sizeof(double)); + if (quantiles == NULL || values == NULL) { + if (quantiles == NULL) { + flb_errno(); + } + if (values == NULL) { + flb_errno(); + } + flb_free(quantiles); + flb_free(values); + return -1; + } + + for (index = 0 ; index < quantile_values_array->size ; index++) { + entry_object = &quantile_values_array->ptr[index]; + if (entry_object->type != MSGPACK_OBJECT_MAP) { + flb_free(quantiles); + flb_free(values); + return -1; + } + + entry_map = &entry_object->via.map; + quantile_index = flb_otel_utils_find_map_entry_by_key(entry_map, + "quantile", + 0, + FLB_TRUE); + value_index = flb_otel_utils_find_map_entry_by_key(entry_map, + "value", + 0, + FLB_TRUE); + if (quantile_index < 0 || value_index < 0) { + flb_free(quantiles); + flb_free(values); + return -1; + } + + if (parse_double_value(&entry_map->ptr[quantile_index].val, + &quantiles[index]) != 0) { + flb_free(quantiles); + flb_free(values); + return -1; + } + + if (parse_double_value(&entry_map->ptr[value_index].val, + &values[index]) != 0) { + flb_free(quantiles); + flb_free(values); + return -1; + } + } + + *out_quantiles = quantiles; + *out_values = values; + *out_count = quantile_values_array->size; + + return 0; +} + +static int check_double_array_layout(size_t expected_count, + double *expected_values, + size_t actual_count, + double *actual_values) +{ + size_t index; + + if (expected_count != actual_count) { + return -1; + } + + for (index = 0 ; index < expected_count ; index++) { + if (expected_values[index] != actual_values[index]) { + return -1; + } + } + + return 0; +} + +static int process_metric_histogram_data_points(struct cmt *context, + msgpack_object_map *metric_map, + msgpack_object *name_object, + int aggregation_type, + msgpack_object_array *data_points) +{ + int index; + int result; + int metric_initialized; + int point_label_count; + int metric_label_count; + int bucket_counts_index; + int explicit_bounds_index; + int count_index; + int sum_index; + int min_index; + int max_index; + int attributes_index; + int has_sum; + int has_min; + int has_max; + uint64_t timestamp; + uint64_t count; + size_t bucket_counts_count; + size_t metric_bound_count; + size_t point_bound_count; + double sum; + double min; + double max; + double *point_bounds; + double *metric_bounds; + flb_sds_t metric_name; + flb_sds_t metric_help; + flb_sds_t *point_label_keys; + flb_sds_t *point_label_values; + flb_sds_t *metric_label_keys; + msgpack_object *point; + msgpack_object *attributes_obj; + msgpack_object_map *point_map; + msgpack_object *bucket_counts_object; + msgpack_object *explicit_bounds_object; + msgpack_object *count_object; + msgpack_object *sum_object; + struct cmt_histogram_buckets *buckets; + struct cmt_histogram *histogram; + struct cmt_metric *sample; + struct cfl_kvlist *point_metadata; + uint64_t *bucket_counts; + + point_bounds = NULL; + metric_bounds = NULL; + bucket_counts = NULL; + metric_name = NULL; + point_label_keys = NULL; + point_label_values = NULL; + metric_label_keys = NULL; + metric_label_count = 0; + metric_bound_count = 0; + histogram = NULL; + buckets = NULL; + metric_help = NULL; + sample = NULL; + point_metadata = NULL; + metric_initialized = FLB_FALSE; + + for (index = 0 ; index < data_points->size ; index++) { + point = &data_points->ptr[index]; + if (point->type != MSGPACK_OBJECT_MAP) { + continue; + } + + point_map = &point->via.map; + + bucket_counts_index = flb_otel_utils_find_map_entry_by_key(point_map, + "bucketCounts", + 0, + FLB_TRUE); + explicit_bounds_index = flb_otel_utils_find_map_entry_by_key(point_map, + "explicitBounds", + 0, + FLB_TRUE); + count_index = flb_otel_utils_find_map_entry_by_key(point_map, + "count", + 0, + FLB_TRUE); + sum_index = flb_otel_utils_find_map_entry_by_key(point_map, + "sum", + 0, + FLB_TRUE); + min_index = flb_otel_utils_find_map_entry_by_key(point_map, + "min", + 0, + FLB_TRUE); + max_index = flb_otel_utils_find_map_entry_by_key(point_map, + "max", + 0, + FLB_TRUE); + + if (bucket_counts_index < 0 || + explicit_bounds_index < 0 || + count_index < 0) { + continue; + } + + bucket_counts_object = &point_map->ptr[bucket_counts_index].val; + explicit_bounds_object = &point_map->ptr[explicit_bounds_index].val; + count_object = &point_map->ptr[count_index].val; + sum_object = NULL; + if (sum_index >= 0) { + sum_object = &point_map->ptr[sum_index].val; + } + has_sum = (sum_object != NULL); + + result = parse_histogram_bucket_counts(bucket_counts_object, + &bucket_counts, + &bucket_counts_count); + if (result != 0) { + continue; + } + + result = parse_histogram_bounds(explicit_bounds_object, + &point_bounds, + &point_bound_count); + if (result != 0) { + flb_free(bucket_counts); + bucket_counts = NULL; + continue; + } + + if (bucket_counts_count != point_bound_count + 1) { + flb_free(bucket_counts); + flb_free(point_bounds); + bucket_counts = NULL; + point_bounds = NULL; + continue; + } + + if (parse_u64_value(count_object, &count) != 0) { + flb_free(bucket_counts); + flb_free(point_bounds); + bucket_counts = NULL; + point_bounds = NULL; + continue; + } + + sum = 0.0; + if (sum_object != NULL && + parse_double_value(sum_object, &sum) != 0) { + flb_free(bucket_counts); + flb_free(point_bounds); + bucket_counts = NULL; + point_bounds = NULL; + continue; + } + + has_min = FLB_FALSE; + min = 0.0; + if (min_index >= 0) { + if (parse_double_value(&point_map->ptr[min_index].val, &min) != 0) { + flb_free(bucket_counts); + flb_free(point_bounds); + bucket_counts = NULL; + point_bounds = NULL; + continue; + } + + has_min = FLB_TRUE; + } + + has_max = FLB_FALSE; + max = 0.0; + if (max_index >= 0) { + if (parse_double_value(&point_map->ptr[max_index].val, &max) != 0) { + flb_free(bucket_counts); + flb_free(point_bounds); + bucket_counts = NULL; + point_bounds = NULL; + continue; + } + + has_max = FLB_TRUE; + } + + result = parse_datapoint_timestamp(point_map, ×tamp); + if (result != 0) { + flb_free(bucket_counts); + flb_free(point_bounds); + bucket_counts = NULL; + point_bounds = NULL; + continue; + } + + attributes_index = flb_otel_utils_find_map_entry_by_key(point_map, + "attributes", + 0, + FLB_TRUE); + if (attributes_index >= 0) { + attributes_obj = &point_map->ptr[attributes_index].val; + } + else { + attributes_obj = NULL; + } + + point_label_count = 0; + point_label_keys = NULL; + point_label_values = NULL; + + result = parse_datapoint_labels(attributes_obj, + &point_label_count, + &point_label_keys, + &point_label_values); + if (result != 0) { + destroy_label_arrays(point_label_count, + point_label_keys, + point_label_values); + flb_free(bucket_counts); + flb_free(point_bounds); + bucket_counts = NULL; + point_bounds = NULL; + continue; + } + + if (!metric_initialized) { + metric_label_count = point_label_count; + metric_label_keys = point_label_keys; + point_label_keys = NULL; + metric_bounds = point_bounds; + metric_bound_count = point_bound_count; + point_bounds = NULL; + + metric_name = flb_sds_create_len(name_object->via.str.ptr, + name_object->via.str.size); + if (metric_name == NULL) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(metric_bounds); + flb_free(bucket_counts); + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + result = get_metric_help_string(metric_map, &metric_help); + if (result != 0) { + flb_sds_destroy(metric_name); + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(metric_bounds); + flb_free(bucket_counts); + return result; + } + + buckets = cmt_histogram_buckets_create_size(metric_bounds, + metric_bound_count); + if (buckets == NULL) { + flb_sds_destroy(metric_name); + flb_sds_destroy(metric_help); + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(metric_bounds); + flb_free(bucket_counts); + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + histogram = cmt_histogram_create(context, + "", + "", + metric_name, + metric_help, + buckets, + metric_label_count, + (char **) metric_label_keys); + flb_sds_destroy(metric_name); + flb_sds_destroy(metric_help); + metric_help = NULL; + + if (histogram == NULL) { + cmt_histogram_buckets_destroy(buckets); + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(metric_bounds); + flb_free(bucket_counts); + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + result = set_metric_unit(histogram->map, metric_map); + if (result != 0) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(metric_bounds); + flb_free(bucket_counts); + cmt_histogram_destroy(histogram); + return result; + } + + result = clone_metric_metadata(context, histogram->map, metric_map); + if (result != 0) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(metric_bounds); + flb_free(bucket_counts); + cmt_histogram_destroy(histogram); + return result; + } + + histogram->aggregation_type = aggregation_type; + metric_initialized = FLB_TRUE; + } + else { + result = check_label_layout(metric_label_count, + metric_label_keys, + point_label_count, + point_label_keys, + point_label_values); + if (result == 0) { + result = check_double_array_layout(metric_bound_count, + metric_bounds, + point_bound_count, + point_bounds); + } + + flb_free(point_bounds); + point_bounds = NULL; + + if (result != 0) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + flb_free(bucket_counts); + bucket_counts = NULL; + continue; + } + } + + cmt_histogram_set_default(histogram, + timestamp, + bucket_counts, + sum, + count, + point_label_count, + (char **) point_label_values); + + sample = cmt_map_metric_get(&histogram->opts, + histogram->map, + point_label_count, + (char **) point_label_values, + CMT_FALSE); + if (sample != NULL) { + point_metadata = get_or_create_data_point_metadata_context(context, + histogram->map, + sample, + timestamp); + if (point_metadata != NULL) { + append_common_datapoint_metadata(point_metadata, point_map); + cfl_kvlist_insert_bool(point_metadata, "has_sum", has_sum); + + if (has_min) { + cfl_kvlist_insert_bool(point_metadata, "has_min", CFL_TRUE); + cfl_kvlist_insert_double(point_metadata, "min", min); + } + + if (has_max) { + cfl_kvlist_insert_bool(point_metadata, "has_max", CFL_TRUE); + cfl_kvlist_insert_double(point_metadata, "max", max); + } + } + } + + destroy_label_arrays(point_label_count, NULL, point_label_values); + flb_free(bucket_counts); + bucket_counts = NULL; + } + + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(metric_bounds); + + return 0; +} + +static int process_metric_summary_data_points(struct cmt *context, + msgpack_object_map *metric_map, + msgpack_object *name_object, + msgpack_object_array *data_points) +{ + int index; + int result; + int metric_initialized; + int point_label_count; + int metric_label_count; + int quantile_values_index; + int count_index; + int sum_index; + int attributes_index; + uint64_t timestamp; + uint64_t count; + size_t point_quantile_count; + size_t metric_quantile_count; + double sum; + double *point_quantiles; + double *point_quantile_values; + double *metric_quantiles; + flb_sds_t metric_name; + flb_sds_t metric_help; + flb_sds_t *point_label_keys; + flb_sds_t *point_label_values; + flb_sds_t *metric_label_keys; + msgpack_object *point; + msgpack_object *attributes_obj; + msgpack_object_map *point_map; + msgpack_object *quantile_values_object; + msgpack_object *count_object; + msgpack_object *sum_object; + struct cmt_summary *summary; + struct cmt_metric *sample; + struct cfl_kvlist *point_metadata; + + point_quantiles = NULL; + point_quantile_values = NULL; + metric_quantiles = NULL; + metric_name = NULL; + point_label_keys = NULL; + point_label_values = NULL; + metric_label_keys = NULL; + metric_label_count = 0; + metric_quantile_count = 0; + summary = NULL; + metric_help = NULL; + sample = NULL; + point_metadata = NULL; + metric_initialized = FLB_FALSE; + + for (index = 0 ; index < data_points->size ; index++) { + point = &data_points->ptr[index]; + if (point->type != MSGPACK_OBJECT_MAP) { + continue; + } + + point_map = &point->via.map; + + quantile_values_index = flb_otel_utils_find_map_entry_by_key(point_map, + "quantileValues", + 0, + FLB_TRUE); + count_index = flb_otel_utils_find_map_entry_by_key(point_map, + "count", + 0, + FLB_TRUE); + sum_index = flb_otel_utils_find_map_entry_by_key(point_map, + "sum", + 0, + FLB_TRUE); + + if (count_index < 0 || sum_index < 0) { + continue; + } + + if (quantile_values_index >= 0) { + quantile_values_object = &point_map->ptr[quantile_values_index].val; + } + else { + quantile_values_object = NULL; + } + + count_object = &point_map->ptr[count_index].val; + sum_object = &point_map->ptr[sum_index].val; + + result = parse_summary_quantile_values(quantile_values_object, + &point_quantiles, + &point_quantile_values, + &point_quantile_count); + if (result != 0) { + continue; + } + + if (parse_u64_value(count_object, &count) != 0) { + flb_free(point_quantiles); + flb_free(point_quantile_values); + point_quantiles = NULL; + point_quantile_values = NULL; + continue; + } + + if (parse_double_value(sum_object, &sum) != 0) { + flb_free(point_quantiles); + flb_free(point_quantile_values); + point_quantiles = NULL; + point_quantile_values = NULL; + continue; + } + + result = parse_datapoint_timestamp(point_map, ×tamp); + if (result != 0) { + flb_free(point_quantiles); + flb_free(point_quantile_values); + point_quantiles = NULL; + point_quantile_values = NULL; + continue; + } + + attributes_index = flb_otel_utils_find_map_entry_by_key(point_map, + "attributes", + 0, + FLB_TRUE); + if (attributes_index >= 0) { + attributes_obj = &point_map->ptr[attributes_index].val; + } + else { + attributes_obj = NULL; + } + + point_label_count = 0; + point_label_keys = NULL; + point_label_values = NULL; + + result = parse_datapoint_labels(attributes_obj, + &point_label_count, + &point_label_keys, + &point_label_values); + if (result != 0) { + destroy_label_arrays(point_label_count, + point_label_keys, + point_label_values); + flb_free(point_quantiles); + flb_free(point_quantile_values); + point_quantiles = NULL; + point_quantile_values = NULL; + continue; + } + + if (!metric_initialized) { + metric_label_count = point_label_count; + metric_label_keys = point_label_keys; + point_label_keys = NULL; + metric_quantiles = point_quantiles; + metric_quantile_count = point_quantile_count; + point_quantiles = NULL; + + metric_name = flb_sds_create_len(name_object->via.str.ptr, + name_object->via.str.size); + if (metric_name == NULL) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(metric_quantiles); + flb_free(point_quantile_values); + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + result = get_metric_help_string(metric_map, &metric_help); + if (result != 0) { + flb_sds_destroy(metric_name); + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(metric_quantiles); + flb_free(point_quantile_values); + return result; + } + + summary = cmt_summary_create(context, + "", + "", + metric_name, + metric_help, + metric_quantile_count, + metric_quantiles, + metric_label_count, + (char **) metric_label_keys); + flb_sds_destroy(metric_name); + flb_sds_destroy(metric_help); + metric_help = NULL; + + if (summary == NULL) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(metric_quantiles); + flb_free(point_quantile_values); + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + result = set_metric_unit(summary->map, metric_map); + if (result != 0) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(metric_quantiles); + flb_free(point_quantile_values); + cmt_summary_destroy(summary); + return result; + } + + result = clone_metric_metadata(context, summary->map, metric_map); + if (result != 0) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(metric_quantiles); + flb_free(point_quantile_values); + cmt_summary_destroy(summary); + return result; + } + + metric_initialized = FLB_TRUE; + } + else { + result = check_label_layout(metric_label_count, + metric_label_keys, + point_label_count, + point_label_keys, + point_label_values); + if (result == 0) { + result = check_double_array_layout(metric_quantile_count, + metric_quantiles, + point_quantile_count, + point_quantiles); + } + + flb_free(point_quantiles); + point_quantiles = NULL; + + if (result != 0) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + flb_free(point_quantile_values); + point_quantile_values = NULL; + continue; + } + } + + cmt_summary_set_default(summary, + timestamp, + point_quantile_values, + sum, + count, + point_label_count, + (char **) point_label_values); + + sample = cmt_map_metric_get(&summary->opts, + summary->map, + point_label_count, + (char **) point_label_values, + CMT_FALSE); + if (sample != NULL) { + point_metadata = get_or_create_data_point_metadata_context(context, + summary->map, + sample, + timestamp); + if (point_metadata != NULL) { + append_common_datapoint_metadata(point_metadata, point_map); + } + } + + destroy_label_arrays(point_label_count, NULL, point_label_values); + flb_free(point_quantile_values); + point_quantile_values = NULL; + } + + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(metric_quantiles); + + return 0; +} + +static int process_metric_exponential_histogram_data_points( + struct cmt *context, + msgpack_object_map *metric_map, + msgpack_object *name_object, + int aggregation_type, + msgpack_object_array *data_points) +{ + int index; + int result; + int metric_initialized; + int point_label_count; + int metric_label_count; + int scale_index; + int zero_count_index; + int zero_threshold_index; + int positive_index; + int negative_index; + int count_index; + int sum_index; + int min_index; + int max_index; + int attributes_index; + int32_t scale; + int32_t positive_offset; + int32_t negative_offset; + uint64_t timestamp; + uint64_t count; + uint64_t zero_count; + int sum_set; + int has_min; + int has_max; + size_t positive_count; + size_t negative_count; + double sum; + double zero_threshold; + double min; + double max; + flb_sds_t metric_name; + flb_sds_t metric_help; + flb_sds_t *point_label_keys; + flb_sds_t *point_label_values; + flb_sds_t *metric_label_keys; + msgpack_object *point; + msgpack_object *attributes_obj; + msgpack_object_map *point_map; + msgpack_object *positive_object; + msgpack_object *negative_object; + msgpack_object *scale_object; + msgpack_object *zero_count_object; + msgpack_object *zero_threshold_object; + msgpack_object *count_object; + msgpack_object *sum_object; + struct cmt_exp_histogram *exp_histogram; + struct cmt_metric *sample; + struct cfl_kvlist *point_metadata; + uint64_t *positive_counts; + uint64_t *negative_counts; + + positive_counts = NULL; + negative_counts = NULL; + metric_name = NULL; + metric_help = NULL; + point_label_keys = NULL; + point_label_values = NULL; + metric_label_keys = NULL; + metric_label_count = 0; + exp_histogram = NULL; + sample = NULL; + point_metadata = NULL; + metric_initialized = FLB_FALSE; + + for (index = 0 ; index < data_points->size ; index++) { + point = &data_points->ptr[index]; + if (point->type != MSGPACK_OBJECT_MAP) { + continue; + } + + point_map = &point->via.map; + + scale_index = flb_otel_utils_find_map_entry_by_key(point_map, + "scale", + 0, + FLB_TRUE); + count_index = flb_otel_utils_find_map_entry_by_key(point_map, + "count", + 0, + FLB_TRUE); + sum_index = flb_otel_utils_find_map_entry_by_key(point_map, + "sum", + 0, + FLB_TRUE); + min_index = flb_otel_utils_find_map_entry_by_key(point_map, + "min", + 0, + FLB_TRUE); + max_index = flb_otel_utils_find_map_entry_by_key(point_map, + "max", + 0, + FLB_TRUE); + zero_count_index = flb_otel_utils_find_map_entry_by_key(point_map, + "zeroCount", + 0, + FLB_TRUE); + zero_threshold_index = flb_otel_utils_find_map_entry_by_key(point_map, + "zeroThreshold", + 0, + FLB_TRUE); + positive_index = flb_otel_utils_find_map_entry_by_key(point_map, + "positive", + 0, + FLB_TRUE); + negative_index = flb_otel_utils_find_map_entry_by_key(point_map, + "negative", + 0, + FLB_TRUE); + + if (scale_index < 0 || count_index < 0) { + continue; + } + + scale_object = &point_map->ptr[scale_index].val; + count_object = &point_map->ptr[count_index].val; + sum_object = NULL; + if (sum_index >= 0) { + sum_object = &point_map->ptr[sum_index].val; + } + + if (zero_count_index >= 0) { + zero_count_object = &point_map->ptr[zero_count_index].val; + } + else { + zero_count_object = NULL; + } + + if (zero_threshold_index >= 0) { + zero_threshold_object = &point_map->ptr[zero_threshold_index].val; + } + else { + zero_threshold_object = NULL; + } + + if (positive_index >= 0) { + positive_object = &point_map->ptr[positive_index].val; + } + else { + positive_object = NULL; + } + + if (negative_index >= 0) { + negative_object = &point_map->ptr[negative_index].val; + } + else { + negative_object = NULL; + } + + if (parse_i32_value(scale_object, &scale) != 0) { + continue; + } + + if (parse_u64_value(count_object, &count) != 0) { + continue; + } + + sum_set = FLB_FALSE; + sum = 0.0; + if (sum_object != NULL) { + if (parse_double_value(sum_object, &sum) != 0) { + continue; + } + sum_set = FLB_TRUE; + } + + has_min = FLB_FALSE; + min = 0.0; + if (min_index >= 0) { + if (parse_double_value(&point_map->ptr[min_index].val, &min) != 0) { + continue; + } + has_min = FLB_TRUE; + } + + has_max = FLB_FALSE; + max = 0.0; + if (max_index >= 0) { + if (parse_double_value(&point_map->ptr[max_index].val, &max) != 0) { + continue; + } + has_max = FLB_TRUE; + } + + zero_count = 0; + if (zero_count_object != NULL && + parse_u64_value(zero_count_object, &zero_count) != 0) { + continue; + } + + zero_threshold = 0.0; + if (zero_threshold_object != NULL && + parse_double_value(zero_threshold_object, &zero_threshold) != 0) { + continue; + } + + result = parse_exponential_histogram_buckets(positive_object, + &positive_offset, + &positive_counts, + &positive_count); + if (result != 0) { + continue; + } + + result = parse_exponential_histogram_buckets(negative_object, + &negative_offset, + &negative_counts, + &negative_count); + if (result != 0) { + flb_free(positive_counts); + positive_counts = NULL; + continue; + } + + result = parse_datapoint_timestamp(point_map, ×tamp); + if (result != 0) { + flb_free(positive_counts); + flb_free(negative_counts); + positive_counts = NULL; + negative_counts = NULL; + continue; + } + + attributes_index = flb_otel_utils_find_map_entry_by_key(point_map, + "attributes", + 0, + FLB_TRUE); + if (attributes_index >= 0) { + attributes_obj = &point_map->ptr[attributes_index].val; + } + else { + attributes_obj = NULL; + } + + point_label_count = 0; + point_label_keys = NULL; + point_label_values = NULL; + + result = parse_datapoint_labels(attributes_obj, + &point_label_count, + &point_label_keys, + &point_label_values); + if (result != 0) { + destroy_label_arrays(point_label_count, + point_label_keys, + point_label_values); + flb_free(positive_counts); + flb_free(negative_counts); + positive_counts = NULL; + negative_counts = NULL; + continue; + } + + if (!metric_initialized) { + metric_label_count = point_label_count; + metric_label_keys = point_label_keys; + point_label_keys = NULL; + + metric_name = flb_sds_create_len(name_object->via.str.ptr, + name_object->via.str.size); + if (metric_name == NULL) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(positive_counts); + flb_free(negative_counts); + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + result = get_metric_help_string(metric_map, &metric_help); + if (result != 0) { + flb_sds_destroy(metric_name); + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(positive_counts); + flb_free(negative_counts); + return result; + } + + exp_histogram = cmt_exp_histogram_create(context, + "", + "", + metric_name, + metric_help, + metric_label_count, + (char **) metric_label_keys); + flb_sds_destroy(metric_name); + flb_sds_destroy(metric_help); + metric_help = NULL; + + if (exp_histogram == NULL) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(positive_counts); + flb_free(negative_counts); + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + result = set_metric_unit(exp_histogram->map, metric_map); + if (result != 0) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(positive_counts); + flb_free(negative_counts); + cmt_exp_histogram_destroy(exp_histogram); + return result; + } + + result = clone_metric_metadata(context, exp_histogram->map, metric_map); + if (result != 0) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + flb_free(positive_counts); + flb_free(negative_counts); + cmt_exp_histogram_destroy(exp_histogram); + return result; + } + + exp_histogram->aggregation_type = aggregation_type; + metric_initialized = FLB_TRUE; + } + else { + result = check_label_layout(metric_label_count, + metric_label_keys, + point_label_count, + point_label_keys, + point_label_values); + + if (result != 0) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + flb_free(positive_counts); + flb_free(negative_counts); + positive_counts = NULL; + negative_counts = NULL; + continue; + } + } + + result = cmt_exp_histogram_set_default(exp_histogram, + timestamp, + scale, + zero_count, + zero_threshold, + positive_offset, + positive_count, + positive_counts, + negative_offset, + negative_count, + negative_counts, + sum_set, + sum, + count, + point_label_count, + (char **) point_label_values); + if (result != 0) { + destroy_label_arrays(point_label_count, NULL, point_label_values); + flb_free(positive_counts); + flb_free(negative_counts); + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + sample = cmt_map_metric_get(&exp_histogram->opts, + exp_histogram->map, + point_label_count, + (char **) point_label_values, + CMT_FALSE); + if (sample != NULL) { + point_metadata = get_or_create_data_point_metadata_context(context, + exp_histogram->map, + sample, + timestamp); + if (point_metadata != NULL) { + append_common_datapoint_metadata(point_metadata, point_map); + cfl_kvlist_insert_bool(point_metadata, "has_sum", sum_set); + + if (has_min) { + cfl_kvlist_insert_bool(point_metadata, "has_min", CFL_TRUE); + cfl_kvlist_insert_double(point_metadata, "min", min); + } + + if (has_max) { + cfl_kvlist_insert_bool(point_metadata, "has_max", CFL_TRUE); + cfl_kvlist_insert_double(point_metadata, "max", max); + } + } + } + + destroy_label_arrays(point_label_count, NULL, point_label_values); + flb_free(positive_counts); + flb_free(negative_counts); + positive_counts = NULL; + negative_counts = NULL; + } + + destroy_label_arrays(metric_label_count, metric_label_keys, NULL); + + return 0; +} + +static int decode_metric_gauge(struct cmt *context, + msgpack_object_map *metric_map, + msgpack_object *name_object) +{ + int result; + int data_points_index; + msgpack_object *gauge_object; + msgpack_object *data_points_object; + msgpack_object_map *gauge_map; + + result = flb_otel_utils_find_map_entry_by_key(metric_map, "gauge", 0, FLB_TRUE); + if (result < 0) { + return 0; + } + + gauge_object = &metric_map->ptr[result].val; + if (gauge_object->type != MSGPACK_OBJECT_MAP) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + gauge_map = &gauge_object->via.map; + data_points_index = flb_otel_utils_find_map_entry_by_key(gauge_map, + "dataPoints", + 0, + FLB_TRUE); + if (data_points_index < 0) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + data_points_object = &gauge_map->ptr[data_points_index].val; + if (data_points_object->type != MSGPACK_OBJECT_ARRAY) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + return process_metric_gauge_data_points(context, + metric_map, + name_object, + &data_points_object->via.array); +} + +static int decode_metric_sum(struct cmt *context, + msgpack_object_map *metric_map, + msgpack_object *name_object) +{ + int result; + int data_points_index; + int monotonic_index; + int temporality_index; + msgpack_object *sum_object; + msgpack_object *data_points_object; + msgpack_object_map *sum_map; + int allow_reset; + int aggregation_type; + + result = flb_otel_utils_find_map_entry_by_key(metric_map, "sum", 0, FLB_TRUE); + if (result < 0) { + return 0; + } + + sum_object = &metric_map->ptr[result].val; + if (sum_object->type != MSGPACK_OBJECT_MAP) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + sum_map = &sum_object->via.map; + data_points_index = flb_otel_utils_find_map_entry_by_key(sum_map, + "dataPoints", + 0, + FLB_TRUE); + if (data_points_index < 0) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + data_points_object = &sum_map->ptr[data_points_index].val; + if (data_points_object->type != MSGPACK_OBJECT_ARRAY) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + allow_reset = FLB_FALSE; + aggregation_type = CMT_AGGREGATION_TYPE_UNSPECIFIED; + + monotonic_index = flb_otel_utils_find_map_entry_by_key(sum_map, + "isMonotonic", + 0, + FLB_TRUE); + if (monotonic_index >= 0 && + sum_map->ptr[monotonic_index].val.type == MSGPACK_OBJECT_BOOLEAN && + !sum_map->ptr[monotonic_index].val.via.boolean) { + allow_reset = FLB_TRUE; + } + + temporality_index = flb_otel_utils_find_map_entry_by_key(sum_map, + "aggregationTemporality", + 0, + FLB_TRUE); + if (temporality_index >= 0) { + if (sum_map->ptr[temporality_index].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + if (sum_map->ptr[temporality_index].val.via.u64 == 1) { + aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + } + else if (sum_map->ptr[temporality_index].val.via.u64 == 2) { + aggregation_type = CMT_AGGREGATION_TYPE_CUMULATIVE; + } + } + } + + return process_metric_sum_data_points(context, + metric_map, + name_object, + allow_reset, + aggregation_type, + &data_points_object->via.array); +} + +static int decode_metric_histogram(struct cmt *context, + msgpack_object_map *metric_map, + msgpack_object *name_object) +{ + int result; + int data_points_index; + int temporality_index; + int aggregation_type; + msgpack_object *histogram_object; + msgpack_object *data_points_object; + msgpack_object_map *histogram_map; + + result = flb_otel_utils_find_map_entry_by_key(metric_map, + "histogram", + 0, + FLB_TRUE); + if (result < 0) { + return 0; + } + + histogram_object = &metric_map->ptr[result].val; + if (histogram_object->type != MSGPACK_OBJECT_MAP) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + histogram_map = &histogram_object->via.map; + data_points_index = flb_otel_utils_find_map_entry_by_key(histogram_map, + "dataPoints", + 0, + FLB_TRUE); + if (data_points_index < 0) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + data_points_object = &histogram_map->ptr[data_points_index].val; + if (data_points_object->type != MSGPACK_OBJECT_ARRAY) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + aggregation_type = CMT_AGGREGATION_TYPE_UNSPECIFIED; + temporality_index = flb_otel_utils_find_map_entry_by_key(histogram_map, + "aggregationTemporality", + 0, + FLB_TRUE); + if (temporality_index >= 0) { + if (histogram_map->ptr[temporality_index].val.type == + MSGPACK_OBJECT_POSITIVE_INTEGER) { + if (histogram_map->ptr[temporality_index].val.via.u64 == 1) { + aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + } + else if (histogram_map->ptr[temporality_index].val.via.u64 == 2) { + aggregation_type = CMT_AGGREGATION_TYPE_CUMULATIVE; + } + } + } + + return process_metric_histogram_data_points(context, + metric_map, + name_object, + aggregation_type, + &data_points_object->via.array); +} + +static int decode_metric_summary(struct cmt *context, + msgpack_object_map *metric_map, + msgpack_object *name_object) +{ + int result; + int data_points_index; + msgpack_object *summary_object; + msgpack_object *data_points_object; + msgpack_object_map *summary_map; + + result = flb_otel_utils_find_map_entry_by_key(metric_map, + "summary", + 0, + FLB_TRUE); + if (result < 0) { + return 0; + } + + summary_object = &metric_map->ptr[result].val; + if (summary_object->type != MSGPACK_OBJECT_MAP) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + summary_map = &summary_object->via.map; + data_points_index = flb_otel_utils_find_map_entry_by_key(summary_map, + "dataPoints", + 0, + FLB_TRUE); + if (data_points_index < 0) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + data_points_object = &summary_map->ptr[data_points_index].val; + if (data_points_object->type != MSGPACK_OBJECT_ARRAY) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + return process_metric_summary_data_points(context, + metric_map, + name_object, + &data_points_object->via.array); +} + +static int decode_metric_exponential_histogram(struct cmt *context, + msgpack_object_map *metric_map, + msgpack_object *name_object) +{ + int result; + int data_points_index; + int temporality_index; + int aggregation_type; + msgpack_object *histogram_object; + msgpack_object *data_points_object; + msgpack_object_map *histogram_map; + + result = flb_otel_utils_find_map_entry_by_key(metric_map, + "exponentialHistogram", + 0, + FLB_TRUE); + if (result < 0) { + return 0; + } + + histogram_object = &metric_map->ptr[result].val; + if (histogram_object->type != MSGPACK_OBJECT_MAP) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + histogram_map = &histogram_object->via.map; + data_points_index = flb_otel_utils_find_map_entry_by_key(histogram_map, + "dataPoints", + 0, + FLB_TRUE); + if (data_points_index < 0) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + data_points_object = &histogram_map->ptr[data_points_index].val; + if (data_points_object->type != MSGPACK_OBJECT_ARRAY) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + aggregation_type = CMT_AGGREGATION_TYPE_UNSPECIFIED; + temporality_index = flb_otel_utils_find_map_entry_by_key(histogram_map, + "aggregationTemporality", + 0, + FLB_TRUE); + if (temporality_index >= 0) { + if (histogram_map->ptr[temporality_index].val.type == + MSGPACK_OBJECT_POSITIVE_INTEGER) { + if (histogram_map->ptr[temporality_index].val.via.u64 == 1) { + aggregation_type = CMT_AGGREGATION_TYPE_DELTA; + } + else if (histogram_map->ptr[temporality_index].val.via.u64 == 2) { + aggregation_type = CMT_AGGREGATION_TYPE_CUMULATIVE; + } + } + } + + return process_metric_exponential_histogram_data_points( + context, + metric_map, + name_object, + aggregation_type, + &data_points_object->via.array); +} + +static int decode_metric_entry(struct cmt *context, msgpack_object *metric_object) +{ + int result; + int name_index; + int gauge_index; + int sum_index; + int histogram_index; + int summary_index; + int exponential_histogram_index; + msgpack_object *name_object; + msgpack_object_map *metric_map; + + if (metric_object->type != MSGPACK_OBJECT_MAP) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + metric_map = &metric_object->via.map; + + name_index = flb_otel_utils_find_map_entry_by_key(metric_map, "name", 0, FLB_TRUE); + if (name_index < 0) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + name_object = &metric_map->ptr[name_index].val; + if (name_object->type != MSGPACK_OBJECT_STR) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + gauge_index = flb_otel_utils_find_map_entry_by_key(metric_map, "gauge", 0, FLB_TRUE); + sum_index = flb_otel_utils_find_map_entry_by_key(metric_map, "sum", 0, FLB_TRUE); + histogram_index = flb_otel_utils_find_map_entry_by_key(metric_map, + "histogram", + 0, + FLB_TRUE); + summary_index = flb_otel_utils_find_map_entry_by_key(metric_map, + "summary", + 0, + FLB_TRUE); + exponential_histogram_index = flb_otel_utils_find_map_entry_by_key( + metric_map, + "exponentialHistogram", + 0, + FLB_TRUE); + + if (gauge_index < 0 && sum_index < 0 && + histogram_index < 0 && summary_index < 0 && + exponential_histogram_index < 0) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + if (gauge_index >= 0) { + result = decode_metric_gauge(context, metric_map, name_object); + if (result != 0) { + return result; + } + } + + if (sum_index >= 0) { + result = decode_metric_sum(context, metric_map, name_object); + if (result != 0) { + return result; + } + } + + if (histogram_index >= 0) { + result = decode_metric_histogram(context, metric_map, name_object); + if (result != 0) { + return result; + } + } + + if (summary_index >= 0) { + result = decode_metric_summary(context, metric_map, name_object); + if (result != 0) { + return result; + } + } + + if (exponential_histogram_index >= 0) { + result = decode_metric_exponential_histogram(context, + metric_map, + name_object); + if (result != 0) { + return result; + } + } + + return 0; +} + +static int decode_scope_metrics_entry(struct cfl_list *context_list, + msgpack_object *scope_metrics_object) +{ + int index; + int scope_index; + int result; + int metrics_index; + int schema_index; + msgpack_object *metrics_object; + msgpack_object *scope_object; + msgpack_object_array *metrics_array; + msgpack_object_map *scope_metrics_map; + struct cmt *context; + + if (scope_metrics_object->type != MSGPACK_OBJECT_MAP) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + scope_metrics_map = &scope_metrics_object->via.map; + + scope_index = flb_otel_utils_find_map_entry_by_key(scope_metrics_map, + "scope", + 0, + FLB_TRUE); + if (scope_index >= 0) { + scope_object = &scope_metrics_map->ptr[scope_index].val; + } + else { + scope_object = NULL; + } + + metrics_index = flb_otel_utils_find_map_entry_by_key(scope_metrics_map, + "metrics", + 0, + FLB_TRUE); + if (metrics_index < 0) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + metrics_object = &scope_metrics_map->ptr[metrics_index].val; + if (metrics_object->type != MSGPACK_OBJECT_ARRAY) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + context = cmt_create(); + if (context == NULL) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + result = cfl_kvlist_insert_string(context->internal_metadata, + "producer", + "opentelemetry"); + if (result != 0) { + cmt_destroy(context); + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + result = clone_scope_metadata_and_attributes(context->external_metadata, + scope_object); + if (result != 0) { + cmt_destroy(context); + return result; + } + + schema_index = flb_otel_utils_find_map_entry_by_key(scope_metrics_map, + "schemaUrl", + 0, + FLB_TRUE); + if (schema_index >= 0) { + result = clone_scope_metrics_metadata(context->external_metadata, + scope_metrics_map); + if (result != 0) { + cmt_destroy(context); + return result; + } + } + + metrics_array = &metrics_object->via.array; + + for (index = 0 ; index < metrics_array->size ; index++) { + result = decode_metric_entry(context, &metrics_array->ptr[index]); + if (result != 0) { + cmt_destroy(context); + return result; + } + } + + cfl_list_add(&context->_head, context_list); + + return 0; +} + +static int decode_resource_metrics_entry(struct cfl_list *context_list, + msgpack_object *resource_metrics_object) +{ + int index; + int resource_index; + int scope_metrics_index; + int result; + int schema_index; + msgpack_object *resource_object; + msgpack_object *scope_metrics_object; + msgpack_object_array *scope_metrics_array; + msgpack_object_map *resource_metrics_map; + struct cmt *context; + + if (resource_metrics_object->type != MSGPACK_OBJECT_MAP) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + resource_metrics_map = &resource_metrics_object->via.map; + + resource_index = flb_otel_utils_find_map_entry_by_key(resource_metrics_map, + "resource", + 0, + FLB_TRUE); + if (resource_index >= 0) { + resource_object = &resource_metrics_map->ptr[resource_index].val; + } + else { + resource_object = NULL; + } + + scope_metrics_index = flb_otel_utils_find_map_entry_by_key(resource_metrics_map, + "scopeMetrics", + 0, + FLB_TRUE); + if (scope_metrics_index < 0) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + scope_metrics_object = &resource_metrics_map->ptr[scope_metrics_index].val; + if (scope_metrics_object->type != MSGPACK_OBJECT_ARRAY) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + scope_metrics_array = &scope_metrics_object->via.array; + for (index = 0 ; index < scope_metrics_array->size ; index++) { + result = decode_scope_metrics_entry(context_list, &scope_metrics_array->ptr[index]); + if (result != 0) { + return result; + } + + context = cfl_list_entry_last(context_list, struct cmt, _head); + if (context == NULL) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + result = clone_resource_metadata_and_attributes(context->external_metadata, + resource_object); + if (result != 0) { + return result; + } + + schema_index = flb_otel_utils_find_map_entry_by_key(resource_metrics_map, + "schemaUrl", + 0, + FLB_TRUE); + if (schema_index >= 0) { + result = clone_resource_metrics_metadata(context->external_metadata, + resource_metrics_map); + if (result != 0) { + return result; + } + } + } + + return 0; +} + +int flb_opentelemetry_metrics_json_to_cmt(struct cfl_list *context_list, + const char *body, size_t len) +{ + int result; + int index; + int root_type; + int resource_metrics_index; + char *msgpack_body; + size_t msgpack_body_size; + size_t off; + msgpack_unpacked result_set; + msgpack_object *root_object; + msgpack_object *resource_metrics_object; + msgpack_object_array *resource_metrics; + + msgpack_body = NULL; + msgpack_body_size = 0; + off = 0; + + cfl_list_init(context_list); + + result = flb_pack_json(body, len, &msgpack_body, &msgpack_body_size, + &root_type, NULL); + if (result != 0) { + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + msgpack_unpacked_init(&result_set); + + if (!msgpack_unpack_next(&result_set, msgpack_body, msgpack_body_size, &off)) { + flb_free(msgpack_body); + msgpack_unpacked_destroy(&result_set); + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + root_object = &result_set.data; + if (root_object->type != MSGPACK_OBJECT_MAP) { + flb_free(msgpack_body); + msgpack_unpacked_destroy(&result_set); + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + resource_metrics_index = flb_otel_utils_find_map_entry_by_key(&root_object->via.map, + "resourceMetrics", + 0, + FLB_TRUE); + if (resource_metrics_index < 0) { + flb_free(msgpack_body); + msgpack_unpacked_destroy(&result_set); + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + resource_metrics_object = &root_object->via.map.ptr[resource_metrics_index].val; + if (resource_metrics_object->type != MSGPACK_OBJECT_ARRAY) { + flb_free(msgpack_body); + msgpack_unpacked_destroy(&result_set); + return OTEL_METRICS_JSON_DECODER_ERROR; + } + + resource_metrics = &resource_metrics_object->via.array; + result = CMT_DECODE_OPENTELEMETRY_SUCCESS; + + for (index = 0 ; + result == CMT_DECODE_OPENTELEMETRY_SUCCESS && index < resource_metrics->size ; + index++) { + result = decode_resource_metrics_entry(context_list, + &resource_metrics->ptr[index]); + } + + if (result != CMT_DECODE_OPENTELEMETRY_SUCCESS) { + destroy_context_list(context_list); + } + + flb_free(msgpack_body); + msgpack_unpacked_destroy(&result_set); + + return result; +} diff --git a/src/opentelemetry/flb_opentelemetry_utils.c b/src/opentelemetry/flb_opentelemetry_utils.c index 952534ceddb..edaaab32a4d 100644 --- a/src/opentelemetry/flb_opentelemetry_utils.c +++ b/src/opentelemetry/flb_opentelemetry_utils.c @@ -25,6 +25,8 @@ #include #include +#include + int flb_otel_utils_find_map_entry_by_key(msgpack_object_map *map, char *key, size_t match_index, @@ -620,3 +622,449 @@ uint64_t flb_otel_utils_convert_string_number_to_u64(char *str, size_t len) val = strtoull(tmp, NULL, 10); return val; } + +static int parse_u64_from_numeric_string(const char *str, + size_t len, + uint64_t *value) +{ + size_t index; + + if (str == NULL || value == NULL || len == 0) { + return -1; + } + + if (str[0] == '-' || len > 31) { + return -1; + } + + for (index = 0; index < len; index++) { + if (!isdigit((unsigned char) str[index])) { + return -1; + } + } + + *value = flb_otel_utils_convert_string_number_to_u64((char *) str, len); + + return 0; +} + +static int parse_u64_from_msgpack_object(msgpack_object *object, uint64_t *value) +{ + if (object == NULL || value == NULL) { + return -1; + } + + if (object->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + *value = object->via.u64; + return 0; + } + else if (object->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + if (object->via.i64 < 0) { + return -1; + } + + *value = (uint64_t) object->via.i64; + return 0; + } + else if (object->type == MSGPACK_OBJECT_STR) { + return parse_u64_from_numeric_string(object->via.str.ptr, + object->via.str.size, + value); + } + + return -1; +} + +static int parse_double_from_msgpack_object(msgpack_object *object, double *value) +{ + char *end; + flb_sds_t string_value; + + if (object == NULL || value == NULL) { + return -1; + } + + if (object->type == MSGPACK_OBJECT_FLOAT32 || + object->type == MSGPACK_OBJECT_FLOAT64) { + *value = object->via.f64; + return 0; + } + else if (object->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + *value = (double) object->via.u64; + return 0; + } + else if (object->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + *value = (double) object->via.i64; + return 0; + } + else if (object->type != MSGPACK_OBJECT_STR) { + return -1; + } + + string_value = flb_sds_create_len(object->via.str.ptr, object->via.str.size); + if (string_value == NULL) { + return -1; + } + + end = NULL; + *value = strtod(string_value, &end); + + if (end == string_value || (end != NULL && *end != '\0')) { + flb_sds_destroy(string_value); + return -1; + } + + flb_sds_destroy(string_value); + + return 0; +} + +static int clone_kvlist_entry_from_otlp_json(struct cfl_kvlist *target, + msgpack_object *entry_object) +{ + int key_index; + int value_index; + struct cfl_variant *value_instance; + msgpack_object *key_object; + msgpack_object *value_object; + msgpack_object_map *entry_map; + + if (target == NULL || entry_object == NULL || + entry_object->type != MSGPACK_OBJECT_MAP) { + return -1; + } + + entry_map = &entry_object->via.map; + + key_index = flb_otel_utils_find_map_entry_by_key(entry_map, "key", 0, FLB_TRUE); + value_index = flb_otel_utils_find_map_entry_by_key(entry_map, "value", 0, FLB_TRUE); + + if (key_index < 0 || value_index < 0) { + return -1; + } + + key_object = &entry_map->ptr[key_index].val; + value_object = &entry_map->ptr[value_index].val; + + if (key_object->type != MSGPACK_OBJECT_STR) { + return -1; + } + + value_instance = flb_otel_utils_msgpack_object_to_cfl_variant(value_object); + if (value_instance == NULL) { + return -1; + } + + if (cfl_kvlist_insert_s(target, + (char *) key_object->via.str.ptr, + key_object->via.str.size, + value_instance) != 0) { + cfl_variant_destroy(value_instance); + return -1; + } + + return 0; +} + +static struct cfl_variant *convert_otlp_anyvalue_wrapper_to_cfl_variant( + msgpack_object *wrapper_object) +{ + int value_type; + int64_t signed_value; + uint64_t unsigned_value; + double double_value; + size_t index; + msgpack_object *value_object; + msgpack_object_map *value_map; + msgpack_object_array *value_array; + struct cfl_array *array_instance; + struct cfl_kvlist *kvlist_instance; + struct cfl_variant *child_variant; + struct cfl_variant *result_variant; + + value_object = NULL; + value_type = MSGPACK_OBJECT_NIL; + + if (flb_otel_utils_json_payload_get_wrapped_value(wrapper_object, + &value_object, + &value_type) != 0) { + return NULL; + } + + if (value_type == MSGPACK_OBJECT_STR) { + if (value_object->type == MSGPACK_OBJECT_NIL) { + return cfl_variant_create_from_null(); + } + + if (value_object->type != MSGPACK_OBJECT_STR) { + return NULL; + } + + return cfl_variant_create_from_string_s((char *) value_object->via.str.ptr, + value_object->via.str.size, + CFL_FALSE); + } + else if (value_type == MSGPACK_OBJECT_BOOLEAN) { + if (value_object->type != MSGPACK_OBJECT_BOOLEAN) { + return NULL; + } + + return cfl_variant_create_from_bool(value_object->via.boolean); + } + else if (value_type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + if (parse_u64_from_msgpack_object(value_object, &unsigned_value) != 0) { + return NULL; + } + + if (value_object->type == MSGPACK_OBJECT_NEGATIVE_INTEGER || + (value_object->type == MSGPACK_OBJECT_STR && + value_object->via.str.size > 0 && + value_object->via.str.ptr[0] == '-')) { + signed_value = (int64_t) unsigned_value; + return cfl_variant_create_from_int64(signed_value); + } + + return cfl_variant_create_from_uint64(unsigned_value); + } + else if (value_type == MSGPACK_OBJECT_FLOAT) { + if (parse_double_from_msgpack_object(value_object, &double_value) != 0) { + return NULL; + } + + return cfl_variant_create_from_double(double_value); + } + else if (value_type == MSGPACK_OBJECT_BIN) { + if (value_object->type != MSGPACK_OBJECT_BIN) { + return NULL; + } + + return cfl_variant_create_from_bytes((char *) value_object->via.bin.ptr, + value_object->via.bin.size, + CFL_FALSE); + } + else if (value_type == MSGPACK_OBJECT_ARRAY) { + if (value_object->type != MSGPACK_OBJECT_ARRAY) { + return NULL; + } + + value_array = &value_object->via.array; + array_instance = cfl_array_create(value_array->size); + if (array_instance == NULL) { + return NULL; + } + + for (index = 0 ; index < value_array->size ; index++) { + child_variant = flb_otel_utils_msgpack_object_to_cfl_variant( + &value_array->ptr[index]); + if (child_variant == NULL) { + cfl_array_destroy(array_instance); + return NULL; + } + + if (cfl_array_append(array_instance, child_variant) != 0) { + cfl_variant_destroy(child_variant); + cfl_array_destroy(array_instance); + return NULL; + } + } + + return cfl_variant_create_from_array(array_instance); + } + else if (value_type == MSGPACK_OBJECT_MAP) { + if (value_object->type != MSGPACK_OBJECT_ARRAY && + value_object->type != MSGPACK_OBJECT_MAP) { + return NULL; + } + + kvlist_instance = cfl_kvlist_create(); + if (kvlist_instance == NULL) { + return NULL; + } + + if (value_object->type == MSGPACK_OBJECT_ARRAY) { + value_array = &value_object->via.array; + + for (index = 0 ; index < value_array->size ; index++) { + if (clone_kvlist_entry_from_otlp_json(kvlist_instance, + &value_array->ptr[index]) != 0) { + cfl_kvlist_destroy(kvlist_instance); + return NULL; + } + } + } + else { + value_map = &value_object->via.map; + + for (index = 0 ; index < value_map->size ; index++) { + if (value_map->ptr[index].key.type != MSGPACK_OBJECT_STR) { + cfl_kvlist_destroy(kvlist_instance); + return NULL; + } + + child_variant = flb_otel_utils_msgpack_object_to_cfl_variant( + &value_map->ptr[index].val); + if (child_variant == NULL) { + cfl_kvlist_destroy(kvlist_instance); + return NULL; + } + + if (cfl_kvlist_insert_s(kvlist_instance, + (char *) value_map->ptr[index].key.via.str.ptr, + value_map->ptr[index].key.via.str.size, + child_variant) != 0) { + cfl_variant_destroy(child_variant); + cfl_kvlist_destroy(kvlist_instance); + return NULL; + } + } + } + + result_variant = cfl_variant_create_from_kvlist(kvlist_instance); + if (result_variant == NULL) { + cfl_kvlist_destroy(kvlist_instance); + } + + return result_variant; + } + + return NULL; +} + +struct cfl_variant *flb_otel_utils_msgpack_object_to_cfl_variant( + msgpack_object *object) +{ + size_t index; + msgpack_object_map *map_object; + msgpack_object_array *array_object; + struct cfl_array *array_instance; + struct cfl_kvlist *kvlist_instance; + struct cfl_variant *child_variant; + struct cfl_variant *result_variant; + + if (object == NULL) { + return NULL; + } + + if (object->type == MSGPACK_OBJECT_MAP) { + if (object->via.map.size == 1) { + result_variant = convert_otlp_anyvalue_wrapper_to_cfl_variant(object); + if (result_variant != NULL) { + return result_variant; + } + } + + kvlist_instance = cfl_kvlist_create(); + if (kvlist_instance == NULL) { + return NULL; + } + + map_object = &object->via.map; + for (index = 0 ; index < map_object->size ; index++) { + if (map_object->ptr[index].key.type != MSGPACK_OBJECT_STR) { + cfl_kvlist_destroy(kvlist_instance); + return NULL; + } + + child_variant = flb_otel_utils_msgpack_object_to_cfl_variant( + &map_object->ptr[index].val); + if (child_variant == NULL) { + cfl_kvlist_destroy(kvlist_instance); + return NULL; + } + + if (cfl_kvlist_insert_s(kvlist_instance, + (char *) map_object->ptr[index].key.via.str.ptr, + map_object->ptr[index].key.via.str.size, + child_variant) != 0) { + cfl_variant_destroy(child_variant); + cfl_kvlist_destroy(kvlist_instance); + return NULL; + } + } + + result_variant = cfl_variant_create_from_kvlist(kvlist_instance); + if (result_variant == NULL) { + cfl_kvlist_destroy(kvlist_instance); + } + + return result_variant; + } + else if (object->type == MSGPACK_OBJECT_ARRAY) { + array_object = &object->via.array; + array_instance = cfl_array_create(array_object->size); + if (array_instance == NULL) { + return NULL; + } + + for (index = 0 ; index < array_object->size ; index++) { + child_variant = flb_otel_utils_msgpack_object_to_cfl_variant( + &array_object->ptr[index]); + if (child_variant == NULL) { + cfl_array_destroy(array_instance); + return NULL; + } + + if (cfl_array_append(array_instance, child_variant) != 0) { + cfl_variant_destroy(child_variant); + cfl_array_destroy(array_instance); + return NULL; + } + } + + return cfl_variant_create_from_array(array_instance); + } + else if (object->type == MSGPACK_OBJECT_STR) { + return cfl_variant_create_from_string_s((char *) object->via.str.ptr, + object->via.str.size, + CFL_FALSE); + } + else if (object->type == MSGPACK_OBJECT_BIN) { + return cfl_variant_create_from_bytes((char *) object->via.bin.ptr, + object->via.bin.size, + CFL_FALSE); + } + else if (object->type == MSGPACK_OBJECT_BOOLEAN) { + return cfl_variant_create_from_bool(object->via.boolean); + } + else if (object->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + return cfl_variant_create_from_uint64(object->via.u64); + } + else if (object->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + return cfl_variant_create_from_int64(object->via.i64); + } + else if (object->type == MSGPACK_OBJECT_FLOAT32 || + object->type == MSGPACK_OBJECT_FLOAT64) { + return cfl_variant_create_from_double(object->via.f64); + } + else if (object->type == MSGPACK_OBJECT_NIL) { + return cfl_variant_create_from_null(); + } + + return NULL; +} + +int flb_otel_utils_clone_kvlist_from_otlp_json_array(struct cfl_kvlist *target, + msgpack_object *attributes_object) +{ + size_t index; + msgpack_object_array *attributes_array; + + if (target == NULL || attributes_object == NULL) { + return -1; + } + + if (attributes_object->type != MSGPACK_OBJECT_ARRAY) { + return -1; + } + + attributes_array = &attributes_object->via.array; + + for (index = 0 ; index < attributes_array->size ; index++) { + if (clone_kvlist_entry_from_otlp_json(target, + &attributes_array->ptr[index]) != 0) { + return -1; + } + } + + return 0; +} diff --git a/tests/internal/CMakeLists.txt b/tests/internal/CMakeLists.txt index 6984f69d80f..e58099ccf5d 100644 --- a/tests/internal/CMakeLists.txt +++ b/tests/internal/CMakeLists.txt @@ -179,6 +179,7 @@ set(UNIT_TESTS_DATA data/input_chunk/log/a_thousand_plus_one_bytes.log data/input_chunk/log/test_buffer_valid.log data/opentelemetry/logs.json + data/opentelemetry/metrics.json ) set(FLB_TESTS_DATA_PATH ${CMAKE_CURRENT_SOURCE_DIR}/) diff --git a/tests/internal/data/opentelemetry/metrics.json b/tests/internal/data/opentelemetry/metrics.json new file mode 100644 index 00000000000..9548a19649b --- /dev/null +++ b/tests/internal/data/opentelemetry/metrics.json @@ -0,0 +1,1248 @@ +{ + "valid_payload": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "temperature_celsius", + "gauge": { + "dataPoints": [ + { + "asDouble": 36.5, + "timeUnixNano": "1704067200000000000" + } + ] + } + }, + { + "name": "requests_total", + "sum": { + "isMonotonic": false, + "aggregationTemporality": 2, + "dataPoints": [ + { + "asInt": 42, + "timeUnixNano": "1704067201000000000" + } + ] + } + } + ] + } + ] + } + ] + }, + "expected": { + "context_count": 1, + "gauge": { + "name": "temperature_celsius", + "value": 36.5 + }, + "counter": { + "name": "requests_total", + "value": 42.0, + "allow_reset": true, + "aggregation_type": 2 + } + } + }, + "valid_sum_monotonic_true": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "jobs_total", + "sum": { + "isMonotonic": true, + "aggregationTemporality": 2, + "dataPoints": [ + { + "asInt": 12 + } + ] + } + } + ] + } + ] + } + ] + }, + "expected": { + "context_count": 1, + "gauge_count": 0, + "counter_count": 1, + "counter": { + "name": "jobs_total", + "value": 12.0, + "allow_reset": false, + "aggregation_type": 2 + } + } + }, + "valid_sum_delta_temporality": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "requests_delta_total", + "sum": { + "isMonotonic": false, + "aggregationTemporality": 1, + "dataPoints": [ + { + "asDouble": 9.5 + } + ] + } + } + ] + } + ] + } + ] + }, + "expected": { + "context_count": 1, + "gauge_count": 0, + "counter_count": 1, + "counter": { + "name": "requests_delta_total", + "value": 9.5, + "allow_reset": true, + "aggregation_type": 1 + } + } + }, + "valid_numeric_string_values": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "cpu_temp", + "gauge": { + "dataPoints": [ + { + "asDouble": "38.25", + "timeUnixNano": "1704067200000000000" + } + ] + } + }, + { + "name": "errors_total", + "sum": { + "isMonotonic": true, + "aggregationTemporality": 2, + "dataPoints": [ + { + "asInt": "7" + } + ] + } + } + ] + } + ] + } + ] + }, + "expected": { + "context_count": 1, + "gauge": { + "name": "cpu_temp", + "value": 38.25 + }, + "counter": { + "name": "errors_total", + "value": 7.0, + "allow_reset": false, + "aggregation_type": 2 + } + } + }, + "valid_missing_timestamp_defaults_zero": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "load", + "gauge": { + "dataPoints": [ + { + "asDouble": 1.5 + } + ] + } + } + ] + } + ] + } + ] + }, + "expected": { + "context_count": 1, + "gauge_count": 1, + "counter_count": 0, + "gauge": { + "name": "load", + "value": 1.5 + } + } + }, + "valid_multiple_resource_metrics_contexts": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "gauge_one", + "gauge": { + "dataPoints": [ + { + "asInt": 1 + } + ] + } + } + ] + } + ] + }, + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "counter_two", + "sum": { + "isMonotonic": false, + "aggregationTemporality": 2, + "dataPoints": [ + { + "asInt": 2 + } + ] + } + } + ] + } + ] + } + ] + }, + "expected": { + "context_count": 2, + "gauge_count": 1, + "counter_count": 0, + "gauge": { + "name": "gauge_one", + "value": 1.0 + } + } + }, + "valid_label_order_independent": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "order_independent_gauge", + "gauge": { + "dataPoints": [ + { + "asInt": 10, + "attributes": [ + { + "key": "k1", + "value": { + "stringValue": "v1" + } + }, + { + "key": "k2", + "value": { + "stringValue": "v2" + } + } + ] + }, + { + "asInt": 20, + "attributes": [ + { + "key": "k2", + "value": { + "stringValue": "v2" + } + }, + { + "key": "k1", + "value": { + "stringValue": "v1" + } + } + ] + } + ] + } + } + ] + } + ] + } + ] + }, + "expected": { + "context_count": 1, + "gauge_count": 1, + "counter_count": 0, + "histogram_count": 0, + "exp_histogram_count": 0, + "summary_count": 0, + "gauge": { + "name": "order_independent_gauge", + "label_values": [ + "v1", + "v2" + ], + "value": 20.0 + } + } + }, + "valid_invalid_numeric_skips_points": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "bad_numeric", + "gauge": { + "dataPoints": [ + { + "asDouble": "12.0oops" + } + ] + } + } + ] + } + ] + } + ] + }, + "expected": { + "context_count": 1, + "gauge_count": 0, + "counter_count": 0 + } + }, + "valid_histogram_payload": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "request_duration_seconds", + "histogram": { + "aggregationTemporality": 2, + "dataPoints": [ + { + "count": 6, + "sum": 18.0, + "bucketCounts": [1, 3, 6], + "explicitBounds": [1.0, 5.0] + } + ] + } + } + ] + } + ] + } + ] + }, + "expected": { + "context_count": 1, + "gauge_count": 0, + "counter_count": 0, + "histogram_count": 1, + "summary_count": 0, + "histogram": { + "name": "request_duration_seconds", + "count": 6, + "sum": 18.0 + } + } + }, + "valid_summary_payload": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "request_size_bytes", + "summary": { + "dataPoints": [ + { + "count": 4, + "sum": 420.0, + "quantileValues": [ + {"quantile": 0.5, "value": 100.0}, + {"quantile": 0.9, "value": 180.0} + ] + } + ] + } + } + ] + } + ] + } + ] + }, + "expected": { + "context_count": 1, + "gauge_count": 0, + "counter_count": 0, + "histogram_count": 0, + "summary_count": 1, + "summary": { + "name": "request_size_bytes", + "count": 4, + "sum": 420.0 + } + } + }, + "invalid_root": { + "input": [], + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_resource_metrics_missing": { + "input": {}, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_resource_metrics_type": { + "input": { + "resourceMetrics": {} + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_scope_metrics_missing": { + "input": { + "resourceMetrics": [ + {} + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_scope_metrics_type": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": {} + } + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_metrics_missing": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + {} + ] + } + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_metrics_type": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": {} + } + ] + } + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_metric_name_missing": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "gauge": { + "dataPoints": [] + } + } + ] + } + ] + } + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_metric_name_type": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": 123, + "gauge": { + "dataPoints": [] + } + } + ] + } + ] + } + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "missing_metric_type": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "metric_without_data" + } + ] + } + ] + } + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_gauge_type": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "broken_gauge", + "gauge": [] + } + ] + } + ] + } + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_gauge_datapoints_missing": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "broken_gauge_points", + "gauge": {} + } + ] + } + ] + } + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_gauge_datapoints_type": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "broken_gauge_points_type", + "gauge": { + "dataPoints": {} + } + } + ] + } + ] + } + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_sum_type": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "broken_sum", + "sum": [] + } + ] + } + ] + } + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_sum_datapoints_missing": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "broken_sum_points", + "sum": {} + } + ] + } + ] + } + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_sum_datapoints_type": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "broken_sum_points_type", + "sum": { + "dataPoints": {} + } + } + ] + } + ] + } + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_histogram_type": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "broken_histogram", + "histogram": [] + } + ] + } + ] + } + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_histogram_datapoints_missing": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "broken_histogram_points", + "histogram": {} + } + ] + } + ] + } + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_summary_type": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "broken_summary", + "summary": [] + } + ] + } + ] + } + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_summary_datapoints_missing": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "broken_summary_points", + "summary": {} + } + ] + } + ] + } + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "valid_exponential_histogram_payload": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "exp_hist", + "exponentialHistogram": { + "aggregationTemporality": 2, + "dataPoints": [ + { + "timeUnixNano": "1735689600000000000", + "count": "8", + "sum": "27.5", + "scale": 0, + "zeroCount": "2", + "positive": { + "offset": 0, + "bucketCounts": [ + "3", + "1" + ] + }, + "negative": { + "offset": -1, + "bucketCounts": [ + "1", + "1" + ] + } + } + ] + } + } + ] + } + ] + } + ] + }, + "expected": { + "context_count": 1, + "gauge_count": 0, + "counter_count": 0, + "histogram_count": 0, + "exp_histogram_count": 1, + "summary_count": 0, + "exp_histogram": { + "name": "exp_hist", + "count": 8, + "sum": 27.5, + "aggregation_type": 2 + } + } + }, + "valid_exponential_histogram_zero_only": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "exp_hist_zero", + "exponentialHistogram": { + "dataPoints": [ + { + "timeUnixNano": 1735689600000000001, + "count": 5, + "sum": 0, + "scale": 1, + "zeroCount": 5 + } + ] + } + } + ] + } + ] + } + ] + }, + "expected": { + "context_count": 1, + "gauge_count": 0, + "counter_count": 0, + "histogram_count": 0, + "exp_histogram_count": 1, + "summary_count": 0, + "exp_histogram": { + "name": "exp_hist_zero", + "count": 5, + "sum": 0 + } + } + }, + "invalid_exponential_histogram_type": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "exp_hist_broken", + "exponentialHistogram": [] + } + ] + } + ] + } + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_exponential_histogram_datapoints_missing": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "exp_hist_missing_datapoints", + "exponentialHistogram": {} + } + ] + } + ] + } + ] + }, + "expected_error": { + "code": "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR" + } + }, + "invalid_exponential_histogram_count_mismatch": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "exp_hist_count_mismatch", + "exponentialHistogram": { + "dataPoints": [ + { + "count": 99, + "sum": 12.0, + "scale": 0, + "zeroCount": 1, + "positive": { + "offset": 0, + "bucketCounts": [ + 1, + 1 + ] + } + } + ] + } + } + ] + } + ] + } + ] + }, + "expected": { + "context_count": 1, + "gauge_count": 0, + "counter_count": 0, + "histogram_count": 0, + "exp_histogram_count": 1, + "summary_count": 0, + "exp_histogram": { + "name": "exp_hist_count_mismatch", + "count": 99, + "sum": 12.0 + } + } + }, + "invalid_exponential_histogram_inconsistent_layout": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "exp_hist_inconsistent_layout", + "exponentialHistogram": { + "dataPoints": [ + { + "timeUnixNano": "1735689600000000002", + "count": 3, + "sum": 6.0, + "scale": 0, + "zeroCount": 1, + "positive": { + "offset": 0, + "bucketCounts": [ + 2 + ] + } + }, + { + "timeUnixNano": "1735689600000000003", + "count": 3, + "sum": 10.0, + "scale": 2, + "zeroCount": 1, + "positive": { + "offset": 0, + "bucketCounts": [ + 2 + ] + } + } + ] + } + } + ] + } + ] + } + ] + }, + "expected": { + "context_count": 1, + "gauge_count": 0, + "counter_count": 0, + "histogram_count": 0, + "exp_histogram_count": 1, + "summary_count": 0, + "exp_histogram": { + "name": "exp_hist_inconsistent_layout", + "count": 3, + "sum": 10.0 + } + } + }, + "valid_gauge_description_and_unit": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "temperature_kelvin", + "description": "temperature sensor gauge", + "unit": "K", + "gauge": { + "dataPoints": [ + { + "asDouble": 298.15 + } + ] + } + } + ] + } + ] + } + ] + }, + "expected": { + "context_count": 1, + "gauge_count": 1, + "counter_count": 0, + "histogram_count": 0, + "exp_histogram_count": 0, + "summary_count": 0, + "gauge": { + "name": "temperature_kelvin", + "description": "temperature sensor gauge", + "unit": "K", + "value": 298.15 + } + } + }, + "valid_histogram_string_numeric_fields": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "latency_ms", + "description": "request latency", + "unit": "ms", + "histogram": { + "aggregationTemporality": 2, + "dataPoints": [ + { + "count": "6", + "sum": "18.5", + "bucketCounts": [ + "1", + "2", + "3" + ], + "explicitBounds": [ + "1.0", + "3.0" + ] + } + ] + } + } + ] + } + ] + } + ] + }, + "expected": { + "context_count": 1, + "gauge_count": 0, + "counter_count": 0, + "histogram_count": 1, + "exp_histogram_count": 0, + "summary_count": 0, + "histogram": { + "name": "latency_ms", + "description": "request latency", + "unit": "ms", + "count": 6, + "sum": 18.5, + "aggregation_type": 2 + } + } + }, + "valid_summary_string_numeric_fields": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "payload_size", + "description": "request payload bytes", + "unit": "By", + "summary": { + "dataPoints": [ + { + "count": "4", + "sum": "1000.5", + "quantileValues": [ + { + "quantile": "0.5", + "value": "100.0" + }, + { + "quantile": "0.9", + "value": "450.0" + } + ] + } + ] + } + } + ] + } + ] + } + ] + }, + "expected": { + "context_count": 1, + "gauge_count": 0, + "counter_count": 0, + "histogram_count": 0, + "exp_histogram_count": 0, + "summary_count": 1, + "summary": { + "name": "payload_size", + "description": "request payload bytes", + "unit": "By", + "count": 4, + "sum": 1000.5 + } + } + }, + "valid_exponential_histogram_description_unit": { + "input": { + "resourceMetrics": [ + { + "scopeMetrics": [ + { + "metrics": [ + { + "name": "exp_duration", + "description": "exp duration histogram", + "unit": "s", + "exponentialHistogram": { + "aggregationTemporality": 2, + "dataPoints": [ + { + "count": "10", + "sum": "50.25", + "scale": "2", + "zeroCount": "1", + "zeroThreshold": 0.01, + "positive": { + "offset": "0", + "bucketCounts": [ + "4", + "3" + ] + }, + "negative": { + "offset": "-1", + "bucketCounts": [ + "1", + "1" + ] + } + } + ] + } + } + ] + } + ] + } + ] + }, + "expected": { + "context_count": 1, + "gauge_count": 0, + "counter_count": 0, + "histogram_count": 0, + "exp_histogram_count": 1, + "summary_count": 0, + "exp_histogram": { + "name": "exp_duration", + "description": "exp duration histogram", + "unit": "s", + "count": 10, + "sum": 50.25, + "aggregation_type": 2 + } + } + } +} diff --git a/tests/internal/opentelemetry.c b/tests/internal/opentelemetry.c index b9b5b9fdbe2..4a6722fa41c 100644 --- a/tests/internal/opentelemetry.c +++ b/tests/internal/opentelemetry.c @@ -28,6 +28,15 @@ // #include "../../plugins/in_opentelemetry/opentelemetry.h" #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -539,6 +548,825 @@ void test_json_payload_get_wrapped_value() #define OTEL_TEST_CASES_PATH FLB_TESTS_DATA_PATH "/data/opentelemetry/logs.json" #define OTEL_TRACES_TEST_CASES_PATH FLB_TESTS_DATA_PATH "/data/opentelemetry/traces.json" +#define OTEL_METRICS_TEST_CASES_PATH FLB_TESTS_DATA_PATH "/data/opentelemetry/metrics.json" + +static void destroy_metrics_context_list(struct cfl_list *context_list) +{ + struct cfl_list *iterator; + struct cfl_list *tmp; + struct cmt *context; + + if (context_list == NULL) { + return; + } + + cfl_list_foreach_safe(iterator, tmp, context_list) { + context = cfl_list_entry(iterator, struct cmt, _head); + cfl_list_del(&context->_head); + cmt_destroy(context); + } +} + +static int test_msgpack_object_to_double(msgpack_object *object, double *value) +{ + if (object->type == MSGPACK_OBJECT_FLOAT32 || + object->type == MSGPACK_OBJECT_FLOAT64) { + *value = object->via.f64; + return 0; + } + + if (object->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + *value = (double) object->via.u64; + return 0; + } + + if (object->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + *value = (double) object->via.i64; + return 0; + } + + return -1; +} + +static int test_metrics_expected_error_code(const char *error_code_name) +{ + if (strcmp(error_code_name, + "CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR") == 0) { + return CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR; + } + + if (strcmp(error_code_name, + "CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR") == 0) { + return CMT_DECODE_OPENTELEMETRY_ALLOCATION_ERROR; + } + + if (strcmp(error_code_name, + "CMT_DECODE_OPENTELEMETRY_KVLIST_ACCESS_ERROR") == 0) { + return CMT_DECODE_OPENTELEMETRY_KVLIST_ACCESS_ERROR; + } + + if (strcmp(error_code_name, + "CMT_DECODE_OPENTELEMETRY_ARRAY_ACCESS_ERROR") == 0) { + return CMT_DECODE_OPENTELEMETRY_ARRAY_ACCESS_ERROR; + } + + return -1; +} + +static int test_msgpack_object_to_int(msgpack_object *object, int *value) +{ + if (object->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + *value = (int) object->via.u64; + return 0; + } + + if (object->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + *value = (int) object->via.i64; + return 0; + } + + return -1; +} + +static int test_msgpack_object_to_u64(msgpack_object *object, uint64_t *value) +{ + if (object->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + *value = object->via.u64; + return 0; + } + + if (object->type == MSGPACK_OBJECT_NEGATIVE_INTEGER) { + if (object->via.i64 < 0) { + return -1; + } + *value = object->via.i64; + return 0; + } + + return -1; +} + +static void test_destroy_label_values(int label_count, char **label_values) +{ + int index; + + if (label_values == NULL) { + return; + } + + for (index = 0; index < label_count; index++) { + if (label_values[index] != NULL) { + flb_free(label_values[index]); + } + } + + flb_free(label_values); +} + +static int test_extract_label_values(msgpack_object_map *container_map, + char *field_name, + int *out_label_count, + char ***out_label_values) +{ + int index; + int label_index; + char **label_values; + msgpack_object *field_obj; + msgpack_object_array *label_array; + + *out_label_count = 0; + *out_label_values = NULL; + + index = flb_otel_utils_find_map_entry_by_key(container_map, + field_name, + 0, + FLB_TRUE); + if (index < 0) { + return 0; + } + + field_obj = &container_map->ptr[index].val; + if (field_obj->type != MSGPACK_OBJECT_ARRAY) { + return -1; + } + + label_array = &field_obj->via.array; + if (label_array->size == 0) { + return 0; + } + + label_values = flb_calloc(label_array->size, sizeof(char *)); + if (label_values == NULL) { + flb_errno(); + return -1; + } + + for (label_index = 0; label_index < label_array->size; label_index++) { + field_obj = &label_array->ptr[label_index]; + if (field_obj->type != MSGPACK_OBJECT_STR) { + test_destroy_label_values((int) label_array->size, label_values); + return -1; + } + + label_values[label_index] = flb_malloc(field_obj->via.str.size + 1); + if (label_values[label_index] == NULL) { + flb_errno(); + test_destroy_label_values((int) label_array->size, label_values); + return -1; + } + + memcpy(label_values[label_index], + field_obj->via.str.ptr, + field_obj->via.str.size); + label_values[label_index][field_obj->via.str.size] = '\0'; + } + + *out_label_count = (int) label_array->size; + *out_label_values = label_values; + + return 0; +} + +static void test_check_metric_description_unit(msgpack_object *metric_obj, + struct cmt_opts *opts, + struct cmt_map *map) +{ + int index; + msgpack_object *field_obj; + + if (metric_obj == NULL || opts == NULL || map == NULL || + metric_obj->type != MSGPACK_OBJECT_MAP) { + return; + } + + index = flb_otel_utils_find_map_entry_by_key(&metric_obj->via.map, + "description", + 0, + FLB_TRUE); + if (index >= 0) { + field_obj = &metric_obj->via.map.ptr[index].val; + TEST_CHECK(field_obj->type == MSGPACK_OBJECT_STR); + if (field_obj->type == MSGPACK_OBJECT_STR) { + TEST_CHECK(opts->description != NULL); + TEST_CHECK(strlen(opts->description) == field_obj->via.str.size); + TEST_CHECK(strncmp(opts->description, + field_obj->via.str.ptr, + field_obj->via.str.size) == 0); + } + } + + index = flb_otel_utils_find_map_entry_by_key(&metric_obj->via.map, + "unit", + 0, + FLB_TRUE); + if (index >= 0) { + field_obj = &metric_obj->via.map.ptr[index].val; + TEST_CHECK(field_obj->type == MSGPACK_OBJECT_STR); + if (field_obj->type == MSGPACK_OBJECT_STR) { + TEST_CHECK(map->unit != NULL); + TEST_CHECK(strlen(map->unit) == field_obj->via.str.size); + TEST_CHECK(strncmp(map->unit, + field_obj->via.str.ptr, + field_obj->via.str.size) == 0); + } + } +} + +static void run_metrics_case(msgpack_object *case_obj, const char *case_name) +{ + int ret; + int index; + int expected_result; + int expected_aggregation_type; + int context_count; + int expected_allow_reset; + double value; + double expected_value; + msgpack_object *input_obj; + msgpack_object *expected_obj; + msgpack_object *gauge_obj; + msgpack_object *counter_obj; + msgpack_object *histogram_obj; + msgpack_object *summary_obj; + msgpack_object *error_obj; + msgpack_object *field_obj; + char *input_json; + char *error_code_name; + struct cfl_list context_list; + struct cmt *context; + struct cmt_gauge *gauge; + struct cmt_counter *counter; + struct cmt_histogram *histogram; + struct cmt_exp_histogram *exp_histogram; + struct cmt_summary *summary; + struct cmt_metric *metric; + int expected_gauge_count; + int expected_counter_count; + int expected_histogram_count; + int expected_exp_histogram_count; + int expected_summary_count; + int gauge_index; + int counter_index; + int histogram_index; + int exp_histogram_index; + int summary_index; + msgpack_object *exp_histogram_obj; + uint64_t expected_count; + int decode_ret; + int expected_label_count; + char **expected_label_values; + + input_json = NULL; + decode_ret = CMT_DECODE_OPENTELEMETRY_INVALID_ARGUMENT_ERROR; + expected_label_count = 0; + expected_label_values = NULL; + (void) case_name; + TEST_CHECK(case_obj->type == MSGPACK_OBJECT_MAP); + if (case_obj->type != MSGPACK_OBJECT_MAP) { + return; + } + + index = flb_otel_utils_find_map_entry_by_key(&case_obj->via.map, + "input", + 0, + FLB_TRUE); + TEST_CHECK(index >= 0); + if (index < 0) { + return; + } + + input_obj = &case_obj->via.map.ptr[index].val; + input_json = flb_msgpack_to_json_str(4096, input_obj, FLB_TRUE); + TEST_CHECK(input_json != NULL); + if (input_json == NULL) { + return; + } + + expected_result = CMT_DECODE_OPENTELEMETRY_SUCCESS; + + index = flb_otel_utils_find_map_entry_by_key(&case_obj->via.map, + "expected_error", + 0, + FLB_TRUE); + if (index >= 0) { + error_obj = &case_obj->via.map.ptr[index].val; + TEST_CHECK(error_obj->type == MSGPACK_OBJECT_MAP); + if (error_obj->type == MSGPACK_OBJECT_MAP) { + index = flb_otel_utils_find_map_entry_by_key(&error_obj->via.map, + "code", + 0, + FLB_TRUE); + TEST_CHECK(index >= 0); + if (index >= 0) { + field_obj = &error_obj->via.map.ptr[index].val; + TEST_CHECK(field_obj->type == MSGPACK_OBJECT_STR); + if (field_obj->type == MSGPACK_OBJECT_STR) { + error_code_name = flb_malloc(field_obj->via.str.size + 1); + TEST_CHECK(error_code_name != NULL); + if (error_code_name != NULL) { + memcpy(error_code_name, + field_obj->via.str.ptr, + field_obj->via.str.size); + error_code_name[field_obj->via.str.size] = '\0'; + expected_result = + test_metrics_expected_error_code(error_code_name); + flb_free(error_code_name); + } + } + } + } + } + + ret = flb_opentelemetry_metrics_json_to_cmt(&context_list, + input_json, + strlen(input_json)); + decode_ret = ret; + TEST_CHECK(ret == expected_result); + + if (expected_result == CMT_DECODE_OPENTELEMETRY_SUCCESS) { + index = flb_otel_utils_find_map_entry_by_key(&case_obj->via.map, + "expected", + 0, + FLB_TRUE); + TEST_CHECK(index >= 0); + if (index >= 0) { + expected_obj = &case_obj->via.map.ptr[index].val; + TEST_CHECK(expected_obj->type == MSGPACK_OBJECT_MAP); + if (expected_obj->type == MSGPACK_OBJECT_MAP) { + index = flb_otel_utils_find_map_entry_by_key(&expected_obj->via.map, + "context_count", + 0, + FLB_TRUE); + TEST_CHECK(index >= 0); + if (index >= 0 && + expected_obj->via.map.ptr[index].val.type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + context_count = (int) expected_obj->via.map.ptr[index].val.via.u64; + TEST_CHECK(cfl_list_size(&context_list) == context_count); + } + + expected_gauge_count = -1; + index = flb_otel_utils_find_map_entry_by_key(&expected_obj->via.map, + "gauge_count", + 0, + FLB_TRUE); + if (index >= 0) { + field_obj = &expected_obj->via.map.ptr[index].val; + TEST_CHECK(test_msgpack_object_to_int(field_obj, + &expected_gauge_count) == 0); + } + + expected_counter_count = -1; + index = flb_otel_utils_find_map_entry_by_key(&expected_obj->via.map, + "counter_count", + 0, + FLB_TRUE); + if (index >= 0) { + field_obj = &expected_obj->via.map.ptr[index].val; + TEST_CHECK(test_msgpack_object_to_int(field_obj, + &expected_counter_count) == 0); + } + + expected_histogram_count = -1; + index = flb_otel_utils_find_map_entry_by_key(&expected_obj->via.map, + "histogram_count", + 0, + FLB_TRUE); + if (index >= 0) { + field_obj = &expected_obj->via.map.ptr[index].val; + TEST_CHECK(test_msgpack_object_to_int(field_obj, + &expected_histogram_count) == 0); + } + + expected_summary_count = -1; + index = flb_otel_utils_find_map_entry_by_key(&expected_obj->via.map, + "summary_count", + 0, + FLB_TRUE); + if (index >= 0) { + field_obj = &expected_obj->via.map.ptr[index].val; + TEST_CHECK(test_msgpack_object_to_int(field_obj, + &expected_summary_count) == 0); + } + + expected_exp_histogram_count = -1; + index = flb_otel_utils_find_map_entry_by_key(&expected_obj->via.map, + "exp_histogram_count", + 0, + FLB_TRUE); + if (index >= 0) { + field_obj = &expected_obj->via.map.ptr[index].val; + TEST_CHECK(test_msgpack_object_to_int(field_obj, + &expected_exp_histogram_count) == 0); + } + + context = cfl_list_entry(context_list.next, struct cmt, _head); + + if (expected_gauge_count >= 0) { + TEST_CHECK(cfl_list_size(&context->gauges) == expected_gauge_count); + } + + if (expected_counter_count >= 0) { + TEST_CHECK(cfl_list_size(&context->counters) == expected_counter_count); + } + + if (expected_histogram_count >= 0) { + TEST_CHECK(cfl_list_size(&context->histograms) == + expected_histogram_count); + } + + if (expected_summary_count >= 0) { + TEST_CHECK(cfl_list_size(&context->summaries) == + expected_summary_count); + } + + if (expected_exp_histogram_count >= 0) { + TEST_CHECK(cfl_list_size(&context->exp_histograms) == + expected_exp_histogram_count); + } + + gauge_index = flb_otel_utils_find_map_entry_by_key(&expected_obj->via.map, + "gauge", + 0, + FLB_TRUE); + if (gauge_index >= 0) { + gauge_obj = &expected_obj->via.map.ptr[gauge_index].val; + gauge = cfl_list_entry(context->gauges.next, + struct cmt_gauge, _head); + test_check_metric_description_unit(gauge_obj, + &gauge->opts, + gauge->map); + + index = flb_otel_utils_find_map_entry_by_key(&gauge_obj->via.map, + "name", + 0, + FLB_TRUE); + TEST_CHECK(index >= 0); + if (index >= 0) { + field_obj = &gauge_obj->via.map.ptr[index].val; + TEST_CHECK(field_obj->type == MSGPACK_OBJECT_STR); + if (field_obj->type == MSGPACK_OBJECT_STR) { + TEST_CHECK(strlen(gauge->opts.name) == + field_obj->via.str.size); + TEST_CHECK(strncmp(gauge->opts.name, + field_obj->via.str.ptr, + field_obj->via.str.size) == 0); + } + } + + index = flb_otel_utils_find_map_entry_by_key(&gauge_obj->via.map, + "value", + 0, + FLB_TRUE); + TEST_CHECK(index >= 0); + if (index >= 0) { + field_obj = &gauge_obj->via.map.ptr[index].val; + TEST_CHECK(test_msgpack_object_to_double(field_obj, + &expected_value) == 0); + if (test_msgpack_object_to_double(field_obj, + &expected_value) == 0) { + ret = test_extract_label_values(&gauge_obj->via.map, + "label_values", + &expected_label_count, + &expected_label_values); + TEST_CHECK(ret == 0); + ret = cmt_gauge_get_val(gauge, + expected_label_count, + expected_label_values, + &value); + TEST_CHECK(ret == 0); + TEST_CHECK(value == expected_value); + test_destroy_label_values(expected_label_count, + expected_label_values); + expected_label_count = 0; + expected_label_values = NULL; + } + } + } + + counter_index = flb_otel_utils_find_map_entry_by_key(&expected_obj->via.map, + "counter", + 0, + FLB_TRUE); + if (counter_index >= 0) { + counter_obj = &expected_obj->via.map.ptr[counter_index].val; + counter = cfl_list_entry(context->counters.next, + struct cmt_counter, _head); + test_check_metric_description_unit(counter_obj, + &counter->opts, + counter->map); + + index = flb_otel_utils_find_map_entry_by_key(&counter_obj->via.map, + "name", + 0, + FLB_TRUE); + TEST_CHECK(index >= 0); + if (index >= 0) { + field_obj = &counter_obj->via.map.ptr[index].val; + TEST_CHECK(field_obj->type == MSGPACK_OBJECT_STR); + if (field_obj->type == MSGPACK_OBJECT_STR) { + TEST_CHECK(strlen(counter->opts.name) == + field_obj->via.str.size); + TEST_CHECK(strncmp(counter->opts.name, + field_obj->via.str.ptr, + field_obj->via.str.size) == 0); + } + } + + index = flb_otel_utils_find_map_entry_by_key(&counter_obj->via.map, + "value", + 0, + FLB_TRUE); + TEST_CHECK(index >= 0); + if (index >= 0) { + field_obj = &counter_obj->via.map.ptr[index].val; + TEST_CHECK(test_msgpack_object_to_double(field_obj, + &expected_value) == 0); + if (test_msgpack_object_to_double(field_obj, + &expected_value) == 0) { + ret = test_extract_label_values(&counter_obj->via.map, + "label_values", + &expected_label_count, + &expected_label_values); + TEST_CHECK(ret == 0); + ret = cmt_counter_get_val(counter, + expected_label_count, + expected_label_values, + &value); + TEST_CHECK(ret == 0); + TEST_CHECK(value == expected_value); + test_destroy_label_values(expected_label_count, + expected_label_values); + expected_label_count = 0; + expected_label_values = NULL; + } + } + + index = flb_otel_utils_find_map_entry_by_key(&counter_obj->via.map, + "allow_reset", + 0, + FLB_TRUE); + TEST_CHECK(index >= 0); + if (index >= 0) { + field_obj = &counter_obj->via.map.ptr[index].val; + TEST_CHECK(field_obj->type == MSGPACK_OBJECT_BOOLEAN); + if (field_obj->type == MSGPACK_OBJECT_BOOLEAN) { + expected_allow_reset = field_obj->via.boolean; + TEST_CHECK(counter->allow_reset == expected_allow_reset); + } + } + + index = flb_otel_utils_find_map_entry_by_key(&counter_obj->via.map, + "aggregation_type", + 0, + FLB_TRUE); + TEST_CHECK(index >= 0); + if (index >= 0) { + field_obj = &counter_obj->via.map.ptr[index].val; + TEST_CHECK(field_obj->type == MSGPACK_OBJECT_POSITIVE_INTEGER); + if (field_obj->type == MSGPACK_OBJECT_POSITIVE_INTEGER) { + expected_aggregation_type = field_obj->via.u64; + TEST_CHECK(counter->aggregation_type == + expected_aggregation_type); + } + } + } + + histogram_index = flb_otel_utils_find_map_entry_by_key( + &expected_obj->via.map, + "histogram", + 0, + FLB_TRUE); + if (histogram_index >= 0) { + histogram_obj = &expected_obj->via.map.ptr[histogram_index].val; + histogram = cfl_list_entry(context->histograms.next, + struct cmt_histogram, _head); + metric = &histogram->map->metric; + test_check_metric_description_unit(histogram_obj, + &histogram->opts, + histogram->map); + + index = flb_otel_utils_find_map_entry_by_key(&histogram_obj->via.map, + "name", + 0, + FLB_TRUE); + TEST_CHECK(index >= 0); + if (index >= 0) { + field_obj = &histogram_obj->via.map.ptr[index].val; + TEST_CHECK(field_obj->type == MSGPACK_OBJECT_STR); + if (field_obj->type == MSGPACK_OBJECT_STR) { + TEST_CHECK(strlen(histogram->opts.name) == + field_obj->via.str.size); + TEST_CHECK(strncmp(histogram->opts.name, + field_obj->via.str.ptr, + field_obj->via.str.size) == 0); + } + } + + index = flb_otel_utils_find_map_entry_by_key(&histogram_obj->via.map, + "count", + 0, + FLB_TRUE); + TEST_CHECK(index >= 0); + if (index >= 0) { + field_obj = &histogram_obj->via.map.ptr[index].val; + TEST_CHECK(test_msgpack_object_to_u64(field_obj, + &expected_count) == 0); + if (test_msgpack_object_to_u64(field_obj, + &expected_count) == 0) { + TEST_CHECK(cmt_metric_hist_get_count_value(metric) == + expected_count); + } + } + + index = flb_otel_utils_find_map_entry_by_key(&histogram_obj->via.map, + "sum", + 0, + FLB_TRUE); + TEST_CHECK(index >= 0); + if (index >= 0) { + field_obj = &histogram_obj->via.map.ptr[index].val; + TEST_CHECK(test_msgpack_object_to_double(field_obj, + &expected_value) == 0); + if (test_msgpack_object_to_double(field_obj, + &expected_value) == 0) { + value = cmt_metric_hist_get_sum_value(metric); + TEST_CHECK(value == expected_value); + } + } + + index = flb_otel_utils_find_map_entry_by_key(&histogram_obj->via.map, + "aggregation_type", + 0, + FLB_TRUE); + if (index >= 0) { + field_obj = &histogram_obj->via.map.ptr[index].val; + TEST_CHECK(test_msgpack_object_to_int(field_obj, + &expected_aggregation_type) == 0); + if (test_msgpack_object_to_int(field_obj, + &expected_aggregation_type) == 0) { + TEST_CHECK(histogram->aggregation_type == + expected_aggregation_type); + } + } + } + + summary_index = flb_otel_utils_find_map_entry_by_key( + &expected_obj->via.map, + "summary", + 0, + FLB_TRUE); + if (summary_index >= 0) { + summary_obj = &expected_obj->via.map.ptr[summary_index].val; + summary = cfl_list_entry(context->summaries.next, + struct cmt_summary, _head); + metric = &summary->map->metric; + test_check_metric_description_unit(summary_obj, + &summary->opts, + summary->map); + + index = flb_otel_utils_find_map_entry_by_key(&summary_obj->via.map, + "name", + 0, + FLB_TRUE); + TEST_CHECK(index >= 0); + if (index >= 0) { + field_obj = &summary_obj->via.map.ptr[index].val; + TEST_CHECK(field_obj->type == MSGPACK_OBJECT_STR); + if (field_obj->type == MSGPACK_OBJECT_STR) { + TEST_CHECK(strlen(summary->opts.name) == + field_obj->via.str.size); + TEST_CHECK(strncmp(summary->opts.name, + field_obj->via.str.ptr, + field_obj->via.str.size) == 0); + } + } + + index = flb_otel_utils_find_map_entry_by_key(&summary_obj->via.map, + "count", + 0, + FLB_TRUE); + TEST_CHECK(index >= 0); + if (index >= 0) { + field_obj = &summary_obj->via.map.ptr[index].val; + TEST_CHECK(test_msgpack_object_to_u64(field_obj, + &expected_count) == 0); + if (test_msgpack_object_to_u64(field_obj, + &expected_count) == 0) { + TEST_CHECK(cmt_summary_get_count_value(metric) == + expected_count); + } + } + + index = flb_otel_utils_find_map_entry_by_key(&summary_obj->via.map, + "sum", + 0, + FLB_TRUE); + TEST_CHECK(index >= 0); + if (index >= 0) { + field_obj = &summary_obj->via.map.ptr[index].val; + TEST_CHECK(test_msgpack_object_to_double(field_obj, + &expected_value) == 0); + if (test_msgpack_object_to_double(field_obj, + &expected_value) == 0) { + value = cmt_summary_get_sum_value(metric); + TEST_CHECK(value == expected_value); + } + } + } + + exp_histogram_index = flb_otel_utils_find_map_entry_by_key( + &expected_obj->via.map, + "exp_histogram", + 0, + FLB_TRUE); + if (exp_histogram_index >= 0) { + exp_histogram_obj = &expected_obj->via.map.ptr[exp_histogram_index].val; + exp_histogram = cfl_list_entry(context->exp_histograms.next, + struct cmt_exp_histogram, _head); + metric = &exp_histogram->map->metric; + test_check_metric_description_unit(exp_histogram_obj, + &exp_histogram->opts, + exp_histogram->map); + + index = flb_otel_utils_find_map_entry_by_key( + &exp_histogram_obj->via.map, + "name", + 0, + FLB_TRUE); + TEST_CHECK(index >= 0); + if (index >= 0) { + field_obj = &exp_histogram_obj->via.map.ptr[index].val; + TEST_CHECK(field_obj->type == MSGPACK_OBJECT_STR); + if (field_obj->type == MSGPACK_OBJECT_STR) { + TEST_CHECK(strlen(exp_histogram->opts.name) == + field_obj->via.str.size); + TEST_CHECK(strncmp(exp_histogram->opts.name, + field_obj->via.str.ptr, + field_obj->via.str.size) == 0); + } + } + + index = flb_otel_utils_find_map_entry_by_key( + &exp_histogram_obj->via.map, + "count", + 0, + FLB_TRUE); + TEST_CHECK(index >= 0); + if (index >= 0) { + field_obj = &exp_histogram_obj->via.map.ptr[index].val; + TEST_CHECK(test_msgpack_object_to_u64(field_obj, + &expected_count) == 0); + if (test_msgpack_object_to_u64(field_obj, + &expected_count) == 0) { + TEST_CHECK(metric->exp_hist_count == expected_count); + } + } + + index = flb_otel_utils_find_map_entry_by_key( + &exp_histogram_obj->via.map, + "sum", + 0, + FLB_TRUE); + if (index >= 0) { + field_obj = &exp_histogram_obj->via.map.ptr[index].val; + TEST_CHECK(test_msgpack_object_to_double(field_obj, + &expected_value) == 0); + if (test_msgpack_object_to_double(field_obj, + &expected_value) == 0) { + value = cmt_math_uint64_to_d64(metric->exp_hist_sum); + TEST_CHECK(value == expected_value); + } + } + + index = flb_otel_utils_find_map_entry_by_key( + &exp_histogram_obj->via.map, + "aggregation_type", + 0, + FLB_TRUE); + if (index >= 0) { + field_obj = &exp_histogram_obj->via.map.ptr[index].val; + TEST_CHECK(test_msgpack_object_to_int(field_obj, + &expected_aggregation_type) == 0); + if (test_msgpack_object_to_int(field_obj, + &expected_aggregation_type) == 0) { + TEST_CHECK(exp_histogram->aggregation_type == + expected_aggregation_type); + } + } + } + + } + } + } + + if (decode_ret == CMT_DECODE_OPENTELEMETRY_SUCCESS) { + destroy_metrics_context_list(&context_list); + } + + test_destroy_label_values(expected_label_count, expected_label_values); + + flb_free(input_json); +} void test_opentelemetry_cases() { @@ -1013,6 +1841,84 @@ void test_trace_span_binary_sizes() TEST_CHECK(found_span_id == 1); } +void test_opentelemetry_metrics_cases() +{ + int ret; + int type; + size_t index; + char *tmp_buf; + char *cases_json; + size_t tmp_size; + msgpack_unpacked result; + msgpack_object *root; + msgpack_object *case_obj; + char *case_name; + + tmp_buf = NULL; + cmt_initialize(); + + cases_json = mk_file_to_buffer(OTEL_METRICS_TEST_CASES_PATH); + + TEST_CHECK(cases_json != NULL); + if (cases_json == NULL) { + flb_error("could not read metrics test cases from '%s'", + OTEL_METRICS_TEST_CASES_PATH); + return; + } + + ret = flb_pack_json(cases_json, strlen(cases_json), + &tmp_buf, &tmp_size, &type, NULL); + TEST_CHECK(ret == 0); + if (ret != 0) { + flb_free(cases_json); + return; + } + + msgpack_unpacked_init(&result); + ret = msgpack_unpack_next(&result, tmp_buf, tmp_size, NULL); + TEST_CHECK(ret == MSGPACK_UNPACK_SUCCESS); + if (ret != MSGPACK_UNPACK_SUCCESS) { + msgpack_unpacked_destroy(&result); + flb_free(tmp_buf); + flb_free(cases_json); + return; + } + + root = &result.data; + TEST_CHECK(root->type == MSGPACK_OBJECT_MAP); + if (root->type != MSGPACK_OBJECT_MAP) { + msgpack_unpacked_destroy(&result); + flb_free(tmp_buf); + flb_free(cases_json); + return; + } + + for (index = 0; index < root->via.map.size; index++) { + case_name = flb_malloc(root->via.map.ptr[index].key.via.str.size + 1); + TEST_CHECK(case_name != NULL); + if (case_name == NULL) { + flb_errno(); + break; + } + + memcpy(case_name, + root->via.map.ptr[index].key.via.str.ptr, + root->via.map.ptr[index].key.via.str.size); + case_name[root->via.map.ptr[index].key.via.str.size] = '\0'; + + printf(">> running metrics test case '%s'\n", case_name); + + case_obj = &root->via.map.ptr[index].val; + run_metrics_case(case_obj, case_name); + + flb_free(case_name); + } + + msgpack_unpacked_destroy(&result); + flb_free(tmp_buf); + flb_free(cases_json); +} + /* Test list */ TEST_LIST = { { "hex_to_id", test_hex_to_id }, @@ -1023,6 +1929,6 @@ TEST_LIST = { { "opentelemetry_cases", test_opentelemetry_cases }, { "opentelemetry_traces_cases", test_opentelemetry_traces_cases }, { "trace_span_binary_sizes", test_trace_span_binary_sizes }, + { "opentelemetry_metrics_cases", test_opentelemetry_metrics_cases }, { 0 } }; -