From 86a004a4df8cb0f53d5b087b2af9d9697219bd83 Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Fri, 6 Nov 2020 15:13:22 -0800 Subject: [PATCH 01/20] Begin transferring metrics Signed-off-by: Luc Perkins --- docs/reference/components.cue | 1 + docs/reference/metrics.cue | 1139 ++++++++++++++++++++++++++++++++- 2 files changed, 1131 insertions(+), 9 deletions(-) diff --git a/docs/reference/components.cue b/docs/reference/components.cue index 779b77d1ecfca..8a4a096c4cf56 100644 --- a/docs/reference/components.cue +++ b/docs/reference/components.cue @@ -901,6 +901,7 @@ components: { telemetry: metrics: { // Default metrics for each component vector_events_processed_total: _vector_events_processed_total + vector_processed_bytes_total: _vector_processed_bytes_total } }} } diff --git a/docs/reference/metrics.cue b/docs/reference/metrics.cue index d015180291973..cdbc44d01c720 100644 --- a/docs/reference/metrics.cue +++ b/docs/reference/metrics.cue @@ -1,11 +1,917 @@ package metadata -// All available metrics -_vector_events_processed_total: { +// Apache metrics +_apache_access_total: { + description: "The total number of time the Apache server has been accessed." + relevant_when: "`ExtendedStatus On`" + type: "counter" + tags: _apache_metrics_tags +} +_apache_connections: { + description: "The total number of time the Apache server has been accessed." + type: "gauge" + tags: _apache_metrics_tags & { + state: { + description: "The state of the connection" + required: true + examples: ["closing", "keepalive", "total", "writing"] + } + } +} +_apache_cpu_load: { + description: "The current CPU of the Apache server." + relevant_when: "`ExtendedStatus On`" + type: "gauge" + tags: _apache_metrics_tags +} +_apache_cpu_seconds_total: { + description: "The CPU time of various Apache processes." + relevant_when: "`ExtendedStatus On`" + type: "counter" + tags: _apache_metrics_tags & { + state: { + description: "The state of the connection" + required: true + examples: ["children_system", "children_user", "system", "user"] + } + } +} +_apache_duration_seconds_total: { + description: "The amount of time the Apache server has been running." + relevant_when: "`ExtendedStatus On`" + type: "counter" + tags: _apache_metrics_tags +} +_apache_scoreboard: { + description: "The amount of times various Apache server tasks have been run." + type: "gauge" + tags: _apache_metrics_tags & { + state: { + description: "The connect state" + required: true + examples: ["closing", "dnslookup", "finishing", "idle_cleanup", "keepalive", "logging", "open", "reading", "sending", "starting", "waiting"] + } + } +} +_apache_sent_bytes_total: { + description: "The amount of bytes sent by the Apache server." + relevant_when: "`ExtendedStatus On`" + type: "counter" + tags: _apache_metrics_tags +} +_apache_up: { + description: "If the Apache server is up or not." + type: "gauge" + tags: _apache_metrics_tags +} +_apache_uptime_seconds_total: { + description: "The amount of time the Apache server has been running." + type: "counter" + tags: _apache_metrics_tags +} +_apache_workers: { + description: "Apache worker statuses." + type: "gauge" + tags: _apache_metrics_tags & { + state: { + description: "The state of the worker" + required: true + examples: ["busy", "idle"] + } + } +} + +// Container metrics +_communication_errors_total: { + description: "The total number of errors stemming from communication with the Docker daemon." + type: "counter" + tags: _component_tags +} + +_container_events_processed_total: { + description: "The total number of container events processed." + type: "counter" + tags: _component_tags +} + +_container_metadata_fetch_errors_total: { + description: "The total number of errors caused by failure to fetch container metadata." + type: "counter" + tags: _component_tags +} + +_containers_unwatched_total: { + description: "The total number of times Vector stopped watching for container logs." + counter: "counter" + tags: _component_tags +} + +_containers_watched_total: { + description: "The total number of times Vector started watching for container logs." + counter: "counter" + tags: _component_tags +} + +_logging_driver_errors_total: { + description: "The total number of logging driver errors encountered caused by not using either the `jsonfile` or `journald` driver." + type: "counter" + tags: _component_tags +} + +// Host metrics +// Host CPU +_host_cpu_seconds_total: { + description: "The number of CPU seconds accumulated in different operating modes." + type: "counter" + tags: _host_metrics_tags & { + collector: examples: ["cpu"] + cpu: { + description: "The index of the CPU core or socket." + required: true + examples: ["1"] + } + mode: { + description: "Which mode the CPU was running in during the given time." + required: true + examples: ["idle", "system", "user", "nice"] + } + } +} + +// Host disk +_host_disk_read_bytes_total: _disk_counter & {description: "The accumulated number of bytes read in."} +_host_disk_reads_completed_total: _disk_counter & {description: "The accumulated number of read operations completed."} +_host_disk_written_bytes_total: _disk_counter & {description: "The accumulated number of bytes written out."} +_host_disk_writes_completed_total: _disk_counter & {description: "The accumulated number of write operations completed."} + +// Host filesystem +_host_filesystem_free_bytes: _filesystem_bytes & {description: "The number of bytes free on the named filesystem."} +_host_filesystem_total_bytes: _filesystem_bytes & {description: "The total number of bytes in the named filesystem."} +_host_filesystem_used_bytes: _filesystem_bytes & {description: "The number of bytes used on the named filesystem."} + +// Host load +_host_load1: _loadavg & {description: "System load averaged over the last 1 second."} +_host_load5: _loadavg & {description: "System load averaged over the last 5 seconds."} +_host_load15: _loadavg & {description: "System load averaged over the last 15 seconds."} + +// Host memory +_host_memory_active_bytes: _memory_gauge & _memory_nowin & {description: "The number of bytes of active main memory."} +_host_memory_available_bytes: _memory_gauge & {description: "The number of bytes of main memory available."} +_host_memory_buffers_bytes: _memory_linux & {description: "The number of bytes of main memory used by buffers."} +_host_memory_cached_bytes: _memory_linux & {description: "The number of bytes of main memory used by cached blocks."} +_host_memory_free_bytes: _memory_gauge & {description: "The number of bytes of main memory not used."} +_host_memory_inactive_bytes: _memory_macos & {description: "The number of bytes of main memory that is not active."} +_host_memory_shared_bytes: _memory_linux & {description: "The number of bytes of main memory shared between processes."} +_host_memory_swap_free_bytes: _memory_gauge & {description: "The number of free bytes of swap space."} +_host_memory_swapped_in_bytes_total: _memory_counter & _memory_nowin & { + description: "The number of bytes that have been swapped in to main memory." +} +_host_memory_swapped_out_bytes_total: _memory_counter & _memory_nowin & { + description: "The number of bytes that have been swapped out from main memory." +} +_host_memory_swap_total_bytes: _memory_gauge & {description: "The total number of bytes of swap space."} +_host_memory_swap_used_bytes: _memory_gauge & {description: "The number of used bytes of swap space."} +_host_memory_total_bytes: _memory_gauge & {description: "The total number of bytes of main memory."} +_host_memory_used_bytes: _memory_linux & {description: "The number of bytes of main memory used by programs or caches."} +_host_memory_wired_bytes: _memory_macos & {description: "The number of wired bytes of main memory."} + +// Host network +_host_network_receive_bytes_total: _network_gauge & {description: "The number of bytes received on this interface."} +_host_network_receive_errs_total: _network_gauge & {description: "The number of errors encountered during receives on this interface."} +_host_network_receive_packets_total: _network_gauge & {description: "The number of packets received on this interface."} +_host_network_transmit_bytes_total: _network_gauge & {description: "The number of bytes transmitted on this interface."} +_host_network_transmit_errs_total: _network_gauge & {description: "The number of errors encountered during transmits on this interface."} +_host_network_transmit_packets_drop_total: _network_nomac & {description: "The number of packets dropped during transmits on this interface."} +_host_network_transmit_packets_total: _network_nomac & {description: "The number of packets transmitted on this interface."} + +// Kubernetes metrics +_k8s_docker_format_parse_failures_total: { + description: "The total number of failures to parse a message as a JSON object." + type: "counter" + tags: _component_tags +} + +_k8s_event_annotation_failures_total: { + description: "The total number of failures to annotate Vector events with Kubernetes Pod metadata." type: "counter" - description: "The total number of events processed by this component." tags: _component_tags } + +// MongoDB metrics +_mongodb_bson_parse_error_total: { + description: "The total number of BSON parsing errors." + type: "counter" + tags: _mongodb_metrics_tags +} + +_mongodb_assets_total: { + description: "Number of assertions raised since the MongoDB process started." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "The assertion type" + required: true + examples: ["regular", "warning", "msg", "user", "rollovers"] + } + } +} +_mongodb_connections: { + description: "Number of connections in some state." + type: "gauge" + tags: _mongodb_metrics_tags & { + state: { + description: "The connection state" + required: true + examples: ["active", "available", "current"] + } + } +} +_mongodb_extra_info_heap_usage_bytes: { + description: "The total size in bytes of heap space used by the database process." + relevant_when: "Unix/Linux" + type: "gauge" + tags: _mongodb_metrics_tags +} +_mongodb_extra_info_page_faults: { + description: "The total number of page faults." + type: "gauge" + tags: _mongodb_metrics_tags +} +_mongodb_instance_local_time: { + description: "The ISODate representing the current time, according to the server, in UTC." + type: "gauge" + tags: _mongodb_metrics_tags +} +_mongodb_instance_uptime_estimate_seconds_total: { + description: "The uptime in seconds as calculated from MongoDB’s internal course-grained time keeping system." + type: "gauge" + tags: _mongodb_metrics_tags +} +_mongodb_instance_uptime_seconds_total: { + description: "The number of seconds that the current MongoDB process has been active." + type: "gauge" + tags: _mongodb_metrics_tags +} +_mongodb_memory: { + description: "Current memory unsage." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Memory type" + required: true + examples: ["resident", "virtual", "mapped", "mapped_with_journal"] + } + } +} +_mongodb_mongod_global_lock_active_clients: { + description: "Number of connected clients and the read and write operations performed by these clients." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Number type." + required: true + examples: ["total", "readers", "writers"] + } + } +} +_mongodb_mongod_global_lock_current_queue: { + description: "Number of operations queued because of a lock." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Number type." + required: true + examples: ["total", "readers", "writers"] + } + } +} +_mongodb_mongod_global_lock_total_time_seconds: { + description: "The time since the database last started and created the globalLock. This is roughly equivalent to total server uptime." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_locks_time_acquiring_global_seconds_total: { + description: "Amount of time that any database has spent waiting for the global lock." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Lock type." + required: true + examples: ["ParallelBatchWriterMode", "ReplicationStateTransition", "Global", "Database", "Collection", "Mutex", "Metadata", "oplog"] + } + mode: { + description: "Lock mode." + required: true + examples: ["read", "write"] + } + } +} +_mongodb_mongod_metrics_cursor_open: { + description: "Number of cursors." + type: "gauge" + tags: _mongodb_metrics_tags & { + state: { + description: "Cursor state." + required: true + examples: ["no_timeout", "pinned", "total"] + } + } +} +_mongodb_mongod_metrics_cursor_timed_out_total: { + description: "The total number of cursors that have timed out since the server process started." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_metrics_document_total: { + description: "Document access and modification patterns." + type: "counter" + tags: _mongodb_metrics_tags & { + state: { + description: "Document state." + required: true + examples: ["deleted", "inserted", "returned", "updated"] + } + } +} +_mongodb_mongod_metrics_get_last_error_wtime_num: { + description: "The total number of getLastError operations with a specified write concern." + type: "gauge" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_metrics_get_last_error_wtime_seconds_total: { + description: "The total amount of time that the mongod has spent performing getLastError operations." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_metrics_get_last_error_wtimeouts_total: { + description: "The number of times that write concern operations have timed out as a result of the wtimeout threshold to getLastError." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_metrics_operation_total: { + description: "Update and query operations that MongoDB handles using special operation types." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Operation type." + required: true + examples: ["scan_and_order", "write_conflicts"] + } + } +} +_mongodb_mongod_metrics_query_executor_total: { + description: "Data from query execution system." + type: "counter" + tags: _mongodb_metrics_tags & { + state: { + description: "Query state." + required: true + examples: ["scanned", "scanned_objects", "collection_scans"] + } + } +} +_mongodb_mongod_metrics_record_moves_total: { + description: "Moves reports the total number of times documents move within the on-disk representation of the MongoDB data set. Documents move as a result of operations that increase the size of the document beyond their allocated record size." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_metrics_repl_apply_batches_num_total: { + description: "The total number of batches applied across all databases." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_metrics_repl_apply_batches_seconds_total: { + description: "The total amount of time the mongod has spent applying operations from the oplog." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_metrics_repl_apply_ops_total: { + description: "The total number of oplog operations applied." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_metrics_repl_buffer_count: { + description: "The current number of operations in the oplog buffer." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_metrics_repl_buffer_max_size_bytes_total: { + description: "The maximum size of the buffer." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_metrics_repl_buffer_size_bytes: { + description: "The current size of the contents of the oplog buffer." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_metrics_repl_executor_queue: { + description: "Number of queued operations in the replication executor." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Queue type." + required: true + examples: ["network_in_progress", "sleepers"] + } + } +} +_mongodb_mongod_metrics_repl_executor_unsignaled_events: { + description: "Number of unsignaled events in the replication executor." + type: "gauge" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_metrics_repl_network_bytes_total: { + description: "The total amount of data read from the replication sync source." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_metrics_repl_network_getmores_num_total: { + description: "The total number of getmore operations, which are operations that request an additional set of operations from the replication sync source." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_metrics_repl_network_getmores_seconds_total: { + description: "The total amount of time required to collect data from getmore operations." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_metrics_repl_network_ops_total: { + description: "The total number of operations read from the replication source." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_metrics_repl_network_readers_created_total: { + description: "The total number of oplog query processes created." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_metrics_ttl_deleted_documents_total: { + description: "The total number of documents deleted from collections with a ttl index." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_metrics_ttl_passes_total: { + description: "The number of times the background process removes documents from collections with a ttl index." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_op_latencies_histogram: { + description: "Latency statistics." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Latency type." + required: true + examples: ["reads", "writes", "commands"] + } + micros: { + description: "Bucket." + required: true + examples: ["1", "2", "4096", "16384", "49152"] + } + } +} +_mongodb_mongod_op_latencies_latency: { + description: "A 64-bit integer giving the total combined latency in microseconds." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Latency type." + required: true + examples: ["network_in_progress", "sleepers"] + } + } +} +_mongodb_mongod_op_latencies_ops_total: { + description: "A 64-bit integer giving the total number of operations performed on the collection since startup." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Latency type." + required: true + examples: ["network_in_progress", "sleepers"] + } + } +} +_mongodb_mongod_storage_engine: { + description: "The name of the current storage engine." + type: "gauge" + tags: _mongodb_metrics_tags & { + engine: { + description: "Engine name." + required: true + examples: ["wiredTiger"] + } + } +} +_mongodb_mongod_wiredtiger_blockmanager_blocks_total: { + description: "Statistics on the block manager operations." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Operation type." + required: true + examples: ["blocks_read", "blocks_read_mapped", "blocks_pre_loaded", "blocks_written"] + } + } +} +_mongodb_mongod_wiredtiger_blockmanager_bytes_total: { + description: "Statistics on the block manager operations." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Operation type." + required: true + examples: ["bytes_read", "bytes_read_mapped", "bytes_written"] + } + } +} +_mongodb_mongod_wiredtiger_cache_bytes: { + description: "Statistics on the cache and page evictions from the cache." + relevant_when: "Storage engine is `wiredTiger`." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Statistics type." + required: true + examples: ["total", "dirty", "internal_pages", "leaf_pages"] + } + } +} +_mongodb_mongod_wiredtiger_cache_bytes_total: { + description: "Statistics on the cache and page evictions from the cache." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Statistics type." + required: true + examples: ["read", "written"] + } + } +} +_mongodb_mongod_wiredtiger_cache_evicted_total: { + description: "Statistics on the cache and page evictions from the cache." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Statistics type." + required: true + examples: ["modified", "unmodified"] + } + } +} +_mongodb_mongod_wiredtiger_cache_max_bytes: { + description: "Maximum cache size." + type: "gauge" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_wiredtiger_cache_overhead_percent: { + description: "Percentage overhead." + type: "gauge" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_wiredtiger_cache_pages: { + description: "Pages in the cache." + relevant_when: "Storage engine is `wiredTiger`." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Pages type." + required: true + examples: ["total", "dirty"] + } + } +} +_mongodb_mongod_wiredtiger_cache_pages_total: { + description: "Pages in the cache." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Pages type." + required: true + examples: ["read", "write"] + } + } +} +_mongodb_mongod_wiredtiger_concurrent_transactions_available_tickets: { + description: "Information on the number of concurrent of read and write transactions allowed into the WiredTiger storage engine" + relevant_when: "Storage engine is `wiredTiger`." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Transactions type." + required: true + examples: ["read", "write"] + } + } +} +_mongodb_mongod_wiredtiger_concurrent_transactions_out_tickets: { + description: "Information on the number of concurrent of read and write transactions allowed into the WiredTiger storage engine" + relevant_when: "Storage engine is `wiredTiger`." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Transactions type." + required: true + examples: ["read", "write"] + } + } +} +_mongodb_mongod_wiredtiger_concurrent_transactions_total_tickets: { + description: "Information on the number of concurrent of read and write transactions allowed into the WiredTiger storage engine" + relevant_when: "Storage engine is `wiredTiger`." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Transactions type." + required: true + examples: ["read", "write"] + } + } +} +_mongodb_mongod_wiredtiger_log_bytes_total: { + description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Bytes type." + required: true + examples: ["payload", "written"] + } + } +} +_mongodb_mongod_wiredtiger_log_operations_total: { + description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Operations type." + required: true + examples: ["write", "scan", "scan_double", "sync", "sync_dir", "flush"] + } + } +} +_mongodb_mongod_wiredtiger_log_records_scanned_total: { + description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Scanned records type." + required: true + examples: ["compressed", "uncompressed"] + } + } +} +_mongodb_mongod_wiredtiger_log_records_total: { + description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_wiredtiger_session_open_sessions: { + description: "Open session count." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_wiredtiger_transactions_checkpoint_seconds: { + description: "Statistics on transaction checkpoints and operations." + relevant_when: "Storage engine is `wiredTiger`." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Checkpoint type." + required: true + examples: ["min", "max"] + } + } +} +_mongodb_mongod_wiredtiger_transactions_checkpoint_seconds_total: { + description: "Statistics on transaction checkpoints and operations." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_wiredtiger_transactions_running_checkpoints: { + description: "Statistics on transaction checkpoints and operations." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_mongod_wiredtiger_transactions_total: { + description: "Statistics on transaction checkpoints and operations." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Transactions type." + required: true + examples: ["begins", "checkpoints", "committed", "rolledback"] + } + } +} +_mongodb_network_bytes_total: { + description: "The number of bytes that reflects the amount of network traffic." + type: "counter" + tags: _mongodb_metrics_tags & { + state: { + description: "Bytes state." + required: true + examples: ["bytes_in", "bytes_out"] + } + } +} +_mongodb_network_metrics_num_requests_total: { + description: "The total number of distinct requests that the server has received." + type: "counter" + tags: _mongodb_metrics_tags +} +_mongodb_op_counters_repl_total: { + description: "Database replication operations by type since the mongod instance last started." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Counter type." + required: true + examples: ["insert", "query", "update", "delete", "getmore", "command"] + } + } +} +_mongodb_op_counters_total: { + description: "Database operations by type since the mongod instance last started." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Counter type." + required: true + examples: ["insert", "query", "update", "delete", "getmore", "command"] + } + } +} +_mongodb_up: { + description: "If the MongoDB server is up or not." + type: "gauge" + tags: _mongodb_metrics_tags +} + +// Vector internal metrics (plus misc) +_vector_api_started_total: { + description: "The number of times the Vector GraphQL API has been started." + type: "counter" + tags: _internal_metrics_tags +} +_vector_auto_concurrency_averaged_rtt: { + description: "TODO" + type: "histogram" + tags: _internal_metrics_tags +} +_vector_auto_concurrency_in_flight: { + description: "The number of outbound requests from the HTTP sink currently awaiting a response." + type: "histogram" + tags: _internal_metrics_tags +} +_vector_auto_concurrency_limit: { + description: "TODO" + type: "histogram" + tags: _internal_metrics_tags +} +_vector_auto_concurrency_observed_rtt: { + description: "TODO" + type: "histogram" + tags: _internal_metrics_tags +} +_vector_checkpoint_write_errors_total: { + description: "The total number of errors writing checkpoints." + type: "counter" + tags: _internal_metrics_tags +} +_vector_checkpoints_total: { + description: "The total number of files checkpointed." + type: "counter" + tags: _internal_metrics_tags +} +_vector_checksum_errors: { + description: "TODO" + type: "counter" + tags: _internal_metrics_tags & { + file: _file + } +} +_vector_collect_duration_nanoseconds: { + description: "TODO" + type: "histogram" + tags: _internal_metrics_tags +} +_vector_collect_completed_total: { + description: "TODO" + type: "counter" + tags: _internal_metrics_tags +} +_vector_events_discarded_total: { + description: "The total number of events discarded by this component." + type: "counter" + tags: _internal_metrics_tags +} +_vector_events_processed_total: { + description: "The total number of events processed by this component." + type: "counter" + tags: _component_tags & { + file: _file + } +} +_vector_file_delete_errors: { + description: "The total number of failures to delete a file." + type: "counter" + tags: _internal_metrics_tags & { + file: _file + } +} +_vector_file_watch_errors: { + description: "The total number of errors caused by failure to watch a file." + type: "counter" + tags: _internal_metrics_tags & { + file: _file + } +} +_vector_files_added: { + description: "The total number of files Vector has found to watch." + type: "counter" + tags: _internal_metrics_tags & { + file: _file + } +} +_vector_files_deleted: { + description: "The total number of files deleted." + type: "counter" + tags: _internal_metrics_tags & { + file: _file + } +} +_vector_files_resumed: { + description: "The total number of times Vector has resumed watching a file." + type: "counter" + tags: _internal_metrics_tags & { + file: _file + } +} +_vector_files_unwatched: { + description: "The total number of times Vector has stopped watching a file." + type: "counter" + tags: _internal_metrics_tags & { + file: _file + } +} +_vector_fingerprint_read_errors: { + description: "The total number of times failing to read a file for fingerprinting." + type: "counter" + tags: _internal_metrics_tags & { + file: _file + } +} +_vector_http_bad_requests_total: { + description: "The total number of HTTP `400 Bad Request` errors encountered." + type: "counter" + tags: _internal_metrics_tags +} +_vector_http_error_response_total: { + description: "TODO" + type: "counter" + tags: _internal_metrics_tags +} +_vector_http_request_errors_total: { + description: "TODO" + type: "counter" + tags: _internal_metrics_tags +} +_vector_memory_used: { + description: "The total memory currently being used by Vector (in bytes)." + type: "gauge" + tags: _internal_metrics_tags +} +_vector_missing_keys_total: { + description: "The total number of events dropped due to keys missing from the event." + type: "counter" + tags: _internal_metrics_tags +} +_vector_open_connections: { + description: "The number of current open connections to Vector." + type: "gauge" + tags: _internal_metrics_tags +} +_vector_parse_errors_total: { + description: "TODO" + type: "counter" + tags: _internal_metrics_tags +} _vector_processed_bytes_total: { description: "The total number of bytes processed by the component." type: "counter" @@ -18,9 +924,39 @@ _vector_processing_errors_total: { error_type: _error_type } } +_vector_request_duration_nanoseconds: { + description: "TODO" + type: "histogram" + tags: _component_tags +} +_vector_request_read_errors_total: { + description: "TODO" + type: "counter" + tags: _component_tags +} +_vector_requests_completed_total: { + description: "TODO" + type: "counter" + tags: _component_tags +} +_vector_requests_received_total: { + description: "TODO" + type: "counter" + tags: _component_tags +} +_vector_timestamp_parse_errors_total: { + description: "The total number of errors encountered RFC3339 parsing timestamps." + type: "counter" + tags: _component_tags +} +_vector_uptime_seconds: { + description: "The total number of seconds the Vector instance has been up." + type: "gauge" + tags: _component_tags +} // Convenient groupings of tags -_component_tags: { +_component_tags: _internal_metrics_tags & { component_kind: _component_kind component_name: _component_name component_type: _component_type @@ -28,15 +964,126 @@ _component_tags: { job: _job } +_apache_metrics_tags: { + endpoint: _endpoint + host: { + description: "The hostname of the Apache HTTP server." + required: true + examples: [_values.local_host] + } +} +_host_metrics_tags: { + collector: _collector + host: _host +} +_internal_metrics_tags: { + instance: _instance + job: _job +} +_mongodb_metrics_tags: { + endpoint: { + description: "The absolute path of the originating file." + required: true + examples: ["mongodb://localhost:27017"] + } + host: { + description: "The hostname of the MongoDB server." + required: true + examples: [_values.local_host] + } +} + +// Vector instance metrics +_vector_config_load_errors_total: { + description: "The total number of errors loading the Vector configuration." + type: "counter" + tags: _internal_metrics_tags +} +_vector_quit_total: { + description: "The total number of times the Vector instance has quit." + type: "counter" + tags: _internal_metrics_tags +} +_vector_recover_errors_total: { + description: "The total number of errors caused by Vector failing to recover from a failed reload." + type: "counter" + tags: _internal_metrics_tags +} +_vector_reload_errors_total: { + description: "The total number of errors encountered when reloading Vector." + type: "counter" + tags: _internal_metrics_tags +} +_vector_reloaded_total: { + description: "The total number of times the Vector instance has been reloaded." + type: "counter" + tags: _internal_metrics_tags +} +_vector_started_total: { + description: "The total number of times the Vector instance has been started." + type: "counter" + tags: _internal_metrics_tags +} +_vector_stopped_total: { + description: "The total number of times the Vector instance has been stopped." + type: "counter" + tags: _internal_metrics_tags +} + +// Windows metrics +_windows_service_does_not_exist: { + description: """ + The total number of errors raised due to the Windows service not + existing. + """ + type: "counter" + tags: _internal_metrics_tags +} +_windows_service_install: { + description: """ + The total number of times the Windows service has been installed. + """ + type: "counter" + tags: _internal_metrics_tags +} +_windows_service_restart: { + description: """ + The total number of times the Windows service has been restarted. + """ + type: "counter" + tags: _internal_metrics_tags +} +_windows_service_start: { + description: """ + The total number of times the Windows service has been started. + """ + type: "counter" + tags: _internal_metrics_tags +} +_windows_service_stop: { + description: """ + The total number of times the Windows service has been stopped. + """ + type: "counter" + tags: _internal_metrics_tags +} +_windows_service_uninstall: { + description: """ + The total number of times the Windows service has been uninstalled. + """ + type: "counter" + tags: _internal_metrics_tags +} + // All available tags +_collector: { + description: "Which collector this metric comes from." + required: true +} _component_kind: { description: "The component's kind (options are `source`, `sink`, or `transform`)." required: true - options: { - sink: "Sink component." - source: "Source component." - transform: "Transform component." - } + options: ["sink", "source", "transform"] } _component_name: { description: "The name of the component as specified in the Vector configuration." @@ -48,6 +1095,11 @@ _component_type: { required: true examples: ["file", "http", "honeycomb", "splunk_hec"] } +_endpoint: { + description: "The absolute path of originating file." + required: true + examples: ["http://localhost:8080/server-status?auto"] +} _error_type: { description: "The type of the error" required: true @@ -62,6 +1114,15 @@ _error_type: { "value_invalid", ] } +_file: { + description: "The file that produced the error" + required: false +} +_host: { + description: "The hostname of the originating system." + required: true + examples: [_values.local_host] +} _instance: { description: "The Vector instance identified by host and port." required: true @@ -72,3 +1133,63 @@ _job: { required: true default: "vector" } + +// Other helpers +_disk_device: { + description: "The disk device name." + required: true + examples: ["sda", "sda1", "dm-1"] +} +_disk_counter: { + type: "counter" + tags: _host_metrics_tags & { + collector: examples: ["disk"] + device: _disk_device + } +} +_filesystem_bytes: { + type: "gauge" + tags: _host_metrics_tags & { + collector: examples: ["filesystem"] + device: _disk_device + filesystem: { + description: "The name of the filesystem type." + required: true + examples: ["ext4", "ntfs"] + } + } +} +_loadavg: { + type: "gauge" + tags: _host_metrics_tags & { + collector: examples: ["loadavg"] + } + relevant_when: "OS is not Windows" +} +_memory_counter: { + type: "counter" + tags: _host_metrics_tags & { + collector: examples: ["memory"] + } +} +_memory_gauge: { + type: "gauge" + tags: _host_metrics_tags & { + collector: examples: ["memory"] + } +} +_memory_linux: _memory_gauge & {relevant_when: "OS is Linux"} +_memory_macos: _memory_gauge & {relevant_when: "OS is MacOS X"} +_memory_nowin: {relevant_when: "OS is not Windows"} +_network_gauge: { + type: "gauge" + tags: _host_metrics_tags & { + collector: examples: ["network"] + device: { + description: "The network interface device name." + required: true + examples: ["eth0", "enp5s3"] + } + } +} +_network_nomac: _network_gauge & {relevant_when: "OS is not MacOS"} From 0fe4e538d5e8cbd640cbc7d887a5d24caf18fc93 Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Fri, 6 Nov 2020 20:49:45 -0800 Subject: [PATCH 02/20] Metrics docs for sinks Signed-off-by: Luc Perkins --- .../components/sinks/aws_cloudwatch_logs.cue | 4 ++ docs/reference/components/sinks/console.cue | 1 - .../components/sinks/elasticsearch.cue | 4 ++ docs/reference/components/sinks/http.cue | 4 ++ docs/reference/components/sinks/kafka.cue | 13 ++++++ .../components/sinks/prometheus_exporter.cue | 2 + .../components/sinks/sematext_metrics.cue | 4 ++ .../reference/components/sinks/splunk_hec.cue | 8 ++++ docs/reference/components/sinks/statsd.cue | 4 ++ docs/reference/components/sinks/vector.cue | 4 ++ docs/reference/metrics.cue | 41 ++++++++++++++++--- 11 files changed, 83 insertions(+), 6 deletions(-) diff --git a/docs/reference/components/sinks/aws_cloudwatch_logs.cue b/docs/reference/components/sinks/aws_cloudwatch_logs.cue index 403172e824e1d..ef5ba061fff55 100644 --- a/docs/reference/components/sinks/aws_cloudwatch_logs.cue +++ b/docs/reference/components/sinks/aws_cloudwatch_logs.cue @@ -119,4 +119,8 @@ components: sinks: aws_cloudwatch_logs: { logs: true metrics: null } + + telemetry: metrics: { + processing_errors_total: _vector_processing_errors_total + } } diff --git a/docs/reference/components/sinks/console.cue b/docs/reference/components/sinks/console.cue index 2f53183d3a54b..77a77a36d1be5 100644 --- a/docs/reference/components/sinks/console.cue +++ b/docs/reference/components/sinks/console.cue @@ -81,7 +81,6 @@ components: sinks: console: { } telemetry: metrics: { - vector_processed_bytes_total: _vector_processed_bytes_total vector_processing_errors_total: _vector_processing_errors_total } } diff --git a/docs/reference/components/sinks/elasticsearch.cue b/docs/reference/components/sinks/elasticsearch.cue index 49bf461eed3bb..51a964a4d432e 100644 --- a/docs/reference/components/sinks/elasticsearch.cue +++ b/docs/reference/components/sinks/elasticsearch.cue @@ -239,4 +239,8 @@ components: sinks: elasticsearch: { """ } } + + telemetry: metrics: { + vector_missing_keys_total: _vector_missing_keys_total + } } diff --git a/docs/reference/components/sinks/http.cue b/docs/reference/components/sinks/http.cue index e35add2c1dc39..b6e86f47c4306 100644 --- a/docs/reference/components/sinks/http.cue +++ b/docs/reference/components/sinks/http.cue @@ -119,4 +119,8 @@ components: sinks: http: { logs: true metrics: null } + + telemetry: metrics: { + vector_http_bad_requests_total: _vector_http_bad_requests_total + } } diff --git a/docs/reference/components/sinks/kafka.cue b/docs/reference/components/sinks/kafka.cue index d6352281380ab..b479b6f4db138 100644 --- a/docs/reference/components/sinks/kafka.cue +++ b/docs/reference/components/sinks/kafka.cue @@ -131,4 +131,17 @@ components: sinks: kafka: { } how_it_works: components._kafka.how_it_works + + telemetry: metrics: { + vector_consumer_offset_updates_failed_total: { + description: "The total number of failures to update a Kafka consumer offset." + type: "counter" + tags: _component_tags + } + vector_events_failed_total: { + description: "The total number of failures to read a Kafka message." + type: "counter" + tags: _component_tags + } + } } diff --git a/docs/reference/components/sinks/prometheus_exporter.cue b/docs/reference/components/sinks/prometheus_exporter.cue index f33630bed7ba3..316f467e14d70 100644 --- a/docs/reference/components/sinks/prometheus_exporter.cue +++ b/docs/reference/components/sinks/prometheus_exporter.cue @@ -342,4 +342,6 @@ components: sinks: prometheus_exporter: { """ } } + + telemetry: metrics: _prometheus_metrics } diff --git a/docs/reference/components/sinks/sematext_metrics.cue b/docs/reference/components/sinks/sematext_metrics.cue index 3bde057adb960..2e25a0af149a6 100644 --- a/docs/reference/components/sinks/sematext_metrics.cue +++ b/docs/reference/components/sinks/sematext_metrics.cue @@ -78,4 +78,8 @@ components: sinks: sematext_metrics: { summary: false } } + + telemetry: metrics: { + vector_processing_errors_total: _vector_processing_errors_total + } } diff --git a/docs/reference/components/sinks/splunk_hec.cue b/docs/reference/components/sinks/splunk_hec.cue index 6c5a3271b05d4..94dd7c70270ad 100644 --- a/docs/reference/components/sinks/splunk_hec.cue +++ b/docs/reference/components/sinks/splunk_hec.cue @@ -165,4 +165,12 @@ components: sinks: splunk_hec: { logs: true metrics: null } + + telemetry: metrics: { + vector_encode_errors_total: _vector_encode_errors_total + vector_http_request_errors_total: _vector_http_request_errors_total + vector_http_requests_total: _vector_http_requests_total + vector_source_missing_keys_total: _vector_source_missing_keys_total + vector_sourcetype_missing_keys_total: _vector_sourcetype_missing_keys_total + } } diff --git a/docs/reference/components/sinks/statsd.cue b/docs/reference/components/sinks/statsd.cue index 3f2c4ca305f00..02fa2fff7d803 100644 --- a/docs/reference/components/sinks/statsd.cue +++ b/docs/reference/components/sinks/statsd.cue @@ -63,4 +63,8 @@ components: sinks: statsd: { } } } + + telemetry: metrics: { + vector_processing_errors_total: _vector_processing_errors_total + } } diff --git a/docs/reference/components/sinks/vector.cue b/docs/reference/components/sinks/vector.cue index e8b9d81fff732..8d3659195743a 100644 --- a/docs/reference/components/sinks/vector.cue +++ b/docs/reference/components/sinks/vector.cue @@ -84,4 +84,8 @@ components: sinks: vector: { } how_it_works: components.sources.vector.how_it_works + + telemetry: metrics: { + vector_protobuf_decode_errors_total: _vector_protobuf_decode_errors_total + } } diff --git a/docs/reference/metrics.cue b/docs/reference/metrics.cue index cdbc44d01c720..59d3530a7d754 100644 --- a/docs/reference/metrics.cue +++ b/docs/reference/metrics.cue @@ -1,5 +1,16 @@ package metadata +// Helpful metrics groupings +_prometheus_metrics: { + vector_events_processed_total: _vector_events_processed_total + vector_http_error_response_total: _vector_http_error_response_total + vector_http_request_errors_total: _vector_http_request_errors_total + vector_parse_errors_total: _vector_parse_errors_total + vector_processed_bytes_total: _vector_processed_bytes_total + vector_request_duration_nanoseconds: _vector_request_duration_nanoseconds + vector_requests_completed_total: _vector_requests_completed_total +} + // Apache metrics _apache_access_total: { description: "The total number of time the Apache server has been accessed." @@ -816,6 +827,11 @@ _vector_collect_completed_total: { type: "counter" tags: _internal_metrics_tags } +_vector_encode_errors_total: { + description: "TODO" + type: "counter" + tags: _internal_metrics_tags +} _vector_events_discarded_total: { description: "The total number of events discarded by this component." type: "counter" @@ -892,6 +908,11 @@ _vector_http_request_errors_total: { type: "counter" tags: _internal_metrics_tags } +_vector_http_requests_total: { + description: "TODO" + type: "counter" + tags: _component_tags +} _vector_memory_used: { description: "The total memory currently being used by Vector (in bytes)." type: "gauge" @@ -924,6 +945,11 @@ _vector_processing_errors_total: { error_type: _error_type } } +_vector_protobuf_decode_errors_total: { + description: "TODO" + type: "counter" + tags: _component_tags +} _vector_request_duration_nanoseconds: { description: "TODO" type: "histogram" @@ -939,11 +965,6 @@ _vector_requests_completed_total: { type: "counter" tags: _component_tags } -_vector_requests_received_total: { - description: "TODO" - type: "counter" - tags: _component_tags -} _vector_timestamp_parse_errors_total: { description: "The total number of errors encountered RFC3339 parsing timestamps." type: "counter" @@ -1019,6 +1040,16 @@ _vector_reloaded_total: { type: "counter" tags: _internal_metrics_tags } +_vector_source_missing_keys_total: { + description: "TODO" + type: "counter" + tags: _internal_metrics_tags +} +_vector_sourcetype_missing_keys_total: { + description: "TODO" + type: "counter" + tags: _internal_metrics_tags +} _vector_started_total: { description: "The total number of times the Vector instance has been started." type: "counter" From af82ac71859ab4f38f57ec18354f121aa47856c9 Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Fri, 6 Nov 2020 21:54:56 -0800 Subject: [PATCH 03/20] Add source metrics and refactor metrics.cue Signed-off-by: Luc Perkins --- docs/reference/components/kafka.cue | 13 + docs/reference/components/sinks/kafka.cue | 13 - .../reference/components/sinks/splunk_hec.cue | 8 +- .../components/sources/apache_metrics.cue | 118 +-- .../sources/aws_kinesis_firehose.cue | 5 + .../components/sources/docker_logs.cue | 8 + docs/reference/components/sources/file.cue | 13 + .../components/sources/host_metrics.cue | 131 +--- docs/reference/components/sources/http.cue | 4 + .../reference/components/sources/journald.cue | 13 + .../components/sources/kubernetes_logs.cue | 5 + docs/reference/components/sources/logplex.cue | 5 + .../components/sources/mongodb_metrics.cue | 730 +----------------- .../components/sources/prometheus.cue | 2 + docs/reference/components/sources/socket.cue | 7 +- .../components/sources/splunk_hec.cue | 8 + docs/reference/components/sources/statsd.cue | 14 + docs/reference/components/sources/stdin.cue | 8 + docs/reference/components/sources/syslog.cue | 29 + docs/reference/components/sources/vector.cue | 4 + docs/reference/metrics.cue | 279 +++++-- 21 files changed, 353 insertions(+), 1064 deletions(-) diff --git a/docs/reference/components/kafka.cue b/docs/reference/components/kafka.cue index af955ab2edc1e..ff01043f02c20 100644 --- a/docs/reference/components/kafka.cue +++ b/docs/reference/components/kafka.cue @@ -88,4 +88,17 @@ components: _kafka: { """ } } + + telemetry: metrics: { + vector_consumer_offset_updates_failed_total: { + description: "The total number of failures to update a Kafka consumer offset." + type: "counter" + tags: _component_tags + } + vector_events_failed_total: { + description: "The total number of failures to read a Kafka message." + type: "counter" + tags: _component_tags + } + } } diff --git a/docs/reference/components/sinks/kafka.cue b/docs/reference/components/sinks/kafka.cue index b479b6f4db138..d6352281380ab 100644 --- a/docs/reference/components/sinks/kafka.cue +++ b/docs/reference/components/sinks/kafka.cue @@ -131,17 +131,4 @@ components: sinks: kafka: { } how_it_works: components._kafka.how_it_works - - telemetry: metrics: { - vector_consumer_offset_updates_failed_total: { - description: "The total number of failures to update a Kafka consumer offset." - type: "counter" - tags: _component_tags - } - vector_events_failed_total: { - description: "The total number of failures to read a Kafka message." - type: "counter" - tags: _component_tags - } - } } diff --git a/docs/reference/components/sinks/splunk_hec.cue b/docs/reference/components/sinks/splunk_hec.cue index 94dd7c70270ad..31ea788d8ef24 100644 --- a/docs/reference/components/sinks/splunk_hec.cue +++ b/docs/reference/components/sinks/splunk_hec.cue @@ -167,10 +167,10 @@ components: sinks: splunk_hec: { } telemetry: metrics: { - vector_encode_errors_total: _vector_encode_errors_total - vector_http_request_errors_total: _vector_http_request_errors_total - vector_http_requests_total: _vector_http_requests_total - vector_source_missing_keys_total: _vector_source_missing_keys_total + vector_encode_errors_total: _vector_encode_errors_total + vector_http_request_errors_total: _vector_http_request_errors_total + vector_http_requests_total: _vector_http_requests_total + vector_source_missing_keys_total: _vector_source_missing_keys_total vector_sourcetype_missing_keys_total: _vector_sourcetype_missing_keys_total } } diff --git a/docs/reference/components/sources/apache_metrics.cue b/docs/reference/components/sources/apache_metrics.cue index 1e2cff445ea1c..7d58eefdaf261 100644 --- a/docs/reference/components/sources/apache_metrics.cue +++ b/docs/reference/components/sources/apache_metrics.cue @@ -111,123 +111,7 @@ components: sources: apache_metrics: { } } - output: metrics: { - _endpoint: { - description: "The absolute path of originating file." - required: true - examples: ["http://localhost:8080/server-status?auto"] - } - _host: { - description: "The hostname of the Apache HTTP server" - required: true - examples: [_values.local_host] - } - access_total: { - description: "The total number of time the Apache server has been accessed." - relevant_when: "`ExtendedStatus On`" - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - connections: { - description: "The total number of time the Apache server has been accessed." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - state: { - description: "The state of the connection" - required: true - examples: ["closing", "keepalive", "total", "writing"] - } - } - } - cpu_load: { - description: "The current CPU of the Apache server." - relevant_when: "`ExtendedStatus On`" - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - } - } - cpu_seconds_total: { - description: "The CPU time of various Apache processes." - relevant_when: "`ExtendedStatus On`" - type: "counter" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "The state of the connection" - required: true - examples: ["children_system", "children_user", "system", "user"] - } - } - } - duration_seconds_total: { - description: "The amount of time the Apache server has been running." - relevant_when: "`ExtendedStatus On`" - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - scoreboard: { - description: "The amount of times various Apache server tasks have been run." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - state: { - description: "The connect state" - required: true - examples: ["closing", "dnslookup", "finishing", "idle_cleanup", "keepalive", "logging", "open", "reading", "sending", "starting", "waiting"] - } - } - } - sent_bytes_total: { - description: "The amount of bytes sent by the Apache server." - relevant_when: "`ExtendedStatus On`" - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - uptime_seconds_total: { - description: "The amount of time the Apache server has been running." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - workers: { - description: "Apache worker statuses." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - state: { - description: "The state of the worker" - required: true - examples: ["busy", "idle"] - } - } - } - up: { - description: "If the Apache server is up or not." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - } - } - } + output: metrics: _apache_metrics how_it_works: {} } diff --git a/docs/reference/components/sources/aws_kinesis_firehose.cue b/docs/reference/components/sources/aws_kinesis_firehose.cue index 1e72ae5b3a9c2..4fb9eead2b7f5 100644 --- a/docs/reference/components/sources/aws_kinesis_firehose.cue +++ b/docs/reference/components/sources/aws_kinesis_firehose.cue @@ -187,4 +187,9 @@ components: sources: aws_kinesis_firehose: { """ } } + + telemetry: metrics: { + vector_request_read_errors_total: _vector_request_read_errors_total + vector_requests_received_total: _vector_requests_received_total + } } diff --git a/docs/reference/components/sources/docker_logs.cue b/docs/reference/components/sources/docker_logs.cue index f8fd9c258a5f0..9f13e29d07b46 100644 --- a/docs/reference/components/sources/docker_logs.cue +++ b/docs/reference/components/sources/docker_logs.cue @@ -265,4 +265,12 @@ components: sources: docker_logs: { """ } } + + telemetry: metrics: { + vector_communication_errors_total: _vector_communication_errors_total + vector_container_events_processed_total: _vector_container_events_processed_total + vector_container_metadata_fetch_errors_total: _vector_container_metadata_fetch_errors_total + vector_containers_watched_total: _vector_containers_watched_total + vector_logging_driver_errors_total: _vector_logging_driver_errors_total + } } diff --git a/docs/reference/components/sources/file.cue b/docs/reference/components/sources/file.cue index 670c8e9ccbbf6..705fcafd0f8a8 100644 --- a/docs/reference/components/sources/file.cue +++ b/docs/reference/components/sources/file.cue @@ -515,4 +515,17 @@ components: sources: file: { """ } } + + telemetry: metrics: { + vector_checkpoint_write_errors_total: _vector_checkpoint_write_errors_total + vector_checkpoints_total: _vector_checkpoints_total + vector_checksum_errors: _vector_checksum_errors + vector_file_delete_errors: _vector_file_delete_errors + vector_file_watch_errors: _vector_file_watch_errors + vector_files_added: _vector_files_added + vector_files_deleted: _vector_files_deleted + vector_files_resumed: _vector_files_resumed + vector_files_unwatched: _vector_files_unwatched + vector_fingerprint_read_errors: _vector_fingerprint_read_errors + } } diff --git a/docs/reference/components/sources/host_metrics.cue b/docs/reference/components/sources/host_metrics.cue index b2d6f78fb1b80..2fc5efbdf0640 100644 --- a/docs/reference/components/sources/host_metrics.cue +++ b/docs/reference/components/sources/host_metrics.cue @@ -260,134 +260,5 @@ components: sources: host_metrics: { } } - output: metrics: { - _tags: { - host: { - description: "The hostname of the originating system." - required: true - examples: [_values.local_host] - } - collector: { - description: "Which collector this metric comes from." - required: true - } - } - - cpu_seconds_total: { - description: "The number of CPU seconds accumulated in different operating modes." - type: "counter" - tags: _tags & { - collector: examples: ["cpu"] - cpu: { - description: "The index of the CPU core or socket." - required: true - examples: ["1"] - } - mode: { - description: "Which mode the CPU was running in during the given time." - required: true - examples: ["idle", "system", "user", "nice"] - } - } - } - - _disk_device: { - description: "The disk device name." - required: true - examples: ["sda", "sda1", "dm-1"] - } - _disk_counter: { - type: "counter" - tags: _tags & { - collector: examples: ["disk"] - device: _disk_device - } - } - disk_read_bytes_total: _disk_counter & {description: "The accumulated number of bytes read in."} - disk_reads_completed_total: _disk_counter & {description: "The accumulated number of read operations completed."} - disk_written_bytes_total: _disk_counter & {description: "The accumulated number of bytes written out."} - disk_writes_completed_total: _disk_counter & {description: "The accumulated number of write operations completed."} - - _filesystem_bytes: { - type: "gauge" - tags: _tags & { - collector: examples: ["filesystem"] - device: _disk_device - filesystem: { - description: "The name of the filesystem type." - required: true - examples: ["ext4", "ntfs"] - } - } - } - filesystem_free_bytes: _filesystem_bytes & {description: "The number of bytes free on the named filesystem."} - filesystem_total_bytes: _filesystem_bytes & {description: "The total number of bytes in the named filesystem."} - filesystem_used_bytes: _filesystem_bytes & {description: "The number of bytes used on the named filesystem."} - - _memory_gauge: { - type: "gauge" - tags: _tags & { - collector: examples: ["memory"] - } - } - _memory_counter: { - type: "counter" - tags: _tags & { - collector: examples: ["memory"] - } - } - _memory_linux: _memory_gauge & {relevant_when: "OS is Linux"} - _memory_macos: _memory_gauge & {relevant_when: "OS is MacOS X"} - _memory_nowin: {relevant_when: "OS is not Windows"} - memory_free_bytes: _memory_gauge & {description: "The number of bytes of main memory not used."} - memory_available_bytes: _memory_gauge & {description: "The number of bytes of main memory available."} - memory_swap_free_bytes: _memory_gauge & {description: "The number of free bytes of swap space."} - memory_swap_total_bytes: _memory_gauge & {description: "The total number of bytes of swap space."} - memory_swap_used_bytes: _memory_gauge & {description: "The number of used bytes of swap space."} - memory_total_bytes: _memory_gauge & {description: "The total number of bytes of main memory."} - memory_active_bytes: _memory_gauge & _memory_nowin & {description: "The number of bytes of active main memory."} - memory_buffers_bytes: _memory_linux & {description: "The number of bytes of main memory used by buffers."} - memory_cached_bytes: _memory_linux & {description: "The number of bytes of main memory used by cached blocks."} - memory_shared_bytes: _memory_linux & {description: "The number of bytes of main memory shared between processes."} - memory_used_bytes: _memory_linux & {description: "The number of bytes of main memory used by programs or caches."} - memory_inactive_bytes: _memory_macos & {description: "The number of bytes of main memory that is not active."} - memory_swapped_in_bytes_total: _memory_counter & _memory_nowin & { - description: "The number of bytes that have been swapped in to main memory." - } - memory_swapped_out_bytes_total: _memory_counter & _memory_nowin & { - description: "The number of bytes that have been swapped out from main memory." - } - memory_wired_bytes: _memory_macos & {description: "The number of wired bytes of main memory."} - - _loadavg: { - type: "gauge" - tags: _tags & { - collector: examples: ["loadavg"] - } - relevant_when: "OS is not Windows" - } - load1: _loadavg & {description: "System load averaged over the last 1 second."} - load5: _loadavg & {description: "System load averaged over the last 5 seconds."} - load15: _loadavg & {description: "System load averaged over the last 15 seconds."} - - _network_gauge: { - type: "gauge" - tags: _tags & { - collector: examples: ["network"] - device: { - description: "The network interface device name." - required: true - examples: ["eth0", "enp5s3"] - } - } - } - _network_nomac: _network_gauge & {relevant_when: "OS is not MacOS"} - network_receive_bytes_total: _network_gauge & {description: "The number of bytes received on this interface."} - network_receive_errs_total: _network_gauge & {description: "The number of errors encountered during receives on this interface."} - network_receive_packets_total: _network_gauge & {description: "The number of packets received on this interface."} - network_transmit_bytes_total: _network_gauge & {description: "The number of bytes transmitted on this interface."} - network_transmit_errs_total: _network_gauge & {description: "The number of errors encountered during transmits on this interface."} - network_transmit_packets_drop_total: _network_nomac & {description: "The number of packets dropped during transmits on this interface."} - network_transmit_packets_total: _network_nomac & {description: "The number of packets transmitted on this interface."} - } + output: metrics: _host_metrics } diff --git a/docs/reference/components/sources/http.cue b/docs/reference/components/sources/http.cue index 7367d7fdb9f7a..1ec388069d241 100644 --- a/docs/reference/components/sources/http.cue +++ b/docs/reference/components/sources/http.cue @@ -206,4 +206,8 @@ components: sources: http: { }] }, ] + + telemetry: metrics: { + vector_http_bad_requests_total: _vector_http_bad_requests_total + } } diff --git a/docs/reference/components/sources/journald.cue b/docs/reference/components/sources/journald.cue index 1344b9de4b552..fb6a78f6d03b1 100644 --- a/docs/reference/components/sources/journald.cue +++ b/docs/reference/components/sources/journald.cue @@ -189,4 +189,17 @@ components: sources: journald: { """ } } + + telemetry: metrics: { + vector_invalid_record_total: { + description: "The total number of invalid journald records discarded." + type: "counter" + tags: _component_tags + } + vector_invalid_record_bytes_total: { + description: "The total number of bytes from discarded journald records." + type: "counter" + tags: _component_tags + } + } } diff --git a/docs/reference/components/sources/kubernetes_logs.cue b/docs/reference/components/sources/kubernetes_logs.cue index 9fd9ffb100b6f..f4ef610e61d92 100644 --- a/docs/reference/components/sources/kubernetes_logs.cue +++ b/docs/reference/components/sources/kubernetes_logs.cue @@ -299,4 +299,9 @@ components: sources: kubernetes_logs: { """ } } + + telemetry: metrics: { + vector_k8s_docker_format_parse_failures_total: _vector_k8s_docker_format_parse_failures_total + vector_k8s_event_annotation_failures_total: _vector_k8s_event_annotation_failures_total + } } diff --git a/docs/reference/components/sources/logplex.cue b/docs/reference/components/sources/logplex.cue index 43178d79387ae..0864d65aeeeb5 100644 --- a/docs/reference/components/sources/logplex.cue +++ b/docs/reference/components/sources/logplex.cue @@ -97,4 +97,9 @@ components: sources: logplex: { timestamp: fields._current_timestamp } } + + telemetry: metrics: { + vector_request_read_errors_total: _vector_request_read_errors_total + vector_requests_received_total: _vector_requests_received_total + } } diff --git a/docs/reference/components/sources/mongodb_metrics.cue b/docs/reference/components/sources/mongodb_metrics.cue index 48250358e2ab5..6dae5bda2e6ad 100644 --- a/docs/reference/components/sources/mongodb_metrics.cue +++ b/docs/reference/components/sources/mongodb_metrics.cue @@ -80,732 +80,10 @@ components: sources: mongodb_metrics: { } } - output: metrics: { - _endpoint: { - description: "The absolute path of originating file." - required: true - examples: ["mongodb://localhost:27017"] - } - _host: { - description: "The hostname of the MongoDB server" - required: true - examples: [_values.local_host] - } - up: { - description: "If the MongoDB server is up or not." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - } - } - assets_total: { - description: "Number of assertions raised since the MongoDB process started." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "The assertion type" - required: true - examples: ["regular", "warning", "msg", "user", "rollovers"] - } - } - } - connections: { - description: "Number of connections in some state." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - state: { - description: "The connection state" - required: true - examples: ["active", "available", "current"] - } - } - } - extra_info_heap_usage_bytes: { - description: "The total size in bytes of heap space used by the database process." - relevant_when: "Unix/Linux" - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - } - } - extra_info_page_faults: { - description: "The total number of page faults." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - } - } - instance_local_time: { - description: "The ISODate representing the current time, according to the server, in UTC." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - } - } - instance_uptime_estimate_seconds_total: { - description: "The uptime in seconds as calculated from MongoDB’s internal course-grained time keeping system." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - } - } - instance_uptime_seconds_total: { - description: "The number of seconds that the current MongoDB process has been active." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - } - } - memory: { - description: "Current memory unsage." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Memory type" - required: true - examples: ["resident", "virtual", "mapped", "mapped_with_journal"] - } - } - } - mongod_global_lock_total_time_seconds: { - description: "The time since the database last started and created the globalLock. This is roughly equivalent to total server uptime." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_global_lock_active_clients: { - description: "Number of connected clients and the read and write operations performed by these clients." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Number type." - required: true - examples: ["total", "readers", "writers"] - } - } - } - mongod_global_lock_current_queue: { - description: "Number of operations queued because of a lock." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Number type." - required: true - examples: ["total", "readers", "writers"] - } - } - } - mongod_locks_time_acquiring_global_seconds_total: { - description: "Amount of time that any database has spent waiting for the global lock." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Lock type." - required: true - examples: ["ParallelBatchWriterMode", "ReplicationStateTransition", "Global", "Database", "Collection", "Mutex", "Metadata", "oplog"] - } - mode: { - description: "Lock mode." - required: true - examples: ["read", "write"] - } - } - } - mongod_metrics_cursor_timed_out_total: { - description: "The total number of cursors that have timed out since the server process started." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_metrics_cursor_open: { - description: "Number of cursors." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - state: { - description: "Cursor state." - required: true - examples: ["no_timeout", "pinned", "total"] - } - } - } - mongod_metrics_document_total: { - description: "Document access and modification patterns." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - state: { - description: "Document state." - required: true - examples: ["deleted", "inserted", "returned", "updated"] - } - } - } - mongod_metrics_get_last_error_wtime_num: { - description: "The total number of getLastError operations with a specified write concern." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_metrics_get_last_error_wtime_seconds_total: { - description: "The total amount of time that the mongod has spent performing getLastError operations." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_metrics_get_last_error_wtimeouts_total: { - description: "The number of times that write concern operations have timed out as a result of the wtimeout threshold to getLastError." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_metrics_operation_total: { - description: "Update and query operations that MongoDB handles using special operation types." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Operation type." - required: true - examples: ["scan_and_order", "write_conflicts"] - } - } - } - mongod_metrics_query_executor_total: { - description: "Data from query execution system." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - state: { - description: "Query state." - required: true - examples: ["scanned", "scanned_objects", "collection_scans"] - } - } - } - mongod_metrics_record_moves_total: { - description: "Moves reports the total number of times documents move within the on-disk representation of the MongoDB data set. Documents move as a result of operations that increase the size of the document beyond their allocated record size." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_metrics_repl_apply_batches_num_total: { - description: "The total number of batches applied across all databases." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_metrics_repl_apply_batches_seconds_total: { - description: "The total amount of time the mongod has spent applying operations from the oplog." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_metrics_repl_apply_ops_total: { - description: "The total number of oplog operations applied." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_metrics_repl_buffer_count: { - description: "The current number of operations in the oplog buffer." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_metrics_repl_buffer_max_size_bytes_total: { - description: "The maximum size of the buffer." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_metrics_repl_buffer_size_bytes: { - description: "The current size of the contents of the oplog buffer." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_metrics_repl_executor_queue: { - description: "Number of queued operations in the replication executor." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Queue type." - required: true - examples: ["network_in_progress", "sleepers"] - } - } - } - mongod_metrics_repl_executor_unsignaled_events: { - description: "Number of unsignaled events in the replication executor." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_metrics_repl_network_bytes_total: { - description: "The total amount of data read from the replication sync source." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_metrics_repl_network_getmores_num_total: { - description: "The total number of getmore operations, which are operations that request an additional set of operations from the replication sync source." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_metrics_repl_network_getmores_seconds_total: { - description: "The total amount of time required to collect data from getmore operations." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_metrics_repl_network_ops_total: { - description: "The total number of operations read from the replication source." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_metrics_repl_network_readers_created_total: { - description: "The total number of oplog query processes created." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_metrics_ttl_deleted_documents_total: { - description: "The total number of documents deleted from collections with a ttl index." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_metrics_ttl_passes_total: { - description: "The number of times the background process removes documents from collections with a ttl index." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_op_latencies_histogram: { - description: "Latency statistics." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Latency type." - required: true - examples: ["reads", "writes", "commands"] - } - micros: { - description: "Bucket." - required: true - examples: ["1", "2", "4096", "16384", "49152"] - } - } - } - mongod_op_latencies_latency: { - description: "A 64-bit integer giving the total combined latency in microseconds." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Latency type." - required: true - examples: ["network_in_progress", "sleepers"] - } - } - } - mongod_op_latencies_ops_total: { - description: "A 64-bit integer giving the total number of operations performed on the collection since startup." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Latency type." - required: true - examples: ["network_in_progress", "sleepers"] - } - } - } - mongod_storage_engine: { - description: "The name of the current storage engine." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - engine: { - description: "Engine name." - required: true - examples: ["wiredTiger"] - } - } - } - mongod_wiredtiger_blockmanager_blocks_total: { - description: "Statistics on the block manager operations." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Operation type." - required: true - examples: ["blocks_read", "blocks_read_mapped", "blocks_pre_loaded", "blocks_written"] - } - } - } - mongod_wiredtiger_blockmanager_bytes_total: { - description: "Statistics on the block manager operations." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Operation type." - required: true - examples: ["bytes_read", "bytes_read_mapped", "bytes_written"] - } - } - } - mongod_wiredtiger_cache_bytes: { - description: "Statistics on the cache and page evictions from the cache." - relevant_when: "Storage engine is `wiredTiger`." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Statistics type." - required: true - examples: ["total", "dirty", "internal_pages", "leaf_pages"] - } - } - } - mongod_wiredtiger_cache_bytes_total: { - description: "Statistics on the cache and page evictions from the cache." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Statistics type." - required: true - examples: ["read", "written"] - } - } - } - mongod_wiredtiger_cache_evicted_total: { - description: "Statistics on the cache and page evictions from the cache." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Statistics type." - required: true - examples: ["modified", "unmodified"] - } - } - } - mongod_wiredtiger_cache_max_bytes: { - description: "Maximum cache size." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_wiredtiger_cache_overhead_percent: { - description: "Percentage overhead." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_wiredtiger_cache_pages: { - description: "Pages in the cache." - relevant_when: "Storage engine is `wiredTiger`." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Pages type." - required: true - examples: ["total", "dirty"] - } - } - } - mongod_wiredtiger_cache_pages_total: { - description: "Pages in the cache." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Pages type." - required: true - examples: ["read", "write"] - } - } - } - mongod_wiredtiger_concurrent_transactions_available_tickets: { - description: "Information on the number of concurrent of read and write transactions allowed into the WiredTiger storage engine" - relevant_when: "Storage engine is `wiredTiger`." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Transactions type." - required: true - examples: ["read", "write"] - } - } - } - mongod_wiredtiger_concurrent_transactions_out_tickets: { - description: "Information on the number of concurrent of read and write transactions allowed into the WiredTiger storage engine" - relevant_when: "Storage engine is `wiredTiger`." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Transactions type." - required: true - examples: ["read", "write"] - } - } - } - mongod_wiredtiger_concurrent_transactions_total_tickets: { - description: "Information on the number of concurrent of read and write transactions allowed into the WiredTiger storage engine" - relevant_when: "Storage engine is `wiredTiger`." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Transactions type." - required: true - examples: ["read", "write"] - } - } - } - mongod_wiredtiger_log_bytes_total: { - description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Bytes type." - required: true - examples: ["payload", "written"] - } - } - } - mongod_wiredtiger_log_operations_total: { - description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Operations type." - required: true - examples: ["write", "scan", "scan_double", "sync", "sync_dir", "flush"] - } - } - } - mongod_wiredtiger_log_records_scanned_total: { - description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Scanned records type." - required: true - examples: ["compressed", "uncompressed"] - } - } - } - mongod_wiredtiger_log_records_total: { - description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_wiredtiger_session_open_sessions: { - description: "Open session count." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - mongod_wiredtiger_transactions_checkpoint_seconds: { - description: "Statistics on transaction checkpoints and operations." - relevant_when: "Storage engine is `wiredTiger`." - type: "gauge" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Checkpoint type." - required: true - examples: ["min", "max"] - } - } - } - mongod_wiredtiger_transactions_checkpoint_seconds_total: { - description: "Statistics on transaction checkpoints and operations." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - } - mongod_wiredtiger_transactions_running_checkpoints: { - description: "Statistics on transaction checkpoints and operations." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - } - mongod_wiredtiger_transactions_total: { - description: "Statistics on transaction checkpoints and operations." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Transactions type." - required: true - examples: ["begins", "checkpoints", "committed", "rolledback"] - } - } - } - network_bytes_total: { - description: "The number of bytes that reflects the amount of network traffic." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - state: { - description: "Bytes state." - required: true - examples: ["bytes_in", "bytes_out"] - } - } - } - network_metrics_num_requests_total: { - description: "The total number of distinct requests that the server has received." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - } - } - op_counters_repl_total: { - description: "Database replication operations by type since the mongod instance last started." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Counter type." - required: true - examples: ["insert", "query", "update", "delete", "getmore", "command"] - } - } - } - op_counters_total: { - description: "Database operations by type since the mongod instance last started." - type: "counter" - tags: { - endpoint: _endpoint - host: _host - type: { - description: "Counter type." - required: true - examples: ["insert", "query", "update", "delete", "getmore", "command"] - } - } - } + output: metrics: _mongodb_metrics & { + vector_collect_completed_total: _vector_collect_completed_total + vector_collect_duration_nanoseconds: _vector_collect_duration_nanoseconds + vector_request_error_total: _vector_request_error_total } how_it_works: { diff --git a/docs/reference/components/sources/prometheus.cue b/docs/reference/components/sources/prometheus.cue index 0c88dc84fd9a8..04a6bb3575f56 100644 --- a/docs/reference/components/sources/prometheus.cue +++ b/docs/reference/components/sources/prometheus.cue @@ -88,4 +88,6 @@ components: sources: prometheus: { histogram: output._passthrough_histogram summary: output._passthrough_summary } + + telemetry: metrics: _prometheus_metrics } diff --git a/docs/reference/components/sources/socket.cue b/docs/reference/components/sources/socket.cue index 54b6b9629c76b..bab3039b7e381 100644 --- a/docs/reference/components/sources/socket.cue +++ b/docs/reference/components/sources/socket.cue @@ -147,5 +147,10 @@ components: sources: socket: { message: _line host: _values.local_host } - }] + }, + ] + + telemetry: metrics: { + vector_connection_errors_total: _vector_connection_errors_total + } } diff --git a/docs/reference/components/sources/splunk_hec.cue b/docs/reference/components/sources/splunk_hec.cue index 1e92294a52836..d27205c2bb619 100644 --- a/docs/reference/components/sources/splunk_hec.cue +++ b/docs/reference/components/sources/splunk_hec.cue @@ -92,4 +92,12 @@ components: sources: splunk_hec: { timestamp: fields._current_timestamp } } + + telemetry: metrics: { + vector_encode_errors_total: _vector_encode_errors_total + vector_http_request_errors_total: _vector_http_request_errors_total + vector_requests_received_total: _vector_requests_received_total + vector_source_missing_keys_total: _vector_source_missing_keys_total + vector_sourcetype_missing_keys_total: _vector_sourcetype_missing_keys_total + } } diff --git a/docs/reference/components/sources/statsd.cue b/docs/reference/components/sources/statsd.cue index 54746f750db11..387c6f3004207 100644 --- a/docs/reference/components/sources/statsd.cue +++ b/docs/reference/components/sources/statsd.cue @@ -120,4 +120,18 @@ components: sources: statsd: { """ } } + + telemetry: metrics: { + vector_connection_errors_total: _vector_connection_errors_total + vector_invalid_record_total: { + description: "The total number of invalid StatsD records discarded." + type: "counter" + tags: _component_tags + } + vector_invalid_record_bytes_total: { + description: "The total number of bytes from StatsD journald records." + type: "counter" + tags: _component_tags + } + } } diff --git a/docs/reference/components/sources/stdin.cue b/docs/reference/components/sources/stdin.cue index 22fa8c3041cd9..b75974115b082 100644 --- a/docs/reference/components/sources/stdin.cue +++ b/docs/reference/components/sources/stdin.cue @@ -100,4 +100,12 @@ components: sources: stdin: { """ } } + + telemetry: metrics: { + vector_stdin_reads_failed_total: { + description: "The total number of errors reading from stdin." + type: "counter" + tags: _component_tags + } + } } diff --git a/docs/reference/components/sources/syslog.cue b/docs/reference/components/sources/syslog.cue index 6fba35e8eed00..4a58db6d9dbe7 100644 --- a/docs/reference/components/sources/syslog.cue +++ b/docs/reference/components/sources/syslog.cue @@ -182,4 +182,33 @@ components: sources: syslog: { """ } } + + telemetry: metrics: { + vector_connection_read_errors_total: { + description: "TODO" + type: "counter" + tags: _component_tags & { + mode: { + description: "" + required: true + options: { + udp: "User Datagram Protocol" + } + } + } + } + vector_utf8_convert_errors_total: { + description: "TODO" + type: "counter" + tags: _component_tags & { + mode: { + description: "" + required: true + options: { + udp: "User Datagram Protocol" + } + } + } + } + } } diff --git a/docs/reference/components/sources/vector.cue b/docs/reference/components/sources/vector.cue index b64c8c92633ab..bb5bbdb5f0954 100644 --- a/docs/reference/components/sources/vector.cue +++ b/docs/reference/components/sources/vector.cue @@ -110,4 +110,8 @@ components: sources: vector: { } } + + telemetry: metrics: { + vector_protobuf_decode_errors_total: _vector_protobuf_decode_errors_total + } } diff --git a/docs/reference/metrics.cue b/docs/reference/metrics.cue index 59d3530a7d754..4ccc096f27b17 100644 --- a/docs/reference/metrics.cue +++ b/docs/reference/metrics.cue @@ -1,16 +1,5 @@ package metadata -// Helpful metrics groupings -_prometheus_metrics: { - vector_events_processed_total: _vector_events_processed_total - vector_http_error_response_total: _vector_http_error_response_total - vector_http_request_errors_total: _vector_http_request_errors_total - vector_parse_errors_total: _vector_parse_errors_total - vector_processed_bytes_total: _vector_processed_bytes_total - vector_request_duration_nanoseconds: _vector_request_duration_nanoseconds - vector_requests_completed_total: _vector_requests_completed_total -} - // Apache metrics _apache_access_total: { description: "The total number of time the Apache server has been accessed." @@ -93,37 +82,37 @@ _apache_workers: { } // Container metrics -_communication_errors_total: { +_vector_communication_errors_total: { description: "The total number of errors stemming from communication with the Docker daemon." type: "counter" tags: _component_tags } -_container_events_processed_total: { +_vector_container_events_processed_total: { description: "The total number of container events processed." type: "counter" tags: _component_tags } -_container_metadata_fetch_errors_total: { +_vector_container_metadata_fetch_errors_total: { description: "The total number of errors caused by failure to fetch container metadata." type: "counter" tags: _component_tags } -_containers_unwatched_total: { +_vector_containers_unwatched_total: { description: "The total number of times Vector stopped watching for container logs." - counter: "counter" + type: "counter" tags: _component_tags } -_containers_watched_total: { +_vector_containers_watched_total: { description: "The total number of times Vector started watching for container logs." - counter: "counter" + type: "counter" tags: _component_tags } -_logging_driver_errors_total: { +_vector_logging_driver_errors_total: { description: "The total number of logging driver errors encountered caused by not using either the `jsonfile` or `journald` driver." type: "counter" tags: _component_tags @@ -196,25 +185,19 @@ _host_network_transmit_packets_drop_total: _network_nomac & {description: "The n _host_network_transmit_packets_total: _network_nomac & {description: "The number of packets transmitted on this interface."} // Kubernetes metrics -_k8s_docker_format_parse_failures_total: { +_vector_k8s_docker_format_parse_failures_total: { description: "The total number of failures to parse a message as a JSON object." type: "counter" tags: _component_tags } -_k8s_event_annotation_failures_total: { +_vector_k8s_event_annotation_failures_total: { description: "The total number of failures to annotate Vector events with Kubernetes Pod metadata." type: "counter" tags: _component_tags } // MongoDB metrics -_mongodb_bson_parse_error_total: { - description: "The total number of BSON parsing errors." - type: "counter" - tags: _mongodb_metrics_tags -} - _mongodb_assets_total: { description: "Number of assertions raised since the MongoDB process started." type: "counter" @@ -226,6 +209,11 @@ _mongodb_assets_total: { } } } +_mongodb_bson_parse_error_total: { + description: "The total number of BSON parsing errors." + type: "counter" + tags: _mongodb_metrics_tags +} _mongodb_connections: { description: "Number of connections in some state." type: "gauge" @@ -829,8 +817,8 @@ _vector_collect_completed_total: { } _vector_encode_errors_total: { description: "TODO" - type: "counter" - tags: _internal_metrics_tags + type: "counter" + tags: _internal_metrics_tags } _vector_events_discarded_total: { description: "The total number of events discarded by this component." @@ -947,8 +935,8 @@ _vector_processing_errors_total: { } _vector_protobuf_decode_errors_total: { description: "TODO" - type: "counter" - tags: _component_tags + type: "counter" + tags: _component_tags } _vector_request_duration_nanoseconds: { description: "TODO" @@ -965,6 +953,11 @@ _vector_requests_completed_total: { type: "counter" tags: _component_tags } +_vector_requests_received_total: { + description: "TODO" + type: "counter" + tags: _component_tags +} _vector_timestamp_parse_errors_total: { description: "The total number of errors encountered RFC3339 parsing timestamps." type: "counter" @@ -976,50 +969,17 @@ _vector_uptime_seconds: { tags: _component_tags } -// Convenient groupings of tags -_component_tags: _internal_metrics_tags & { - component_kind: _component_kind - component_name: _component_name - component_type: _component_type - instance: _instance - job: _job -} - -_apache_metrics_tags: { - endpoint: _endpoint - host: { - description: "The hostname of the Apache HTTP server." - required: true - examples: [_values.local_host] - } -} -_host_metrics_tags: { - collector: _collector - host: _host -} -_internal_metrics_tags: { - instance: _instance - job: _job -} -_mongodb_metrics_tags: { - endpoint: { - description: "The absolute path of the originating file." - required: true - examples: ["mongodb://localhost:27017"] - } - host: { - description: "The hostname of the MongoDB server." - required: true - examples: [_values.local_host] - } -} - // Vector instance metrics _vector_config_load_errors_total: { description: "The total number of errors loading the Vector configuration." type: "counter" tags: _internal_metrics_tags } +_vector_connection_errors_total: { + description: "TODO" + type: "counter" + tags: _internal_metrics_tags +} _vector_quit_total: { description: "The total number of times the Vector instance has quit." type: "counter" @@ -1040,15 +1000,20 @@ _vector_reloaded_total: { type: "counter" tags: _internal_metrics_tags } +_vector_request_error_total: { + description: "TODO" + type: "counter" + tags: _internal_metrics_tags +} _vector_source_missing_keys_total: { description: "TODO" - type: "counter" - tags: _internal_metrics_tags + type: "counter" + tags: _internal_metrics_tags } _vector_sourcetype_missing_keys_total: { description: "TODO" - type: "counter" - tags: _internal_metrics_tags + type: "counter" + tags: _internal_metrics_tags } _vector_started_total: { description: "The total number of times the Vector instance has been started." @@ -1165,6 +1130,44 @@ _job: { default: "vector" } +// Convenient groupings of tags +_component_tags: _internal_metrics_tags & { + component_kind: _component_kind + component_name: _component_name + component_type: _component_type + instance: _instance + job: _job +} + +_apache_metrics_tags: { + endpoint: _endpoint + host: { + description: "The hostname of the Apache HTTP server." + required: true + examples: [_values.local_host] + } +} +_host_metrics_tags: { + collector: _collector + host: _host +} +_internal_metrics_tags: { + instance: _instance + job: _job +} +_mongodb_metrics_tags: { + endpoint: { + description: "The absolute path of the originating file." + required: true + examples: ["mongodb://localhost:27017"] + } + host: { + description: "The hostname of the MongoDB server." + required: true + examples: [_values.local_host] + } +} + // Other helpers _disk_device: { description: "The disk device name." @@ -1224,3 +1227,133 @@ _network_gauge: { } } _network_nomac: _network_gauge & {relevant_when: "OS is not MacOS"} + +// Helpful metrics groupings +_apache_metrics: { + apache_access_total: _apache_access_total + apache_connections: _apache_connections + apache_cpu_load: _apache_cpu_load + apache_cpu_seconds_total: _apache_cpu_seconds_total + apache_duration_seconds_total: _apache_duration_seconds_total + apache_scoreboard: _apache_scoreboard + apache_sent_bytes_total: _apache_sent_bytes_total + apache_sent_bytes_total: _apache_sent_bytes_total + apache_up: _apache_up + apache_uptime_seconds_total: _apache_uptime_seconds_total + apache_workers: _apache_workers +} + +_host_metrics: { + host_cpu_seconds_total: _host_cpu_seconds_total + host_disk_read_bytes_total: _host_disk_read_bytes_total + host_disk_reads_completed_total: _host_disk_reads_completed_total + host_disk_written_bytes_total: _host_disk_written_bytes_total + host_disk_writes_completed_total: _host_disk_writes_completed_total + host_filesystem_free_bytes: _host_filesystem_free_bytes + host_filesystem_total_bytes: _host_filesystem_total_bytes + host_filesystem_used_bytes: _host_filesystem_used_bytes + host_load1: _host_load1 + host_load5: _host_load5 + host_load15: _host_load15 + host_memory_active_bytes: _host_memory_active_bytes + host_memory_available_bytes: _host_memory_available_bytes + host_memory_buffers_bytes: _host_memory_buffers_bytes + host_memory_cached_bytes: _host_memory_cached_bytes + host_memory_free_bytes: _host_memory_free_bytes + host_memory_inactive_bytes: _host_memory_inactive_bytes + host_memory_shared_bytes: _host_memory_shared_bytes + host_memory_swap_free_bytes: _host_memory_swap_free_bytes + host_memory_swapped_in_bytes_total: _host_memory_swapped_in_bytes_total + host_memory_swapped_out_bytes_total: _host_memory_swapped_out_bytes_total + host_memory_swap_total_bytes: _host_memory_swap_total_bytes + host_memory_swap_used_bytes: _host_memory_swap_used_bytes + host_memory_total_bytes: _host_memory_total_bytes + host_memory_used_bytes: _host_memory_used_bytes + host_memory_wired_bytes: _host_memory_wired_bytes + host_network_receive_bytes_total: _host_network_receive_bytes_total + host_network_receive_errs_total: _host_network_receive_errs_total + host_network_receive_packets_total: _host_network_receive_packets_total + host_network_transmit_bytes_total: _host_network_transmit_bytes_total + host_network_transmit_errs_total: _host_network_transmit_errs_total + host_network_transmit_packets_drop_total: _host_network_transmit_packets_drop_total + host_network_transmit_packets_total: _host_network_transmit_packets_total +} + +_mongodb_metrics: { + mongodb_assets_total: _mongodb_assets_total + mongodb_bson_parse_error_total: _mongodb_bson_parse_error_total + mongodb_connections: _mongodb_connections + mongodb_extra_info_heap_usage_bytes: _mongodb_extra_info_heap_usage_bytes + mongodb_extra_info_page_faults: _mongodb_extra_info_page_faults + mongodb_instance_local_time: _mongodb_instance_local_time + mongodb_instance_uptime_estimate_seconds_total: _mongodb_instance_uptime_estimate_seconds_total + mongodb_instance_uptime_seconds_total: _mongodb_instance_uptime_seconds_total + mongodb_memory: _mongodb_memory + mongodb_mongod_global_lock_active_clients: _mongodb_mongod_global_lock_active_clients + mongodb_mongod_global_lock_current_queue: _mongodb_mongod_global_lock_current_queue + mongodb_mongod_locks_time_acquiring_global_seconds_total: _mongodb_mongod_locks_time_acquiring_global_seconds_total + mongodb_mongod_metrics_cursor_open: _mongodb_mongod_metrics_cursor_open + mongodb_mongod_metrics_cursor_timed_out_total: _mongodb_mongod_metrics_cursor_timed_out_total + mongodb_mongod_metrics_document_total: _mongodb_mongod_metrics_document_total + mongodb_mongod_metrics_get_last_error_wtime_num: _mongodb_mongod_metrics_get_last_error_wtime_num + mongodb_mongod_metrics_get_last_error_wtime_seconds_total: _mongodb_mongod_metrics_get_last_error_wtime_seconds_total + mongodb_mongod_metrics_get_last_error_wtimeouts_total: _mongodb_mongod_metrics_get_last_error_wtimeouts_total + mongodb_mongod_metrics_operation_total: _mongodb_mongod_metrics_operation_total + mongodb_mongod_metrics_query_executor_total: _mongodb_mongod_metrics_query_executor_total + mongodb_mongod_metrics_record_moves_total: _mongodb_mongod_metrics_record_moves_total + mongodb_mongod_metrics_repl_apply_batches_num_total: _mongodb_mongod_metrics_repl_apply_batches_num_total + mongodb_mongod_metrics_repl_apply_batches_seconds_total: _mongodb_mongod_metrics_repl_apply_batches_seconds_total + mongodb_mongod_metrics_repl_apply_ops_total: _mongodb_mongod_metrics_repl_apply_ops_total + mongodb_mongod_metrics_repl_buffer_count: _mongodb_mongod_metrics_repl_buffer_count + mongodb_mongod_metrics_repl_buffer_max_size_bytes_total: _mongodb_mongod_metrics_repl_buffer_max_size_bytes_total + mongodb_mongod_metrics_repl_buffer_size_bytes: _mongodb_mongod_metrics_repl_buffer_size_bytes + mongodb_mongod_metrics_repl_executor_queue: _mongodb_mongod_metrics_repl_executor_queue + mongodb_mongod_metrics_repl_executor_unsignaled_events: _mongodb_mongod_metrics_repl_executor_unsignaled_events + mongodb_mongod_metrics_repl_network_bytes_total: _mongodb_mongod_metrics_repl_network_bytes_total + mongodb_mongod_metrics_repl_network_getmores_num_total: _mongodb_mongod_metrics_repl_network_getmores_num_total + mongodb_mongod_metrics_repl_network_getmores_seconds_total: _mongodb_mongod_metrics_repl_network_getmores_seconds_total + mongodb_mongod_metrics_repl_network_ops_total: _mongodb_mongod_metrics_repl_network_ops_total + mongodb_mongod_metrics_repl_network_readers_created_total: _mongodb_mongod_metrics_repl_network_readers_created_total + mongodb_mongod_metrics_ttl_deleted_documents_total: _mongodb_mongod_metrics_ttl_deleted_documents_total + mongodb_mongod_metrics_ttl_passes_total: _mongodb_mongod_metrics_ttl_passes_total + mongodb_mongod_op_latencies_histogram: _mongodb_mongod_op_latencies_histogram + mongodb_mongod_op_latencies_latency: _mongodb_mongod_op_latencies_latency + mongodb_mongod_op_latencies_ops_total: _mongodb_mongod_op_latencies_ops_total + mongodb_mongod_storage_engine: _mongodb_mongod_storage_engine + mongodb_mongod_wiredtiger_blockmanager_blocks_total: _mongodb_mongod_wiredtiger_blockmanager_blocks_total + mongodb_mongod_wiredtiger_blockmanager_bytes_total: _mongodb_mongod_wiredtiger_blockmanager_bytes_total + mongodb_mongod_wiredtiger_cache_bytes: _mongodb_mongod_wiredtiger_cache_bytes + mongodb_mongod_wiredtiger_cache_bytes_total: _mongodb_mongod_wiredtiger_cache_bytes_total + mongodb_mongod_wiredtiger_cache_evicted_total: _mongodb_mongod_wiredtiger_cache_evicted_total + mongodb_mongod_wiredtiger_cache_max_bytes: _mongodb_mongod_wiredtiger_cache_max_bytes + mongodb_mongod_wiredtiger_cache_overhead_percent: _mongodb_mongod_wiredtiger_cache_overhead_percent + mongodb_mongod_wiredtiger_cache_pages: _mongodb_mongod_wiredtiger_cache_pages + mongodb_mongod_wiredtiger_cache_pages_total: _mongodb_mongod_wiredtiger_cache_pages_total + mongodb_mongod_wiredtiger_concurrent_transactions_available_tickets: _mongodb_mongod_wiredtiger_concurrent_transactions_available_tickets + mongodb_mongod_wiredtiger_concurrent_transactions_out_tickets: _mongodb_mongod_wiredtiger_concurrent_transactions_out_tickets + mongodb_mongod_wiredtiger_concurrent_transactions_total_tickets: _mongodb_mongod_wiredtiger_concurrent_transactions_total_tickets + mongodb_mongod_wiredtiger_log_bytes_total: _mongodb_mongod_wiredtiger_log_bytes_total + mongodb_mongod_wiredtiger_log_operations_total: _mongodb_mongod_wiredtiger_log_operations_total + mongodb_mongod_wiredtiger_log_records_scanned_total: _mongodb_mongod_wiredtiger_log_records_scanned_total + mongodb_mongod_wiredtiger_log_records_total: _mongodb_mongod_wiredtiger_log_records_total + mongodb_mongod_wiredtiger_session_open_sessions: _mongodb_mongod_wiredtiger_session_open_sessions + mongodb_mongod_wiredtiger_transactions_checkpoint_seconds: _mongodb_mongod_wiredtiger_transactions_checkpoint_seconds + mongodb_mongod_wiredtiger_transactions_checkpoint_seconds_total: _mongodb_mongod_wiredtiger_transactions_checkpoint_seconds_total + mongodb_mongod_wiredtiger_transactions_running_checkpoints: _mongodb_mongod_wiredtiger_transactions_running_checkpoints + mongodb_mongod_wiredtiger_transactions_total: _mongodb_mongod_wiredtiger_transactions_total + mongodb_network_bytes_total: _mongodb_network_bytes_total + mongodb_network_metrics_num_requests_total: _mongodb_network_metrics_num_requests_total + mongodb_op_counters_repl_total: _mongodb_op_counters_repl_total + mongodb_op_counters_total: _mongodb_op_counters_total + mongodb_up: _mongodb_up +} + +_prometheus_metrics: { + vector_events_processed_total: _vector_events_processed_total + vector_http_error_response_total: _vector_http_error_response_total + vector_http_request_errors_total: _vector_http_request_errors_total + vector_parse_errors_total: _vector_parse_errors_total + vector_processed_bytes_total: _vector_processed_bytes_total + vector_request_duration_nanoseconds: _vector_request_duration_nanoseconds + vector_requests_completed_total: _vector_requests_completed_total +} From 19f126c48f139b11ec3c0185a3d0cb33fa1fcfc6 Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Fri, 6 Nov 2020 22:09:13 -0800 Subject: [PATCH 04/20] Add metrics for transforms Signed-off-by: Luc Perkins --- docs/reference/components/sources/statsd.cue | 4 ++-- .../components/transforms/ansi_stripper.cue | 4 ++++ ...ws_cloudwatch_logs_subscription_parser.cue | 4 ++++ .../transforms/aws_ec2_metadata.cue | 5 +++++ .../components/transforms/dedupe.cue | 4 ++++ .../components/transforms/grok_parser.cue | 4 ++++ .../components/transforms/logfmt_parser.cue | 4 ++++ docs/reference/components/transforms/lua.cue | 5 +++++ .../components/transforms/metric_to_log.cue | 4 ++++ .../components/transforms/reduce.cue | 8 ++++++++ .../components/transforms/regex_parser.cue | 4 ++++ .../reference/components/transforms/remap.cue | 4 ++++ .../components/transforms/sampler.cue | 4 ++++ .../reference/components/transforms/split.cue | 4 ++++ .../components/transforms/swimlanes.cue | 4 ++++ .../transforms/tag_cardinality_limit.cue | 19 +++++++++++++++++++ .../components/transforms/tokenizer.cue | 4 ++++ docs/reference/metrics.cue | 10 ++++++++++ 18 files changed, 97 insertions(+), 2 deletions(-) diff --git a/docs/reference/components/sources/statsd.cue b/docs/reference/components/sources/statsd.cue index 387c6f3004207..2b8ed11554943 100644 --- a/docs/reference/components/sources/statsd.cue +++ b/docs/reference/components/sources/statsd.cue @@ -126,12 +126,12 @@ components: sources: statsd: { vector_invalid_record_total: { description: "The total number of invalid StatsD records discarded." type: "counter" - tags: _component_tags + tags: _component_tags } vector_invalid_record_bytes_total: { description: "The total number of bytes from StatsD journald records." type: "counter" - tags: _component_tags + tags: _component_tags } } } diff --git a/docs/reference/components/transforms/ansi_stripper.cue b/docs/reference/components/transforms/ansi_stripper.cue index 99617617760b3..d4b1181e00059 100644 --- a/docs/reference/components/transforms/ansi_stripper.cue +++ b/docs/reference/components/transforms/ansi_stripper.cue @@ -45,4 +45,8 @@ components: transforms: ansi_stripper: { logs: true metrics: null } + + telemetry: metrics: { + vector_processing_errors_total: _vector_processing_errors_total + } } diff --git a/docs/reference/components/transforms/aws_cloudwatch_logs_subscription_parser.cue b/docs/reference/components/transforms/aws_cloudwatch_logs_subscription_parser.cue index 3923e2fa5fc1f..b0bdcff89713b 100644 --- a/docs/reference/components/transforms/aws_cloudwatch_logs_subscription_parser.cue +++ b/docs/reference/components/transforms/aws_cloudwatch_logs_subscription_parser.cue @@ -142,4 +142,8 @@ components: transforms: aws_cloudwatch_logs_subscription_parser: { body: "Note that the events themselves are not parsed. If they are structured data, you will typically want to pass them through a [parsing transform](\(urls.vector_parsing_transforms))." } } + + telemetry: metrics: { + vector_processing_errors_total: _vector_processing_errors_total + } } diff --git a/docs/reference/components/transforms/aws_ec2_metadata.cue b/docs/reference/components/transforms/aws_ec2_metadata.cue index d4ef44f7b1b75..9792eb4070eed 100644 --- a/docs/reference/components/transforms/aws_ec2_metadata.cue +++ b/docs/reference/components/transforms/aws_ec2_metadata.cue @@ -170,4 +170,9 @@ components: transforms: aws_ec2_metadata: { } } } + + telemetry: metrics: { + vector_metadata_refresh_successful_total: _vector_metadata_refresh_successful_total + vector_metadata_refresh_failed_total: _vector_metadata_refresh_failed_total + } } diff --git a/docs/reference/components/transforms/dedupe.cue b/docs/reference/components/transforms/dedupe.cue index 3612228bab056..93e5126ac7ca1 100644 --- a/docs/reference/components/transforms/dedupe.cue +++ b/docs/reference/components/transforms/dedupe.cue @@ -146,4 +146,8 @@ components: transforms: dedupe: { """ } } + + telemetry: metrics: { + vector_events_discarded_total: _vector_events_discarded_total + } } diff --git a/docs/reference/components/transforms/grok_parser.cue b/docs/reference/components/transforms/grok_parser.cue index b7070265f1521..a44e1f4442a12 100644 --- a/docs/reference/components/transforms/grok_parser.cue +++ b/docs/reference/components/transforms/grok_parser.cue @@ -95,4 +95,8 @@ components: transforms: grok_parser: { """ } } + + telemetry: metrics: { + vector_processing_errors_total: _vector_processing_errors_total + } } diff --git a/docs/reference/components/transforms/logfmt_parser.cue b/docs/reference/components/transforms/logfmt_parser.cue index 28d978f6b299c..cf1ce245cbf20 100644 --- a/docs/reference/components/transforms/logfmt_parser.cue +++ b/docs/reference/components/transforms/logfmt_parser.cue @@ -178,4 +178,8 @@ components: transforms: logfmt_parser: { """ } } + + telemetry: metrics: { + vector_processing_errors_total: _vector_processing_errors_total + } } diff --git a/docs/reference/components/transforms/lua.cue b/docs/reference/components/transforms/lua.cue index f73491ad01b7e..76b0f4bd73267 100644 --- a/docs/reference/components/transforms/lua.cue +++ b/docs/reference/components/transforms/lua.cue @@ -470,4 +470,9 @@ components: transforms: lua: { """ } } + + telemetry: metrics: { + vector_memory_used: _vector_memory_used + vector_processing_errors_total: _vector_processing_errors_total + } } diff --git a/docs/reference/components/transforms/metric_to_log.cue b/docs/reference/components/transforms/metric_to_log.cue index 54051a696075d..1d7ed4fe2b3e9 100644 --- a/docs/reference/components/transforms/metric_to_log.cue +++ b/docs/reference/components/transforms/metric_to_log.cue @@ -93,4 +93,8 @@ components: transforms: metric_to_log: { ] how_it_works: {} + + telemetry: metrics: { + vector_processing_errors_total: _vector_processing_errors_total + } } diff --git a/docs/reference/components/transforms/reduce.cue b/docs/reference/components/transforms/reduce.cue index fe9971cc211f9..e6581158c1a1a 100644 --- a/docs/reference/components/transforms/reduce.cue +++ b/docs/reference/components/transforms/reduce.cue @@ -144,4 +144,12 @@ components: transforms: reduce: { } }, ] + + telemetry: metrics: { + vector_stale_events_flushed_total: { + description: "The number of stale events that Vector has flushed." + type: "counter" + tags: _component_tags + } + } } diff --git a/docs/reference/components/transforms/regex_parser.cue b/docs/reference/components/transforms/regex_parser.cue index 5c4cb7fcc256b..bc4a8a49f3c0c 100644 --- a/docs/reference/components/transforms/regex_parser.cue +++ b/docs/reference/components/transforms/regex_parser.cue @@ -178,4 +178,8 @@ components: transforms: regex_parser: { """ } } + + telemetry: metrics: { + vector_processing_errors_total: _vector_processing_errors_total + } } diff --git a/docs/reference/components/transforms/remap.cue b/docs/reference/components/transforms/remap.cue index 5e3c3cc9f3279..7e5807b2496d8 100644 --- a/docs/reference/components/transforms/remap.cue +++ b/docs/reference/components/transforms/remap.cue @@ -130,4 +130,8 @@ components: transforms: remap: { """# } } + + telemetry: metrics: { + vector_processing_errors_total: _vector_processing_errors_total + } } diff --git a/docs/reference/components/transforms/sampler.cue b/docs/reference/components/transforms/sampler.cue index 3b86d481c0192..4962e7cc31303 100644 --- a/docs/reference/components/transforms/sampler.cue +++ b/docs/reference/components/transforms/sampler.cue @@ -64,4 +64,8 @@ components: transforms: sampler: { logs: true metrics: null } + + telemetry: metrics: { + vector_events_discarded_total: _vector_events_discarded_total + } } diff --git a/docs/reference/components/transforms/split.cue b/docs/reference/components/transforms/split.cue index 4de8c9156fee8..81477aa9173f3 100644 --- a/docs/reference/components/transforms/split.cue +++ b/docs/reference/components/transforms/split.cue @@ -95,4 +95,8 @@ components: transforms: split: { } }, ] + + telemetry: metrics: { + vector_processing_errors_total: _vector_processing_errors_total + } } diff --git a/docs/reference/components/transforms/swimlanes.cue b/docs/reference/components/transforms/swimlanes.cue index 55c8878f48283..0777e7e559ab6 100644 --- a/docs/reference/components/transforms/swimlanes.cue +++ b/docs/reference/components/transforms/swimlanes.cue @@ -70,4 +70,8 @@ components: transforms: swimlanes: { } }, ] + + telemetry: metrics: { + vector_events_discarded_total: _vector_events_discarded_total + } } diff --git a/docs/reference/components/transforms/tag_cardinality_limit.cue b/docs/reference/components/transforms/tag_cardinality_limit.cue index 943785d1af791..fa11cefab0301 100644 --- a/docs/reference/components/transforms/tag_cardinality_limit.cue +++ b/docs/reference/components/transforms/tag_cardinality_limit.cue @@ -218,4 +218,23 @@ components: transforms: tag_cardinality_limit: { """ } } + + telemetry: metrics: { + vector_tag_value_limit_exceeded_total: { + description: """ + The total number of events discarded because the tag has been rejected after + hitting the configured `value_limit`. + """ + type: "counter" + tags: _component_tags + } + vector_value_limit_reached_total: { + description: """ + The total number of times new values for a key have been rejected because the + value limit has been reached. + """ + type: "counter" + tags: _component_tags + } + } } diff --git a/docs/reference/components/transforms/tokenizer.cue b/docs/reference/components/transforms/tokenizer.cue index 9f0481a8d494b..a28af4fed4ab6 100644 --- a/docs/reference/components/transforms/tokenizer.cue +++ b/docs/reference/components/transforms/tokenizer.cue @@ -114,4 +114,8 @@ components: transforms: tokenizer: { """# } } + + telemetry: metrics: { + vector_processing_errors_total: _vector_processing_errors_total + } } diff --git a/docs/reference/metrics.cue b/docs/reference/metrics.cue index 4ccc096f27b17..0ea92d824ffe7 100644 --- a/docs/reference/metrics.cue +++ b/docs/reference/metrics.cue @@ -906,6 +906,16 @@ _vector_memory_used: { type: "gauge" tags: _internal_metrics_tags } +_vector_metadata_refresh_failed_total: { + description: "TODO" + type: "counter" + tags: _component_tags +} +_vector_metadata_refresh_successful_total: { + description: "TODO" + type: "counter" + tags: _component_tags +} _vector_missing_keys_total: { description: "The total number of events dropped due to keys missing from the event." type: "counter" From 451c7db0665db479d180b72d70b10c06bc52200c Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Sat, 7 Nov 2020 16:34:47 -0800 Subject: [PATCH 05/20] Fill in remaining TODO descriptions Signed-off-by: Luc Perkins --- .../components/sources/mongodb_metrics.cue | 24 +++-- .../components/sources/splunk_hec.cue | 4 +- docs/reference/components/sources/syslog.cue | 6 +- .../transforms/aws_ec2_metadata.cue | 12 ++- docs/reference/metrics.cue | 89 +++++++------------ docs/reference/urls.cue | 2 + 6 files changed, 69 insertions(+), 68 deletions(-) diff --git a/docs/reference/components/sources/mongodb_metrics.cue b/docs/reference/components/sources/mongodb_metrics.cue index 6dae5bda2e6ad..490d02d739cad 100644 --- a/docs/reference/components/sources/mongodb_metrics.cue +++ b/docs/reference/components/sources/mongodb_metrics.cue @@ -80,11 +80,7 @@ components: sources: mongodb_metrics: { } } - output: metrics: _mongodb_metrics & { - vector_collect_completed_total: _vector_collect_completed_total - vector_collect_duration_nanoseconds: _vector_collect_duration_nanoseconds - vector_request_error_total: _vector_request_error_total - } + output: metrics: _mongodb_metrics how_it_works: { mod_status: { @@ -99,4 +95,22 @@ components: sources: mongodb_metrics: { """ } } + + telemetry: metrics: { + vector_collect_duration_nanoseconds: { + description: "The duration spent collecting MongoDB metrics." + type: "histogram" + tags: _internal_metrics_tags + } + vector_collect_completed_total: { + description: "The total number of MongoDB metrics collections completed." + type: "counter" + tags: _internal_metrics_tags + } + vector_request_error_total: { + description: "The total number of MongoDB request errors." + type: "counter" + tags: _internal_metrics_tags + } + } } diff --git a/docs/reference/components/sources/splunk_hec.cue b/docs/reference/components/sources/splunk_hec.cue index d27205c2bb619..5dd1972dee280 100644 --- a/docs/reference/components/sources/splunk_hec.cue +++ b/docs/reference/components/sources/splunk_hec.cue @@ -94,10 +94,10 @@ components: sources: splunk_hec: { } telemetry: metrics: { + vector_source_missing_keys_total: _vector_source_missing_keys_total + vector_sourcetype_missing_keys_total: _vector_sourcetype_missing_keys_total vector_encode_errors_total: _vector_encode_errors_total vector_http_request_errors_total: _vector_http_request_errors_total vector_requests_received_total: _vector_requests_received_total - vector_source_missing_keys_total: _vector_source_missing_keys_total - vector_sourcetype_missing_keys_total: _vector_sourcetype_missing_keys_total } } diff --git a/docs/reference/components/sources/syslog.cue b/docs/reference/components/sources/syslog.cue index 4a58db6d9dbe7..e18fb25008fd6 100644 --- a/docs/reference/components/sources/syslog.cue +++ b/docs/reference/components/sources/syslog.cue @@ -185,7 +185,7 @@ components: sources: syslog: { telemetry: metrics: { vector_connection_read_errors_total: { - description: "TODO" + description: "The total number of errors reading datagram." type: "counter" tags: _component_tags & { mode: { @@ -198,11 +198,11 @@ components: sources: syslog: { } } vector_utf8_convert_errors_total: { - description: "TODO" + description: "The total number of errors converting bytes to a UTF-8 string in UDP mode." type: "counter" tags: _component_tags & { mode: { - description: "" + description: "The connection mode used by the component." required: true options: { udp: "User Datagram Protocol" diff --git a/docs/reference/components/transforms/aws_ec2_metadata.cue b/docs/reference/components/transforms/aws_ec2_metadata.cue index 9792eb4070eed..25da0a2ec96c6 100644 --- a/docs/reference/components/transforms/aws_ec2_metadata.cue +++ b/docs/reference/components/transforms/aws_ec2_metadata.cue @@ -172,7 +172,15 @@ components: transforms: aws_ec2_metadata: { } telemetry: metrics: { - vector_metadata_refresh_successful_total: _vector_metadata_refresh_successful_total - vector_metadata_refresh_failed_total: _vector_metadata_refresh_failed_total + vector_metadata_refresh_failed_total: { + description: "The total number of failed efforts to refresh AWS EC2 metadata." + type: "counter" + tags: _component_tags + } + vector_metadata_refresh_successful_total: { + description: "The total number of AWS EC2 metadata refreshes." + type: "counter" + tags: _component_tags + } } } diff --git a/docs/reference/metrics.cue b/docs/reference/metrics.cue index 0ea92d824ffe7..11465e68828de 100644 --- a/docs/reference/metrics.cue +++ b/docs/reference/metrics.cue @@ -769,7 +769,7 @@ _vector_api_started_total: { tags: _internal_metrics_tags } _vector_auto_concurrency_averaged_rtt: { - description: "TODO" + description: "The average round-trip time (RTT) from the HTTP sink across the current window." type: "histogram" tags: _internal_metrics_tags } @@ -779,12 +779,12 @@ _vector_auto_concurrency_in_flight: { tags: _internal_metrics_tags } _vector_auto_concurrency_limit: { - description: "TODO" + description: "The concurrency limit that the auto-concurrency feature has decided on for this current window." type: "histogram" tags: _internal_metrics_tags } _vector_auto_concurrency_observed_rtt: { - description: "TODO" + description: "The observed round-trip time (RTT) for requests from this HTTP sink." type: "histogram" tags: _internal_metrics_tags } @@ -799,27 +799,12 @@ _vector_checkpoints_total: { tags: _internal_metrics_tags } _vector_checksum_errors: { - description: "TODO" + description: "The total number of errors identifying files via checksum." type: "counter" tags: _internal_metrics_tags & { file: _file } } -_vector_collect_duration_nanoseconds: { - description: "TODO" - type: "histogram" - tags: _internal_metrics_tags -} -_vector_collect_completed_total: { - description: "TODO" - type: "counter" - tags: _internal_metrics_tags -} -_vector_encode_errors_total: { - description: "TODO" - type: "counter" - tags: _internal_metrics_tags -} _vector_events_discarded_total: { description: "The total number of events discarded by this component." type: "counter" @@ -887,17 +872,17 @@ _vector_http_bad_requests_total: { tags: _internal_metrics_tags } _vector_http_error_response_total: { - description: "TODO" + description: "The total number of HTTP error responses for this component." type: "counter" tags: _internal_metrics_tags } _vector_http_request_errors_total: { - description: "TODO" + description: "The total number of HTTP request errors for this component." type: "counter" tags: _internal_metrics_tags } _vector_http_requests_total: { - description: "TODO" + description: "The total number of HTTP requests issued by this component." type: "counter" tags: _component_tags } @@ -906,16 +891,6 @@ _vector_memory_used: { type: "gauge" tags: _internal_metrics_tags } -_vector_metadata_refresh_failed_total: { - description: "TODO" - type: "counter" - tags: _component_tags -} -_vector_metadata_refresh_successful_total: { - description: "TODO" - type: "counter" - tags: _component_tags -} _vector_missing_keys_total: { description: "The total number of events dropped due to keys missing from the event." type: "counter" @@ -927,7 +902,7 @@ _vector_open_connections: { tags: _internal_metrics_tags } _vector_parse_errors_total: { - description: "TODO" + description: "The total number of errors parsing Prometheus metrics." type: "counter" tags: _internal_metrics_tags } @@ -937,39 +912,39 @@ _vector_processed_bytes_total: { tags: _component_tags } _vector_processing_errors_total: { - description: "The total number of processing errors encountered by the component." + description: "The total number of processing errors encountered by this component." type: "counter" tags: _component_tags & { error_type: _error_type } } _vector_protobuf_decode_errors_total: { - description: "TODO" + description: "The total number of [Protocol Buffers](\(urls.protobuf)) errors thrown during communication between Vector instances." type: "counter" tags: _component_tags } _vector_request_duration_nanoseconds: { - description: "TODO" + description: "The request duration for this component (in nanoseconds)." type: "histogram" tags: _component_tags } _vector_request_read_errors_total: { - description: "TODO" + description: "The total number of request read errors for this component." type: "counter" tags: _component_tags } _vector_requests_completed_total: { - description: "TODO" + description: "The total number of requests completed by this component." type: "counter" tags: _component_tags } _vector_requests_received_total: { - description: "TODO" + description: "The total number of requests received by this component." type: "counter" tags: _component_tags } _vector_timestamp_parse_errors_total: { - description: "The total number of errors encountered RFC3339 parsing timestamps." + description: "The total number of errors encountered parsing [RFC3339](\(urls.rfc_3339)) timestamps." type: "counter" tags: _component_tags } @@ -979,6 +954,23 @@ _vector_uptime_seconds: { tags: _component_tags } +// Splunk +_vector_encode_errors_total: { + description: "The total number of errors encoding [Splunk HEC](\(urls.splunk_hec_protocol)) events to JSON for this `splunk_hec` sink." + type: "counter" + tags: _component_tags +} +_vector_source_missing_keys_total: { + description: "The total number of errors rendering the template for this source." + type: "counter" + tags: _component_tags +} +_vector_sourcetype_missing_keys_total: { + description: "The total number of errors rendering the template for this sourcetype." + type: "counter" + tags: _component_tags +} + // Vector instance metrics _vector_config_load_errors_total: { description: "The total number of errors loading the Vector configuration." @@ -986,7 +978,7 @@ _vector_config_load_errors_total: { tags: _internal_metrics_tags } _vector_connection_errors_total: { - description: "TODO" + description: "The total number of connection errors for this Vector instance." type: "counter" tags: _internal_metrics_tags } @@ -1010,21 +1002,6 @@ _vector_reloaded_total: { type: "counter" tags: _internal_metrics_tags } -_vector_request_error_total: { - description: "TODO" - type: "counter" - tags: _internal_metrics_tags -} -_vector_source_missing_keys_total: { - description: "TODO" - type: "counter" - tags: _internal_metrics_tags -} -_vector_sourcetype_missing_keys_total: { - description: "TODO" - type: "counter" - tags: _internal_metrics_tags -} _vector_started_total: { description: "The total number of times the Vector instance has been started." type: "counter" diff --git a/docs/reference/urls.cue b/docs/reference/urls.cue index b425a89249ef7..76b616dcbfa63 100644 --- a/docs/reference/urls.cue +++ b/docs/reference/urls.cue @@ -263,12 +263,14 @@ urls: { prometheus_metric_naming: "https://prometheus.io/docs/practices/naming/#metric-names" prometheus_remote_integrations: "https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage" prometheus_remote_write: "https://prometheus.io/docs/prometheus/latest/configuration/configuration/#remote_write" + protobuf: "https://developers.google.com/protocol-buffers" pulsar: "https://pulsar.apache.org/" pulsar_protocol: "https://pulsar.apache.org/docs/en/develop-binary-protocol/" rdkafka: "https://github.com/edenhill/librdkafka" regex: "https://en.wikipedia.org/wiki/Regular_expression" regex_grouping_and_flags: "https://docs.rs/regex/1.3.9/regex/#grouping-and-flags" regex_tester: "https://rustexp.lpil.uk/" + rfc_3339: "https://tools.ietf.org/html/rfc3339" rfc_4180: "https://tools.ietf.org/html/rfc4180" rlua: "https://github.com/kyren/rlua" rpm: "https://rpm.org/" From e9a375e984e0c362f5ea1a682951269dc739b3b7 Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Sat, 7 Nov 2020 16:48:24 -0800 Subject: [PATCH 06/20] Add internal_metrics source Signed-off-by: Luc Perkins --- .../components/sources/internal_metrics.cue | 44 +++++++++++++++++++ docs/reference/metrics.cue | 10 +++++ 2 files changed, 54 insertions(+) create mode 100644 docs/reference/components/sources/internal_metrics.cue diff --git a/docs/reference/components/sources/internal_metrics.cue b/docs/reference/components/sources/internal_metrics.cue new file mode 100644 index 0000000000000..dee8f52f6684d --- /dev/null +++ b/docs/reference/components/sources/internal_metrics.cue @@ -0,0 +1,44 @@ +package metadata + +components: sources: internal_metrics: { + title: "Internal Metrics" + description: "The internal metrics source exposes metrics emitted by the running Vector instance (as opposed to components in its topology)." + + classes: { + commonly_used: true + delivery: "at_least_once" + deployment_roles: ["aggregator", "daemon", "sidecar"] + development: "beta" + egress_method: "batch" + } + + features: { + collect: { + checkpoint: enabled: false + from: { + name: "Vector instance" + thing: "a \(name)" + url: urls.vector_docs + versions: ">= 0.11.0" + } + } + multiline: enabled: false + } + + support: { + platforms: { + "aarch64-unknown-linux-gnu": true + "aarch64-unknown-linux-musl": true + "x86_64-apple-darwin": true + "x86_64-pc-windows-msv": true + "x86_64-unknown-linux-gnu": true + "x86_64-unknown-linux-musl": true + } + + notices: [] + requirements: [] + warnings: [] + } + + output: metrics: _internal_metrics +} diff --git a/docs/reference/metrics.cue b/docs/reference/metrics.cue index 11465e68828de..21d29c846ada3 100644 --- a/docs/reference/metrics.cue +++ b/docs/reference/metrics.cue @@ -1266,6 +1266,16 @@ _host_metrics: { host_network_transmit_packets_total: _host_network_transmit_packets_total } +_internal_metrics: { + vector_config_load_errors_total: _vector_config_load_errors_total + vector_quit_total: _vector_quit_total + vector_recover_errors_total: _vector_recover_errors_total + vector_reload_errors_total: _vector_reload_errors_total + vector_reloaded_total: _vector_reloaded_total + vector_started_total: _vector_started_total + vector_stopped_total: _vector_stopped_total +} + _mongodb_metrics: { mongodb_assets_total: _mongodb_assets_total mongodb_bson_parse_error_total: _mongodb_bson_parse_error_total From bd36ba72949f1b32d98046aad846bd68431999f6 Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Sun, 8 Nov 2020 16:36:36 -0800 Subject: [PATCH 07/20] Fix service/aggregator discrepancy Signed-off-by: Luc Perkins --- docs/reference.cue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference.cue b/docs/reference.cue index 36ae0bac0342a..e728bbe86f597 100644 --- a/docs/reference.cue +++ b/docs/reference.cue @@ -44,7 +44,7 @@ _values: { // * `sidecar` - Vector is installed alongside each process it is // monitoring. Therefore, there might be multiple Vector processes // on the host. -// * `service` - Vector receives data from one or more upstream +// * `aggregator` - Vector receives data from one or more upstream // sources, typically over a network protocol. #DeploymentRole: "aggregator" | "daemon" | "sidecar" From 12bbaed833e104fbd8679249e09db09f53c9acea Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Thu, 12 Nov 2020 12:18:55 -0800 Subject: [PATCH 08/20] Move Apache, MongoDB, and host metrics out of metrics.cue Signed-off-by: Luc Perkins --- .../components/sources/apache_metrics.cue | 82 +- .../components/sources/host_metrics.cue | 127 ++- .../components/sources/mongodb_metrics.cue | 587 +++++++++++- docs/reference/metrics.cue | 903 ------------------ 4 files changed, 789 insertions(+), 910 deletions(-) diff --git a/docs/reference/components/sources/apache_metrics.cue b/docs/reference/components/sources/apache_metrics.cue index 7d58eefdaf261..998522224732e 100644 --- a/docs/reference/components/sources/apache_metrics.cue +++ b/docs/reference/components/sources/apache_metrics.cue @@ -111,7 +111,87 @@ components: sources: apache_metrics: { } } - output: metrics: _apache_metrics + output: metrics: { + apache_access_total: { + description: "The total number of time the Apache server has been accessed." + relevant_when: "`ExtendedStatus On`" + type: "counter" + tags: _apache_metrics_tags + } + apache_connections: { + description: "The total number of time the Apache server has been accessed." + type: "gauge" + tags: _apache_metrics_tags & { + state: { + description: "The state of the connection" + required: true + examples: ["closing", "keepalive", "total", "writing"] + } + } + } + apache_cpu_load: { + description: "The current CPU of the Apache server." + relevant_when: "`ExtendedStatus On`" + type: "gauge" + tags: _apache_metrics_tags + } + apache_cpu_seconds_total: { + description: "The CPU time of various Apache processes." + relevant_when: "`ExtendedStatus On`" + type: "counter" + tags: _apache_metrics_tags & { + state: { + description: "The state of the connection" + required: true + examples: ["children_system", "children_user", "system", "user"] + } + } + } + apache_duration_seconds_total: { + description: "The amount of time the Apache server has been running." + relevant_when: "`ExtendedStatus On`" + type: "counter" + tags: _apache_metrics_tags + } + apache_scoreboard: { + description: "The amount of times various Apache server tasks have been run." + type: "gauge" + tags: _apache_metrics_tags & { + state: { + description: "The connect state" + required: true + examples: ["closing", "dnslookup", "finishing", "idle_cleanup", "keepalive", "logging", "open", "reading", "sending", "starting", "waiting"] + } + } + } + apache_sent_bytes_total: { + description: "The amount of bytes sent by the Apache server." + relevant_when: "`ExtendedStatus On`" + type: "counter" + tags: _apache_metrics_tags + } + apache_up: { + description: "If the Apache server is up or not." + type: "gauge" + tags: _apache_metrics_tags + } + apache_uptime_seconds_total: { + description: "The amount of time the Apache server has been running." + type: "counter" + tags: _apache_metrics_tags + } + apache_workers: { + description: "Apache worker statuses." + type: "gauge" + tags: _apache_metrics_tags & { + state: { + description: "The state of the worker" + required: true + examples: ["busy", "idle"] + } + } + } + } how_it_works: {} } diff --git a/docs/reference/components/sources/host_metrics.cue b/docs/reference/components/sources/host_metrics.cue index 2fc5efbdf0640..912a80788eee1 100644 --- a/docs/reference/components/sources/host_metrics.cue +++ b/docs/reference/components/sources/host_metrics.cue @@ -260,5 +260,130 @@ components: sources: host_metrics: { } } - output: metrics: _host_metrics + output: metrics: { + // Host CPU + host_cpu_seconds_total: { + description: "The number of CPU seconds accumulated in different operating modes." + type: "counter" + tags: _host_metrics_tags & { + collector: examples: ["cpu"] + cpu: { + description: "The index of the CPU core or socket." + required: true + examples: ["1"] + } + mode: { + description: "Which mode the CPU was running in during the given time." + required: true + examples: ["idle", "system", "user", "nice"] + } + } + } + + // Host disk + host_disk_read_bytes_total: _disk_counter & {description: "The accumulated number of bytes read in."} + host_disk_reads_completed_total: _disk_counter & {description: "The accumulated number of read operations completed."} + host_disk_written_bytes_total: _disk_counter & {description: "The accumulated number of bytes written out."} + host_disk_writes_completed_total: _disk_counter & {description: "The accumulated number of write operations completed."} + + // Host filesystem + host_filesystem_free_bytes: _filesystem_bytes & {description: "The number of bytes free on the named filesystem."} + host_filesystem_total_bytes: _filesystem_bytes & {description: "The total number of bytes in the named filesystem."} + host_filesystem_used_bytes: _filesystem_bytes & {description: "The number of bytes used on the named filesystem."} + + // Host load + host_load1: _loadavg & {description: "System load averaged over the last 1 second."} + host_load5: _loadavg & {description: "System load averaged over the last 5 seconds."} + host_load15: _loadavg & {description: "System load averaged over the last 15 seconds."} + + // Host memory + host_memory_active_bytes: _memory_gauge & _memory_nowin & {description: "The number of bytes of active main memory."} + host_memory_available_bytes: _memory_gauge & {description: "The number of bytes of main memory available."} + host_memory_buffers_bytes: _memory_linux & {description: "The number of bytes of main memory used by buffers."} + host_memory_cached_bytes: _memory_linux & {description: "The number of bytes of main memory used by cached blocks."} + host_memory_free_bytes: _memory_gauge & {description: "The number of bytes of main memory not used."} + host_memory_inactive_bytes: _memory_macos & {description: "The number of bytes of main memory that is not active."} + host_memory_shared_bytes: _memory_linux & {description: "The number of bytes of main memory shared between processes."} + host_memory_swap_free_bytes: _memory_gauge & {description: "The number of free bytes of swap space."} + host_memory_swapped_in_bytes_total: _memory_counter & _memory_nowin & { + description: "The number of bytes that have been swapped in to main memory." + } + host_memory_swapped_out_bytes_total: _memory_counter & _memory_nowin & { + description: "The number of bytes that have been swapped out from main memory." + } + host_memory_swap_total_bytes: _memory_gauge & {description: "The total number of bytes of swap space."} + host_memory_swap_used_bytes: _memory_gauge & {description: "The number of used bytes of swap space."} + host_memory_total_bytes: _memory_gauge & {description: "The total number of bytes of main memory."} + host_memory_used_bytes: _memory_linux & {description: "The number of bytes of main memory used by programs or caches."} + host_memory_wired_bytes: _memory_macos & {description: "The number of wired bytes of main memory."} + + // Host network + host_network_receive_bytes_total: _network_gauge & {description: "The number of bytes received on this interface."} + host_network_receive_errs_total: _network_gauge & {description: "The number of errors encountered during receives on this interface."} + host_network_receive_packets_total: _network_gauge & {description: "The number of packets received on this interface."} + host_network_transmit_bytes_total: _network_gauge & {description: "The number of bytes transmitted on this interface."} + host_network_transmit_errs_total: _network_gauge & {description: "The number of errors encountered during transmits on this interface."} + host_network_transmit_packets_drop_total: _network_nomac & {description: "The number of packets dropped during transmits on this interface."} + host_network_transmit_packets_total: _network_nomac & {description: "The number of packets transmitted on this interface."} + + // Helpers + _disk_device: { + description: "The disk device name." + required: true + examples: ["sda", "sda1", "dm-1"] + } + _disk_counter: { + type: "counter" + tags: _host_metrics_tags & { + collector: examples: ["disk"] + device: _disk_device + } + } + _filesystem_bytes: { + type: "gauge" + tags: _host_metrics_tags & { + collector: examples: ["filesystem"] + device: _disk_device + filesystem: { + description: "The name of the filesystem type." + required: true + examples: ["ext4", "ntfs"] + } + } + } + _loadavg: { + type: "gauge" + tags: _host_metrics_tags & { + collector: examples: ["loadavg"] + } + relevant_when: "OS is not Windows" + } + _memory_counter: { + type: "counter" + tags: _host_metrics_tags & { + collector: examples: ["memory"] + } + } + _memory_gauge: { + type: "gauge" + tags: _host_metrics_tags & { + collector: examples: ["memory"] + } + } + _memory_linux: _memory_gauge & {relevant_when: "OS is Linux"} + _memory_macos: _memory_gauge & {relevant_when: "OS is MacOS X"} + _memory_nowin: {relevant_when: "OS is not Windows"} + _network_gauge: { + type: "gauge" + tags: _host_metrics_tags & { + collector: examples: ["network"] + device: { + description: "The network interface device name." + required: true + examples: ["eth0", "enp5s3"] + } + } + } + _network_nomac: _network_gauge & {relevant_when: "OS is not MacOS"} + } } diff --git a/docs/reference/components/sources/mongodb_metrics.cue b/docs/reference/components/sources/mongodb_metrics.cue index 490d02d739cad..963d40d77305a 100644 --- a/docs/reference/components/sources/mongodb_metrics.cue +++ b/docs/reference/components/sources/mongodb_metrics.cue @@ -80,8 +80,6 @@ components: sources: mongodb_metrics: { } } - output: metrics: _mongodb_metrics - how_it_works: { mod_status: { title: "MongoDB `serverStatus` command" @@ -97,20 +95,599 @@ components: sources: mongodb_metrics: { } telemetry: metrics: { - vector_collect_duration_nanoseconds: { + collect_duration_nanoseconds: { description: "The duration spent collecting MongoDB metrics." type: "histogram" tags: _internal_metrics_tags } - vector_collect_completed_total: { + collect_completed_total: { description: "The total number of MongoDB metrics collections completed." type: "counter" tags: _internal_metrics_tags } - vector_request_error_total: { + request_error_total: { description: "The total number of MongoDB request errors." type: "counter" tags: _internal_metrics_tags } } + + output: metrics: { + _mongodb_metrics_tags: { + endpoint: { + description: "The absolute path of the originating file." + required: true + examples: ["mongodb://localhost:27017"] + } + host: { + description: "The hostname of the MongoDB server." + required: true + examples: [_values.local_host] + } + } + + mongodb_assets_total: { + description: "Number of assertions raised since the MongoDB process started." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "The assertion type" + required: true + examples: ["regular", "warning", "msg", "user", "rollovers"] + } + } + } + mongodb_bson_parse_error_total: { + description: "The total number of BSON parsing errors." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_connections: { + description: "Number of connections in some state." + type: "gauge" + tags: _mongodb_metrics_tags & { + state: { + description: "The connection state" + required: true + examples: ["active", "available", "current"] + } + } + } + mongodb_extra_info_heap_usage_bytes: { + description: "The total size in bytes of heap space used by the database process." + relevant_when: "Unix/Linux" + type: "gauge" + tags: _mongodb_metrics_tags + } + mongodb_extra_info_page_faults: { + description: "The total number of page faults." + type: "gauge" + tags: _mongodb_metrics_tags + } + mongodb_instance_local_time: { + description: "The ISODate representing the current time, according to the server, in UTC." + type: "gauge" + tags: _mongodb_metrics_tags + } + mongodb_instance_uptime_estimate_seconds_total: { + description: "The uptime in seconds as calculated from MongoDB’s internal course-grained time keeping system." + type: "gauge" + tags: _mongodb_metrics_tags + } + mongodb_instance_uptime_seconds_total: { + description: "The number of seconds that the current MongoDB process has been active." + type: "gauge" + tags: _mongodb_metrics_tags + } + mongodb_memory: { + description: "Current memory unsage." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Memory type" + required: true + examples: ["resident", "virtual", "mapped", "mapped_with_journal"] + } + } + } + mongodb_mongod_global_lock_active_clients: { + description: "Number of connected clients and the read and write operations performed by these clients." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Number type." + required: true + examples: ["total", "readers", "writers"] + } + } + } + mongodb_mongod_global_lock_current_queue: { + description: "Number of operations queued because of a lock." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Number type." + required: true + examples: ["total", "readers", "writers"] + } + } + } + mongodb_mongod_global_lock_total_time_seconds: { + description: "The time since the database last started and created the globalLock. This is roughly equivalent to total server uptime." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_locks_time_acquiring_global_seconds_total: { + description: "Amount of time that any database has spent waiting for the global lock." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Lock type." + required: true + examples: ["ParallelBatchWriterMode", "ReplicationStateTransition", "Global", "Database", "Collection", "Mutex", "Metadata", "oplog"] + } + mode: { + description: "Lock mode." + required: true + examples: ["read", "write"] + } + } + } + mongodb_mongod_metrics_cursor_open: { + description: "Number of cursors." + type: "gauge" + tags: _mongodb_metrics_tags & { + state: { + description: "Cursor state." + required: true + examples: ["no_timeout", "pinned", "total"] + } + } + } + mongodb_mongod_metrics_cursor_timed_out_total: { + description: "The total number of cursors that have timed out since the server process started." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_metrics_document_total: { + description: "Document access and modification patterns." + type: "counter" + tags: _mongodb_metrics_tags & { + state: { + description: "Document state." + required: true + examples: ["deleted", "inserted", "returned", "updated"] + } + } + } + mongodb_mongod_metrics_get_last_error_wtime_num: { + description: "The total number of getLastError operations with a specified write concern." + type: "gauge" + tags: _mongodb_metrics_tags + } + mongodb_mongod_metrics_get_last_error_wtime_seconds_total: { + description: "The total amount of time that the mongod has spent performing getLastError operations." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_metrics_get_last_error_wtimeouts_total: { + description: "The number of times that write concern operations have timed out as a result of the wtimeout threshold to getLastError." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_metrics_operation_total: { + description: "Update and query operations that MongoDB handles using special operation types." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Operation type." + required: true + examples: ["scan_and_order", "write_conflicts"] + } + } + } + mongodb_mongod_metrics_query_executor_total: { + description: "Data from query execution system." + type: "counter" + tags: _mongodb_metrics_tags & { + state: { + description: "Query state." + required: true + examples: ["scanned", "scanned_objects", "collection_scans"] + } + } + } + mongodb_mongod_metrics_record_moves_total: { + description: "Moves reports the total number of times documents move within the on-disk representation of the MongoDB data set. Documents move as a result of operations that increase the size of the document beyond their allocated record size." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_metrics_repl_apply_batches_num_total: { + description: "The total number of batches applied across all databases." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_metrics_repl_apply_batches_seconds_total: { + description: "The total amount of time the mongod has spent applying operations from the oplog." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_metrics_repl_apply_ops_total: { + description: "The total number of oplog operations applied." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_metrics_repl_buffer_count: { + description: "The current number of operations in the oplog buffer." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_metrics_repl_buffer_max_size_bytes_total: { + description: "The maximum size of the buffer." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_metrics_repl_buffer_size_bytes: { + description: "The current size of the contents of the oplog buffer." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_metrics_repl_executor_queue: { + description: "Number of queued operations in the replication executor." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Queue type." + required: true + examples: ["network_in_progress", "sleepers"] + } + } + } + mongodb_mongod_metrics_repl_executor_unsignaled_events: { + description: "Number of unsignaled events in the replication executor." + type: "gauge" + tags: _mongodb_metrics_tags + } + mongodb_mongod_metrics_repl_network_bytes_total: { + description: "The total amount of data read from the replication sync source." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_metrics_repl_network_getmores_num_total: { + description: "The total number of getmore operations, which are operations that request an additional set of operations from the replication sync source." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_metrics_repl_network_getmores_seconds_total: { + description: "The total amount of time required to collect data from getmore operations." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_metrics_repl_network_ops_total: { + description: "The total number of operations read from the replication source." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_metrics_repl_network_readers_created_total: { + description: "The total number of oplog query processes created." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_metrics_ttl_deleted_documents_total: { + description: "The total number of documents deleted from collections with a ttl index." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_metrics_ttl_passes_total: { + description: "The number of times the background process removes documents from collections with a ttl index." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_op_latencies_histogram: { + description: "Latency statistics." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Latency type." + required: true + examples: ["reads", "writes", "commands"] + } + micros: { + description: "Bucket." + required: true + examples: ["1", "2", "4096", "16384", "49152"] + } + } + } + mongodb_mongod_op_latencies_latency: { + description: "A 64-bit integer giving the total combined latency in microseconds." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Latency type." + required: true + examples: ["network_in_progress", "sleepers"] + } + } + } + mongodb_mongod_op_latencies_ops_total: { + description: "A 64-bit integer giving the total number of operations performed on the collection since startup." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Latency type." + required: true + examples: ["network_in_progress", "sleepers"] + } + } + } + mongodb_mongod_storage_engine: { + description: "The name of the current storage engine." + type: "gauge" + tags: _mongodb_metrics_tags & { + engine: { + description: "Engine name." + required: true + examples: ["wiredTiger"] + } + } + } + mongodb_mongod_wiredtiger_blockmanager_blocks_total: { + description: "Statistics on the block manager operations." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Operation type." + required: true + examples: ["blocks_read", "blocks_read_mapped", "blocks_pre_loaded", "blocks_written"] + } + } + } + mongodb_mongod_wiredtiger_blockmanager_bytes_total: { + description: "Statistics on the block manager operations." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Operation type." + required: true + examples: ["bytes_read", "bytes_read_mapped", "bytes_written"] + } + } + } + mongodb_mongod_wiredtiger_cache_bytes: { + description: "Statistics on the cache and page evictions from the cache." + relevant_when: "Storage engine is `wiredTiger`." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Statistics type." + required: true + examples: ["total", "dirty", "internal_pages", "leaf_pages"] + } + } + } + mongodb_mongod_wiredtiger_cache_bytes_total: { + description: "Statistics on the cache and page evictions from the cache." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Statistics type." + required: true + examples: ["read", "written"] + } + } + } + mongodb_mongod_wiredtiger_cache_evicted_total: { + description: "Statistics on the cache and page evictions from the cache." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Statistics type." + required: true + examples: ["modified", "unmodified"] + } + } + } + mongodb_mongod_wiredtiger_cache_max_bytes: { + description: "Maximum cache size." + type: "gauge" + tags: _mongodb_metrics_tags + } + mongodb_mongod_wiredtiger_cache_overhead_percent: { + description: "Percentage overhead." + type: "gauge" + tags: _mongodb_metrics_tags + } + mongodb_mongod_wiredtiger_cache_pages: { + description: "Pages in the cache." + relevant_when: "Storage engine is `wiredTiger`." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Pages type." + required: true + examples: ["total", "dirty"] + } + } + } + mongodb_mongod_wiredtiger_cache_pages_total: { + description: "Pages in the cache." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Pages type." + required: true + examples: ["read", "write"] + } + } + } + mongodb_mongod_wiredtiger_concurrent_transactions_available_tickets: { + description: "Information on the number of concurrent of read and write transactions allowed into the WiredTiger storage engine" + relevant_when: "Storage engine is `wiredTiger`." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Transactions type." + required: true + examples: ["read", "write"] + } + } + } + mongodb_mongod_wiredtiger_concurrent_transactions_out_tickets: { + description: "Information on the number of concurrent of read and write transactions allowed into the WiredTiger storage engine" + relevant_when: "Storage engine is `wiredTiger`." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Transactions type." + required: true + examples: ["read", "write"] + } + } + } + mongodb_mongod_wiredtiger_concurrent_transactions_total_tickets: { + description: "Information on the number of concurrent of read and write transactions allowed into the WiredTiger storage engine" + relevant_when: "Storage engine is `wiredTiger`." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Transactions type." + required: true + examples: ["read", "write"] + } + } + } + mongodb_mongod_wiredtiger_log_bytes_total: { + description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Bytes type." + required: true + examples: ["payload", "written"] + } + } + } + mongodb_mongod_wiredtiger_log_operations_total: { + description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Operations type." + required: true + examples: ["write", "scan", "scan_double", "sync", "sync_dir", "flush"] + } + } + } + mongodb_mongod_wiredtiger_log_records_scanned_total: { + description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Scanned records type." + required: true + examples: ["compressed", "uncompressed"] + } + } + } + mongodb_mongod_wiredtiger_log_records_total: { + description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_wiredtiger_session_open_sessions: { + description: "Open session count." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_wiredtiger_transactions_checkpoint_seconds: { + description: "Statistics on transaction checkpoints and operations." + relevant_when: "Storage engine is `wiredTiger`." + type: "gauge" + tags: _mongodb_metrics_tags & { + type: { + description: "Checkpoint type." + required: true + examples: ["min", "max"] + } + } + } + mongodb_mongod_wiredtiger_transactions_checkpoint_seconds_total: { + description: "Statistics on transaction checkpoints and operations." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_wiredtiger_transactions_running_checkpoints: { + description: "Statistics on transaction checkpoints and operations." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_mongod_wiredtiger_transactions_total: { + description: "Statistics on transaction checkpoints and operations." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Transactions type." + required: true + examples: ["begins", "checkpoints", "committed", "rolledback"] + } + } + } + mongodb_network_bytes_total: { + description: "The number of bytes that reflects the amount of network traffic." + type: "counter" + tags: _mongodb_metrics_tags & { + state: { + description: "Bytes state." + required: true + examples: ["bytes_in", "bytes_out"] + } + } + } + mongodb_network_metrics_num_requests_total: { + description: "The total number of distinct requests that the server has received." + type: "counter" + tags: _mongodb_metrics_tags + } + mongodb_op_counters_repl_total: { + description: "Database replication operations by type since the mongod instance last started." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Counter type." + required: true + examples: ["insert", "query", "update", "delete", "getmore", "command"] + } + } + } + mongodb_op_counters_total: { + description: "Database operations by type since the mongod instance last started." + type: "counter" + tags: _mongodb_metrics_tags & { + type: { + description: "Counter type." + required: true + examples: ["insert", "query", "update", "delete", "getmore", "command"] + } + } + } + mongodb_up: { + description: "If the MongoDB server is up or not." + type: "gauge" + tags: _mongodb_metrics_tags + } + } } diff --git a/docs/reference/metrics.cue b/docs/reference/metrics.cue index 21d29c846ada3..de2f3d324d8e9 100644 --- a/docs/reference/metrics.cue +++ b/docs/reference/metrics.cue @@ -1,86 +1,5 @@ package metadata -// Apache metrics -_apache_access_total: { - description: "The total number of time the Apache server has been accessed." - relevant_when: "`ExtendedStatus On`" - type: "counter" - tags: _apache_metrics_tags -} -_apache_connections: { - description: "The total number of time the Apache server has been accessed." - type: "gauge" - tags: _apache_metrics_tags & { - state: { - description: "The state of the connection" - required: true - examples: ["closing", "keepalive", "total", "writing"] - } - } -} -_apache_cpu_load: { - description: "The current CPU of the Apache server." - relevant_when: "`ExtendedStatus On`" - type: "gauge" - tags: _apache_metrics_tags -} -_apache_cpu_seconds_total: { - description: "The CPU time of various Apache processes." - relevant_when: "`ExtendedStatus On`" - type: "counter" - tags: _apache_metrics_tags & { - state: { - description: "The state of the connection" - required: true - examples: ["children_system", "children_user", "system", "user"] - } - } -} -_apache_duration_seconds_total: { - description: "The amount of time the Apache server has been running." - relevant_when: "`ExtendedStatus On`" - type: "counter" - tags: _apache_metrics_tags -} -_apache_scoreboard: { - description: "The amount of times various Apache server tasks have been run." - type: "gauge" - tags: _apache_metrics_tags & { - state: { - description: "The connect state" - required: true - examples: ["closing", "dnslookup", "finishing", "idle_cleanup", "keepalive", "logging", "open", "reading", "sending", "starting", "waiting"] - } - } -} -_apache_sent_bytes_total: { - description: "The amount of bytes sent by the Apache server." - relevant_when: "`ExtendedStatus On`" - type: "counter" - tags: _apache_metrics_tags -} -_apache_up: { - description: "If the Apache server is up or not." - type: "gauge" - tags: _apache_metrics_tags -} -_apache_uptime_seconds_total: { - description: "The amount of time the Apache server has been running." - type: "counter" - tags: _apache_metrics_tags -} -_apache_workers: { - description: "Apache worker statuses." - type: "gauge" - tags: _apache_metrics_tags & { - state: { - description: "The state of the worker" - required: true - examples: ["busy", "idle"] - } - } -} - // Container metrics _vector_communication_errors_total: { description: "The total number of errors stemming from communication with the Docker daemon." @@ -118,72 +37,6 @@ _vector_logging_driver_errors_total: { tags: _component_tags } -// Host metrics -// Host CPU -_host_cpu_seconds_total: { - description: "The number of CPU seconds accumulated in different operating modes." - type: "counter" - tags: _host_metrics_tags & { - collector: examples: ["cpu"] - cpu: { - description: "The index of the CPU core or socket." - required: true - examples: ["1"] - } - mode: { - description: "Which mode the CPU was running in during the given time." - required: true - examples: ["idle", "system", "user", "nice"] - } - } -} - -// Host disk -_host_disk_read_bytes_total: _disk_counter & {description: "The accumulated number of bytes read in."} -_host_disk_reads_completed_total: _disk_counter & {description: "The accumulated number of read operations completed."} -_host_disk_written_bytes_total: _disk_counter & {description: "The accumulated number of bytes written out."} -_host_disk_writes_completed_total: _disk_counter & {description: "The accumulated number of write operations completed."} - -// Host filesystem -_host_filesystem_free_bytes: _filesystem_bytes & {description: "The number of bytes free on the named filesystem."} -_host_filesystem_total_bytes: _filesystem_bytes & {description: "The total number of bytes in the named filesystem."} -_host_filesystem_used_bytes: _filesystem_bytes & {description: "The number of bytes used on the named filesystem."} - -// Host load -_host_load1: _loadavg & {description: "System load averaged over the last 1 second."} -_host_load5: _loadavg & {description: "System load averaged over the last 5 seconds."} -_host_load15: _loadavg & {description: "System load averaged over the last 15 seconds."} - -// Host memory -_host_memory_active_bytes: _memory_gauge & _memory_nowin & {description: "The number of bytes of active main memory."} -_host_memory_available_bytes: _memory_gauge & {description: "The number of bytes of main memory available."} -_host_memory_buffers_bytes: _memory_linux & {description: "The number of bytes of main memory used by buffers."} -_host_memory_cached_bytes: _memory_linux & {description: "The number of bytes of main memory used by cached blocks."} -_host_memory_free_bytes: _memory_gauge & {description: "The number of bytes of main memory not used."} -_host_memory_inactive_bytes: _memory_macos & {description: "The number of bytes of main memory that is not active."} -_host_memory_shared_bytes: _memory_linux & {description: "The number of bytes of main memory shared between processes."} -_host_memory_swap_free_bytes: _memory_gauge & {description: "The number of free bytes of swap space."} -_host_memory_swapped_in_bytes_total: _memory_counter & _memory_nowin & { - description: "The number of bytes that have been swapped in to main memory." -} -_host_memory_swapped_out_bytes_total: _memory_counter & _memory_nowin & { - description: "The number of bytes that have been swapped out from main memory." -} -_host_memory_swap_total_bytes: _memory_gauge & {description: "The total number of bytes of swap space."} -_host_memory_swap_used_bytes: _memory_gauge & {description: "The number of used bytes of swap space."} -_host_memory_total_bytes: _memory_gauge & {description: "The total number of bytes of main memory."} -_host_memory_used_bytes: _memory_linux & {description: "The number of bytes of main memory used by programs or caches."} -_host_memory_wired_bytes: _memory_macos & {description: "The number of wired bytes of main memory."} - -// Host network -_host_network_receive_bytes_total: _network_gauge & {description: "The number of bytes received on this interface."} -_host_network_receive_errs_total: _network_gauge & {description: "The number of errors encountered during receives on this interface."} -_host_network_receive_packets_total: _network_gauge & {description: "The number of packets received on this interface."} -_host_network_transmit_bytes_total: _network_gauge & {description: "The number of bytes transmitted on this interface."} -_host_network_transmit_errs_total: _network_gauge & {description: "The number of errors encountered during transmits on this interface."} -_host_network_transmit_packets_drop_total: _network_nomac & {description: "The number of packets dropped during transmits on this interface."} -_host_network_transmit_packets_total: _network_nomac & {description: "The number of packets transmitted on this interface."} - // Kubernetes metrics _vector_k8s_docker_format_parse_failures_total: { description: "The total number of failures to parse a message as a JSON object." @@ -197,571 +50,6 @@ _vector_k8s_event_annotation_failures_total: { tags: _component_tags } -// MongoDB metrics -_mongodb_assets_total: { - description: "Number of assertions raised since the MongoDB process started." - type: "counter" - tags: _mongodb_metrics_tags & { - type: { - description: "The assertion type" - required: true - examples: ["regular", "warning", "msg", "user", "rollovers"] - } - } -} -_mongodb_bson_parse_error_total: { - description: "The total number of BSON parsing errors." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_connections: { - description: "Number of connections in some state." - type: "gauge" - tags: _mongodb_metrics_tags & { - state: { - description: "The connection state" - required: true - examples: ["active", "available", "current"] - } - } -} -_mongodb_extra_info_heap_usage_bytes: { - description: "The total size in bytes of heap space used by the database process." - relevant_when: "Unix/Linux" - type: "gauge" - tags: _mongodb_metrics_tags -} -_mongodb_extra_info_page_faults: { - description: "The total number of page faults." - type: "gauge" - tags: _mongodb_metrics_tags -} -_mongodb_instance_local_time: { - description: "The ISODate representing the current time, according to the server, in UTC." - type: "gauge" - tags: _mongodb_metrics_tags -} -_mongodb_instance_uptime_estimate_seconds_total: { - description: "The uptime in seconds as calculated from MongoDB’s internal course-grained time keeping system." - type: "gauge" - tags: _mongodb_metrics_tags -} -_mongodb_instance_uptime_seconds_total: { - description: "The number of seconds that the current MongoDB process has been active." - type: "gauge" - tags: _mongodb_metrics_tags -} -_mongodb_memory: { - description: "Current memory unsage." - type: "gauge" - tags: _mongodb_metrics_tags & { - type: { - description: "Memory type" - required: true - examples: ["resident", "virtual", "mapped", "mapped_with_journal"] - } - } -} -_mongodb_mongod_global_lock_active_clients: { - description: "Number of connected clients and the read and write operations performed by these clients." - type: "gauge" - tags: _mongodb_metrics_tags & { - type: { - description: "Number type." - required: true - examples: ["total", "readers", "writers"] - } - } -} -_mongodb_mongod_global_lock_current_queue: { - description: "Number of operations queued because of a lock." - type: "gauge" - tags: _mongodb_metrics_tags & { - type: { - description: "Number type." - required: true - examples: ["total", "readers", "writers"] - } - } -} -_mongodb_mongod_global_lock_total_time_seconds: { - description: "The time since the database last started and created the globalLock. This is roughly equivalent to total server uptime." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_locks_time_acquiring_global_seconds_total: { - description: "Amount of time that any database has spent waiting for the global lock." - type: "counter" - tags: _mongodb_metrics_tags & { - type: { - description: "Lock type." - required: true - examples: ["ParallelBatchWriterMode", "ReplicationStateTransition", "Global", "Database", "Collection", "Mutex", "Metadata", "oplog"] - } - mode: { - description: "Lock mode." - required: true - examples: ["read", "write"] - } - } -} -_mongodb_mongod_metrics_cursor_open: { - description: "Number of cursors." - type: "gauge" - tags: _mongodb_metrics_tags & { - state: { - description: "Cursor state." - required: true - examples: ["no_timeout", "pinned", "total"] - } - } -} -_mongodb_mongod_metrics_cursor_timed_out_total: { - description: "The total number of cursors that have timed out since the server process started." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_metrics_document_total: { - description: "Document access and modification patterns." - type: "counter" - tags: _mongodb_metrics_tags & { - state: { - description: "Document state." - required: true - examples: ["deleted", "inserted", "returned", "updated"] - } - } -} -_mongodb_mongod_metrics_get_last_error_wtime_num: { - description: "The total number of getLastError operations with a specified write concern." - type: "gauge" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_metrics_get_last_error_wtime_seconds_total: { - description: "The total amount of time that the mongod has spent performing getLastError operations." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_metrics_get_last_error_wtimeouts_total: { - description: "The number of times that write concern operations have timed out as a result of the wtimeout threshold to getLastError." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_metrics_operation_total: { - description: "Update and query operations that MongoDB handles using special operation types." - type: "counter" - tags: _mongodb_metrics_tags & { - type: { - description: "Operation type." - required: true - examples: ["scan_and_order", "write_conflicts"] - } - } -} -_mongodb_mongod_metrics_query_executor_total: { - description: "Data from query execution system." - type: "counter" - tags: _mongodb_metrics_tags & { - state: { - description: "Query state." - required: true - examples: ["scanned", "scanned_objects", "collection_scans"] - } - } -} -_mongodb_mongod_metrics_record_moves_total: { - description: "Moves reports the total number of times documents move within the on-disk representation of the MongoDB data set. Documents move as a result of operations that increase the size of the document beyond their allocated record size." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_metrics_repl_apply_batches_num_total: { - description: "The total number of batches applied across all databases." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_metrics_repl_apply_batches_seconds_total: { - description: "The total amount of time the mongod has spent applying operations from the oplog." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_metrics_repl_apply_ops_total: { - description: "The total number of oplog operations applied." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_metrics_repl_buffer_count: { - description: "The current number of operations in the oplog buffer." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_metrics_repl_buffer_max_size_bytes_total: { - description: "The maximum size of the buffer." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_metrics_repl_buffer_size_bytes: { - description: "The current size of the contents of the oplog buffer." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_metrics_repl_executor_queue: { - description: "Number of queued operations in the replication executor." - type: "gauge" - tags: _mongodb_metrics_tags & { - type: { - description: "Queue type." - required: true - examples: ["network_in_progress", "sleepers"] - } - } -} -_mongodb_mongod_metrics_repl_executor_unsignaled_events: { - description: "Number of unsignaled events in the replication executor." - type: "gauge" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_metrics_repl_network_bytes_total: { - description: "The total amount of data read from the replication sync source." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_metrics_repl_network_getmores_num_total: { - description: "The total number of getmore operations, which are operations that request an additional set of operations from the replication sync source." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_metrics_repl_network_getmores_seconds_total: { - description: "The total amount of time required to collect data from getmore operations." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_metrics_repl_network_ops_total: { - description: "The total number of operations read from the replication source." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_metrics_repl_network_readers_created_total: { - description: "The total number of oplog query processes created." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_metrics_ttl_deleted_documents_total: { - description: "The total number of documents deleted from collections with a ttl index." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_metrics_ttl_passes_total: { - description: "The number of times the background process removes documents from collections with a ttl index." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_op_latencies_histogram: { - description: "Latency statistics." - type: "gauge" - tags: _mongodb_metrics_tags & { - type: { - description: "Latency type." - required: true - examples: ["reads", "writes", "commands"] - } - micros: { - description: "Bucket." - required: true - examples: ["1", "2", "4096", "16384", "49152"] - } - } -} -_mongodb_mongod_op_latencies_latency: { - description: "A 64-bit integer giving the total combined latency in microseconds." - type: "gauge" - tags: _mongodb_metrics_tags & { - type: { - description: "Latency type." - required: true - examples: ["network_in_progress", "sleepers"] - } - } -} -_mongodb_mongod_op_latencies_ops_total: { - description: "A 64-bit integer giving the total number of operations performed on the collection since startup." - type: "gauge" - tags: _mongodb_metrics_tags & { - type: { - description: "Latency type." - required: true - examples: ["network_in_progress", "sleepers"] - } - } -} -_mongodb_mongod_storage_engine: { - description: "The name of the current storage engine." - type: "gauge" - tags: _mongodb_metrics_tags & { - engine: { - description: "Engine name." - required: true - examples: ["wiredTiger"] - } - } -} -_mongodb_mongod_wiredtiger_blockmanager_blocks_total: { - description: "Statistics on the block manager operations." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags & { - type: { - description: "Operation type." - required: true - examples: ["blocks_read", "blocks_read_mapped", "blocks_pre_loaded", "blocks_written"] - } - } -} -_mongodb_mongod_wiredtiger_blockmanager_bytes_total: { - description: "Statistics on the block manager operations." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags & { - type: { - description: "Operation type." - required: true - examples: ["bytes_read", "bytes_read_mapped", "bytes_written"] - } - } -} -_mongodb_mongod_wiredtiger_cache_bytes: { - description: "Statistics on the cache and page evictions from the cache." - relevant_when: "Storage engine is `wiredTiger`." - type: "gauge" - tags: _mongodb_metrics_tags & { - type: { - description: "Statistics type." - required: true - examples: ["total", "dirty", "internal_pages", "leaf_pages"] - } - } -} -_mongodb_mongod_wiredtiger_cache_bytes_total: { - description: "Statistics on the cache and page evictions from the cache." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags & { - type: { - description: "Statistics type." - required: true - examples: ["read", "written"] - } - } -} -_mongodb_mongod_wiredtiger_cache_evicted_total: { - description: "Statistics on the cache and page evictions from the cache." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags & { - type: { - description: "Statistics type." - required: true - examples: ["modified", "unmodified"] - } - } -} -_mongodb_mongod_wiredtiger_cache_max_bytes: { - description: "Maximum cache size." - type: "gauge" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_wiredtiger_cache_overhead_percent: { - description: "Percentage overhead." - type: "gauge" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_wiredtiger_cache_pages: { - description: "Pages in the cache." - relevant_when: "Storage engine is `wiredTiger`." - type: "gauge" - tags: _mongodb_metrics_tags & { - type: { - description: "Pages type." - required: true - examples: ["total", "dirty"] - } - } -} -_mongodb_mongod_wiredtiger_cache_pages_total: { - description: "Pages in the cache." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags & { - type: { - description: "Pages type." - required: true - examples: ["read", "write"] - } - } -} -_mongodb_mongod_wiredtiger_concurrent_transactions_available_tickets: { - description: "Information on the number of concurrent of read and write transactions allowed into the WiredTiger storage engine" - relevant_when: "Storage engine is `wiredTiger`." - type: "gauge" - tags: _mongodb_metrics_tags & { - type: { - description: "Transactions type." - required: true - examples: ["read", "write"] - } - } -} -_mongodb_mongod_wiredtiger_concurrent_transactions_out_tickets: { - description: "Information on the number of concurrent of read and write transactions allowed into the WiredTiger storage engine" - relevant_when: "Storage engine is `wiredTiger`." - type: "gauge" - tags: _mongodb_metrics_tags & { - type: { - description: "Transactions type." - required: true - examples: ["read", "write"] - } - } -} -_mongodb_mongod_wiredtiger_concurrent_transactions_total_tickets: { - description: "Information on the number of concurrent of read and write transactions allowed into the WiredTiger storage engine" - relevant_when: "Storage engine is `wiredTiger`." - type: "gauge" - tags: _mongodb_metrics_tags & { - type: { - description: "Transactions type." - required: true - examples: ["read", "write"] - } - } -} -_mongodb_mongod_wiredtiger_log_bytes_total: { - description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags & { - type: { - description: "Bytes type." - required: true - examples: ["payload", "written"] - } - } -} -_mongodb_mongod_wiredtiger_log_operations_total: { - description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags & { - type: { - description: "Operations type." - required: true - examples: ["write", "scan", "scan_double", "sync", "sync_dir", "flush"] - } - } -} -_mongodb_mongod_wiredtiger_log_records_scanned_total: { - description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags & { - type: { - description: "Scanned records type." - required: true - examples: ["compressed", "uncompressed"] - } - } -} -_mongodb_mongod_wiredtiger_log_records_total: { - description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_wiredtiger_session_open_sessions: { - description: "Open session count." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_wiredtiger_transactions_checkpoint_seconds: { - description: "Statistics on transaction checkpoints and operations." - relevant_when: "Storage engine is `wiredTiger`." - type: "gauge" - tags: _mongodb_metrics_tags & { - type: { - description: "Checkpoint type." - required: true - examples: ["min", "max"] - } - } -} -_mongodb_mongod_wiredtiger_transactions_checkpoint_seconds_total: { - description: "Statistics on transaction checkpoints and operations." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_wiredtiger_transactions_running_checkpoints: { - description: "Statistics on transaction checkpoints and operations." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_mongod_wiredtiger_transactions_total: { - description: "Statistics on transaction checkpoints and operations." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags & { - type: { - description: "Transactions type." - required: true - examples: ["begins", "checkpoints", "committed", "rolledback"] - } - } -} -_mongodb_network_bytes_total: { - description: "The number of bytes that reflects the amount of network traffic." - type: "counter" - tags: _mongodb_metrics_tags & { - state: { - description: "Bytes state." - required: true - examples: ["bytes_in", "bytes_out"] - } - } -} -_mongodb_network_metrics_num_requests_total: { - description: "The total number of distinct requests that the server has received." - type: "counter" - tags: _mongodb_metrics_tags -} -_mongodb_op_counters_repl_total: { - description: "Database replication operations by type since the mongod instance last started." - type: "counter" - tags: _mongodb_metrics_tags & { - type: { - description: "Counter type." - required: true - examples: ["insert", "query", "update", "delete", "getmore", "command"] - } - } -} -_mongodb_op_counters_total: { - description: "Database operations by type since the mongod instance last started." - type: "counter" - tags: _mongodb_metrics_tags & { - type: { - description: "Counter type." - required: true - examples: ["insert", "query", "update", "delete", "getmore", "command"] - } - } -} -_mongodb_up: { - description: "If the MongoDB server is up or not." - type: "gauge" - tags: _mongodb_metrics_tags -} - // Vector internal metrics (plus misc) _vector_api_started_total: { description: "The number of times the Vector GraphQL API has been started." @@ -1142,130 +430,8 @@ _internal_metrics_tags: { instance: _instance job: _job } -_mongodb_metrics_tags: { - endpoint: { - description: "The absolute path of the originating file." - required: true - examples: ["mongodb://localhost:27017"] - } - host: { - description: "The hostname of the MongoDB server." - required: true - examples: [_values.local_host] - } -} - -// Other helpers -_disk_device: { - description: "The disk device name." - required: true - examples: ["sda", "sda1", "dm-1"] -} -_disk_counter: { - type: "counter" - tags: _host_metrics_tags & { - collector: examples: ["disk"] - device: _disk_device - } -} -_filesystem_bytes: { - type: "gauge" - tags: _host_metrics_tags & { - collector: examples: ["filesystem"] - device: _disk_device - filesystem: { - description: "The name of the filesystem type." - required: true - examples: ["ext4", "ntfs"] - } - } -} -_loadavg: { - type: "gauge" - tags: _host_metrics_tags & { - collector: examples: ["loadavg"] - } - relevant_when: "OS is not Windows" -} -_memory_counter: { - type: "counter" - tags: _host_metrics_tags & { - collector: examples: ["memory"] - } -} -_memory_gauge: { - type: "gauge" - tags: _host_metrics_tags & { - collector: examples: ["memory"] - } -} -_memory_linux: _memory_gauge & {relevant_when: "OS is Linux"} -_memory_macos: _memory_gauge & {relevant_when: "OS is MacOS X"} -_memory_nowin: {relevant_when: "OS is not Windows"} -_network_gauge: { - type: "gauge" - tags: _host_metrics_tags & { - collector: examples: ["network"] - device: { - description: "The network interface device name." - required: true - examples: ["eth0", "enp5s3"] - } - } -} -_network_nomac: _network_gauge & {relevant_when: "OS is not MacOS"} // Helpful metrics groupings -_apache_metrics: { - apache_access_total: _apache_access_total - apache_connections: _apache_connections - apache_cpu_load: _apache_cpu_load - apache_cpu_seconds_total: _apache_cpu_seconds_total - apache_duration_seconds_total: _apache_duration_seconds_total - apache_scoreboard: _apache_scoreboard - apache_sent_bytes_total: _apache_sent_bytes_total - apache_sent_bytes_total: _apache_sent_bytes_total - apache_up: _apache_up - apache_uptime_seconds_total: _apache_uptime_seconds_total - apache_workers: _apache_workers -} - -_host_metrics: { - host_cpu_seconds_total: _host_cpu_seconds_total - host_disk_read_bytes_total: _host_disk_read_bytes_total - host_disk_reads_completed_total: _host_disk_reads_completed_total - host_disk_written_bytes_total: _host_disk_written_bytes_total - host_disk_writes_completed_total: _host_disk_writes_completed_total - host_filesystem_free_bytes: _host_filesystem_free_bytes - host_filesystem_total_bytes: _host_filesystem_total_bytes - host_filesystem_used_bytes: _host_filesystem_used_bytes - host_load1: _host_load1 - host_load5: _host_load5 - host_load15: _host_load15 - host_memory_active_bytes: _host_memory_active_bytes - host_memory_available_bytes: _host_memory_available_bytes - host_memory_buffers_bytes: _host_memory_buffers_bytes - host_memory_cached_bytes: _host_memory_cached_bytes - host_memory_free_bytes: _host_memory_free_bytes - host_memory_inactive_bytes: _host_memory_inactive_bytes - host_memory_shared_bytes: _host_memory_shared_bytes - host_memory_swap_free_bytes: _host_memory_swap_free_bytes - host_memory_swapped_in_bytes_total: _host_memory_swapped_in_bytes_total - host_memory_swapped_out_bytes_total: _host_memory_swapped_out_bytes_total - host_memory_swap_total_bytes: _host_memory_swap_total_bytes - host_memory_swap_used_bytes: _host_memory_swap_used_bytes - host_memory_total_bytes: _host_memory_total_bytes - host_memory_used_bytes: _host_memory_used_bytes - host_memory_wired_bytes: _host_memory_wired_bytes - host_network_receive_bytes_total: _host_network_receive_bytes_total - host_network_receive_errs_total: _host_network_receive_errs_total - host_network_receive_packets_total: _host_network_receive_packets_total - host_network_transmit_bytes_total: _host_network_transmit_bytes_total - host_network_transmit_errs_total: _host_network_transmit_errs_total - host_network_transmit_packets_drop_total: _host_network_transmit_packets_drop_total - host_network_transmit_packets_total: _host_network_transmit_packets_total -} - _internal_metrics: { vector_config_load_errors_total: _vector_config_load_errors_total vector_quit_total: _vector_quit_total @@ -1276,75 +442,6 @@ _internal_metrics: { vector_stopped_total: _vector_stopped_total } -_mongodb_metrics: { - mongodb_assets_total: _mongodb_assets_total - mongodb_bson_parse_error_total: _mongodb_bson_parse_error_total - mongodb_connections: _mongodb_connections - mongodb_extra_info_heap_usage_bytes: _mongodb_extra_info_heap_usage_bytes - mongodb_extra_info_page_faults: _mongodb_extra_info_page_faults - mongodb_instance_local_time: _mongodb_instance_local_time - mongodb_instance_uptime_estimate_seconds_total: _mongodb_instance_uptime_estimate_seconds_total - mongodb_instance_uptime_seconds_total: _mongodb_instance_uptime_seconds_total - mongodb_memory: _mongodb_memory - mongodb_mongod_global_lock_active_clients: _mongodb_mongod_global_lock_active_clients - mongodb_mongod_global_lock_current_queue: _mongodb_mongod_global_lock_current_queue - mongodb_mongod_locks_time_acquiring_global_seconds_total: _mongodb_mongod_locks_time_acquiring_global_seconds_total - mongodb_mongod_metrics_cursor_open: _mongodb_mongod_metrics_cursor_open - mongodb_mongod_metrics_cursor_timed_out_total: _mongodb_mongod_metrics_cursor_timed_out_total - mongodb_mongod_metrics_document_total: _mongodb_mongod_metrics_document_total - mongodb_mongod_metrics_get_last_error_wtime_num: _mongodb_mongod_metrics_get_last_error_wtime_num - mongodb_mongod_metrics_get_last_error_wtime_seconds_total: _mongodb_mongod_metrics_get_last_error_wtime_seconds_total - mongodb_mongod_metrics_get_last_error_wtimeouts_total: _mongodb_mongod_metrics_get_last_error_wtimeouts_total - mongodb_mongod_metrics_operation_total: _mongodb_mongod_metrics_operation_total - mongodb_mongod_metrics_query_executor_total: _mongodb_mongod_metrics_query_executor_total - mongodb_mongod_metrics_record_moves_total: _mongodb_mongod_metrics_record_moves_total - mongodb_mongod_metrics_repl_apply_batches_num_total: _mongodb_mongod_metrics_repl_apply_batches_num_total - mongodb_mongod_metrics_repl_apply_batches_seconds_total: _mongodb_mongod_metrics_repl_apply_batches_seconds_total - mongodb_mongod_metrics_repl_apply_ops_total: _mongodb_mongod_metrics_repl_apply_ops_total - mongodb_mongod_metrics_repl_buffer_count: _mongodb_mongod_metrics_repl_buffer_count - mongodb_mongod_metrics_repl_buffer_max_size_bytes_total: _mongodb_mongod_metrics_repl_buffer_max_size_bytes_total - mongodb_mongod_metrics_repl_buffer_size_bytes: _mongodb_mongod_metrics_repl_buffer_size_bytes - mongodb_mongod_metrics_repl_executor_queue: _mongodb_mongod_metrics_repl_executor_queue - mongodb_mongod_metrics_repl_executor_unsignaled_events: _mongodb_mongod_metrics_repl_executor_unsignaled_events - mongodb_mongod_metrics_repl_network_bytes_total: _mongodb_mongod_metrics_repl_network_bytes_total - mongodb_mongod_metrics_repl_network_getmores_num_total: _mongodb_mongod_metrics_repl_network_getmores_num_total - mongodb_mongod_metrics_repl_network_getmores_seconds_total: _mongodb_mongod_metrics_repl_network_getmores_seconds_total - mongodb_mongod_metrics_repl_network_ops_total: _mongodb_mongod_metrics_repl_network_ops_total - mongodb_mongod_metrics_repl_network_readers_created_total: _mongodb_mongod_metrics_repl_network_readers_created_total - mongodb_mongod_metrics_ttl_deleted_documents_total: _mongodb_mongod_metrics_ttl_deleted_documents_total - mongodb_mongod_metrics_ttl_passes_total: _mongodb_mongod_metrics_ttl_passes_total - mongodb_mongod_op_latencies_histogram: _mongodb_mongod_op_latencies_histogram - mongodb_mongod_op_latencies_latency: _mongodb_mongod_op_latencies_latency - mongodb_mongod_op_latencies_ops_total: _mongodb_mongod_op_latencies_ops_total - mongodb_mongod_storage_engine: _mongodb_mongod_storage_engine - mongodb_mongod_wiredtiger_blockmanager_blocks_total: _mongodb_mongod_wiredtiger_blockmanager_blocks_total - mongodb_mongod_wiredtiger_blockmanager_bytes_total: _mongodb_mongod_wiredtiger_blockmanager_bytes_total - mongodb_mongod_wiredtiger_cache_bytes: _mongodb_mongod_wiredtiger_cache_bytes - mongodb_mongod_wiredtiger_cache_bytes_total: _mongodb_mongod_wiredtiger_cache_bytes_total - mongodb_mongod_wiredtiger_cache_evicted_total: _mongodb_mongod_wiredtiger_cache_evicted_total - mongodb_mongod_wiredtiger_cache_max_bytes: _mongodb_mongod_wiredtiger_cache_max_bytes - mongodb_mongod_wiredtiger_cache_overhead_percent: _mongodb_mongod_wiredtiger_cache_overhead_percent - mongodb_mongod_wiredtiger_cache_pages: _mongodb_mongod_wiredtiger_cache_pages - mongodb_mongod_wiredtiger_cache_pages_total: _mongodb_mongod_wiredtiger_cache_pages_total - mongodb_mongod_wiredtiger_concurrent_transactions_available_tickets: _mongodb_mongod_wiredtiger_concurrent_transactions_available_tickets - mongodb_mongod_wiredtiger_concurrent_transactions_out_tickets: _mongodb_mongod_wiredtiger_concurrent_transactions_out_tickets - mongodb_mongod_wiredtiger_concurrent_transactions_total_tickets: _mongodb_mongod_wiredtiger_concurrent_transactions_total_tickets - mongodb_mongod_wiredtiger_log_bytes_total: _mongodb_mongod_wiredtiger_log_bytes_total - mongodb_mongod_wiredtiger_log_operations_total: _mongodb_mongod_wiredtiger_log_operations_total - mongodb_mongod_wiredtiger_log_records_scanned_total: _mongodb_mongod_wiredtiger_log_records_scanned_total - mongodb_mongod_wiredtiger_log_records_total: _mongodb_mongod_wiredtiger_log_records_total - mongodb_mongod_wiredtiger_session_open_sessions: _mongodb_mongod_wiredtiger_session_open_sessions - mongodb_mongod_wiredtiger_transactions_checkpoint_seconds: _mongodb_mongod_wiredtiger_transactions_checkpoint_seconds - mongodb_mongod_wiredtiger_transactions_checkpoint_seconds_total: _mongodb_mongod_wiredtiger_transactions_checkpoint_seconds_total - mongodb_mongod_wiredtiger_transactions_running_checkpoints: _mongodb_mongod_wiredtiger_transactions_running_checkpoints - mongodb_mongod_wiredtiger_transactions_total: _mongodb_mongod_wiredtiger_transactions_total - mongodb_network_bytes_total: _mongodb_network_bytes_total - mongodb_network_metrics_num_requests_total: _mongodb_network_metrics_num_requests_total - mongodb_op_counters_repl_total: _mongodb_op_counters_repl_total - mongodb_op_counters_total: _mongodb_op_counters_total - mongodb_up: _mongodb_up -} - _prometheus_metrics: { vector_events_processed_total: _vector_events_processed_total vector_http_error_response_total: _vector_http_error_response_total From abc31e62ac39b14f67b3f87e412fae161c5ae5f8 Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Thu, 12 Nov 2020 13:00:21 -0800 Subject: [PATCH 09/20] Move Docker metrics out of metrics.cue Signed-off-by: Luc Perkins --- docs/reference/components.cue | 71 +++++++++++++++++++ .../components/sources/docker_logs.cue | 35 +++++++-- docs/reference/metrics.cue | 39 ---------- 3 files changed, 101 insertions(+), 44 deletions(-) diff --git a/docs/reference/components.cue b/docs/reference/components.cue index 8a4a096c4cf56..5af6048fe5275 100644 --- a/docs/reference/components.cue +++ b/docs/reference/components.cue @@ -902,6 +902,77 @@ components: { // Default metrics for each component vector_events_processed_total: _vector_events_processed_total vector_processed_bytes_total: _vector_processed_bytes_total + + // Helpful tag groupings + _component_tags: _internal_metrics_tags & { + component_kind: _component_kind + component_name: _component_name + component_type: _component_type + } + + _internal_metrics_tags: { + instance: _instance + job: _job + } + + // All available tags + _collector: { + description: "Which collector this metric comes from." + required: true + } + _component_kind: { + description: "The component's kind (options are `source`, `sink`, or `transform`)." + required: true + options: ["sink", "source", "transform"] + } + _component_name: { + description: "The name of the component as specified in the Vector configuration." + required: true + examples: ["file_source", "splunk_sink"] + } + _component_type: { + description: "The type of component (source, transform, or sink)." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + _endpoint: { + description: "The absolute path of originating file." + required: true + examples: ["http://localhost:8080/server-status?auto"] + } + _error_type: { + description: "The type of the error" + required: true + options: [ + "field_missing", + "invalid_metric", + "mapping_failed", + "match_failed", + "parse_failed", + "render_error", + "type_conversion_failed", + "value_invalid", + ] + } + _file: { + description: "The file that produced the error" + required: false + } + _host: { + description: "The hostname of the originating system." + required: true + examples: [_values.local_host] + } + _instance: { + description: "The Vector instance identified by host and port." + required: true + examples: [_values.instance] + } + _job: { + description: "The name of the job producing Vector metrics." + required: true + default: "vector" + } } }} } diff --git a/docs/reference/components/sources/docker_logs.cue b/docs/reference/components/sources/docker_logs.cue index 9f13e29d07b46..45ef06ef14978 100644 --- a/docs/reference/components/sources/docker_logs.cue +++ b/docs/reference/components/sources/docker_logs.cue @@ -267,10 +267,35 @@ components: sources: docker_logs: { } telemetry: metrics: { - vector_communication_errors_total: _vector_communication_errors_total - vector_container_events_processed_total: _vector_container_events_processed_total - vector_container_metadata_fetch_errors_total: _vector_container_metadata_fetch_errors_total - vector_containers_watched_total: _vector_containers_watched_total - vector_logging_driver_errors_total: _vector_logging_driver_errors_total + vector_communication_errors_total: { + description: "The total number of errors stemming from communication with the Docker daemon." + type: "counter" + tags: telemetry.metrics._component_tags + } + vector_container_events_processed_total: { + description: "The total number of container events processed." + type: "counter" + tags: telemetry.metrics._component_tags + } + vector_container_metadata_fetch_errors_total: { + description: "The total number of errors caused by failure to fetch container metadata." + type: "counter" + tags: telemetry.metrics._component_tags + } + vector_containers_unwatched_total: { + description: "The total number of times Vector stopped watching for container logs." + type: "counter" + tags: telemetry.metrics._component_tags + } + vector_containers_watched_total: { + description: "The total number of times Vector started watching for container logs." + type: "counter" + tags: telemetry.metrics._component_tags + } + vector_logging_driver_errors_total: { + description: "The total number of logging driver errors encountered caused by not using either the `jsonfile` or `journald` driver." + type: "counter" + tags: telemetry.metrics._component_tags + } } } diff --git a/docs/reference/metrics.cue b/docs/reference/metrics.cue index de2f3d324d8e9..6b46f59c3f2dd 100644 --- a/docs/reference/metrics.cue +++ b/docs/reference/metrics.cue @@ -1,42 +1,5 @@ package metadata -// Container metrics -_vector_communication_errors_total: { - description: "The total number of errors stemming from communication with the Docker daemon." - type: "counter" - tags: _component_tags -} - -_vector_container_events_processed_total: { - description: "The total number of container events processed." - type: "counter" - tags: _component_tags -} - -_vector_container_metadata_fetch_errors_total: { - description: "The total number of errors caused by failure to fetch container metadata." - type: "counter" - tags: _component_tags -} - -_vector_containers_unwatched_total: { - description: "The total number of times Vector stopped watching for container logs." - type: "counter" - tags: _component_tags -} - -_vector_containers_watched_total: { - description: "The total number of times Vector started watching for container logs." - type: "counter" - tags: _component_tags -} - -_vector_logging_driver_errors_total: { - description: "The total number of logging driver errors encountered caused by not using either the `jsonfile` or `journald` driver." - type: "counter" - tags: _component_tags -} - // Kubernetes metrics _vector_k8s_docker_format_parse_failures_total: { description: "The total number of failures to parse a message as a JSON object." @@ -410,8 +373,6 @@ _component_tags: _internal_metrics_tags & { component_kind: _component_kind component_name: _component_name component_type: _component_type - instance: _instance - job: _job } _apache_metrics_tags: { From 02436f2b713d1895c72c58df743ff475de979586 Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Thu, 12 Nov 2020 13:01:29 -0800 Subject: [PATCH 10/20] Move k8s metrics out of metrics.cue Signed-off-by: Luc Perkins --- .../components/sources/kubernetes_logs.cue | 13 +++++++++++-- docs/reference/metrics.cue | 13 ------------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/docs/reference/components/sources/kubernetes_logs.cue b/docs/reference/components/sources/kubernetes_logs.cue index f4ef610e61d92..f5c92420dfb25 100644 --- a/docs/reference/components/sources/kubernetes_logs.cue +++ b/docs/reference/components/sources/kubernetes_logs.cue @@ -301,7 +301,16 @@ components: sources: kubernetes_logs: { } telemetry: metrics: { - vector_k8s_docker_format_parse_failures_total: _vector_k8s_docker_format_parse_failures_total - vector_k8s_event_annotation_failures_total: _vector_k8s_event_annotation_failures_total + vector_k8s_docker_format_parse_failures_total: { + description: "The total number of failures to parse a message as a JSON object." + type: "counter" + tags: telemetry.metrics._component_tags + } + + vector_k8s_event_annotation_failures_total: { + description: "The total number of failures to annotate Vector events with Kubernetes Pod metadata." + type: "counter" + tags: telemetry.metrics._component_tags + } } } diff --git a/docs/reference/metrics.cue b/docs/reference/metrics.cue index 6b46f59c3f2dd..4e022f8ddb0c9 100644 --- a/docs/reference/metrics.cue +++ b/docs/reference/metrics.cue @@ -1,18 +1,5 @@ package metadata -// Kubernetes metrics -_vector_k8s_docker_format_parse_failures_total: { - description: "The total number of failures to parse a message as a JSON object." - type: "counter" - tags: _component_tags -} - -_vector_k8s_event_annotation_failures_total: { - description: "The total number of failures to annotate Vector events with Kubernetes Pod metadata." - type: "counter" - tags: _component_tags -} - // Vector internal metrics (plus misc) _vector_api_started_total: { description: "The number of times the Vector GraphQL API has been started." From cdcf9af95fed88e59ffaac8b4b23fb93787c3b91 Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Thu, 12 Nov 2020 13:28:21 -0800 Subject: [PATCH 11/20] Migrate internal metrics out of metrics.cue Signed-off-by: Luc Perkins --- .../components/sources/internal_metrics.cue | 57 ++++++++++++++++++- docs/reference/metrics.cue | 10 ---- 2 files changed, 56 insertions(+), 11 deletions(-) diff --git a/docs/reference/components/sources/internal_metrics.cue b/docs/reference/components/sources/internal_metrics.cue index dee8f52f6684d..442724170eead 100644 --- a/docs/reference/components/sources/internal_metrics.cue +++ b/docs/reference/components/sources/internal_metrics.cue @@ -40,5 +40,60 @@ components: sources: internal_metrics: { warnings: [] } - output: metrics: _internal_metrics + output: metrics: { + // Default internal metrics tags + _internal_metrics_tags: { + instance: { + description: "The Vector instance identified by host and port." + required: true + examples: [_values.instance] + } + job: { + description: "The name of the job producing Vector metrics." + required: true + default: "vector" + } + } + + vector_config_load_errors_total: { + description: "The total number of errors loading the Vector configuration." + type: "counter" + tags: _internal_metrics_tags + } + vector_connection_errors_total: { + description: "The total number of connection errors for this Vector instance." + type: "counter" + tags: _internal_metrics_tags + } + vector_quit_total: { + description: "The total number of times the Vector instance has quit." + type: "counter" + tags: _internal_metrics_tags + } + vector_recover_errors_total: { + description: "The total number of errors caused by Vector failing to recover from a failed reload." + type: "counter" + tags: _internal_metrics_tags + } + vector_reload_errors_total: { + description: "The total number of errors encountered when reloading Vector." + type: "counter" + tags: _internal_metrics_tags + } + vector_reloaded_total: { + description: "The total number of times the Vector instance has been reloaded." + type: "counter" + tags: _internal_metrics_tags + } + vector_started_total: { + description: "The total number of times the Vector instance has been started." + type: "counter" + tags: _internal_metrics_tags + } + vector_stopped_total: { + description: "The total number of times the Vector instance has been stopped." + type: "counter" + tags: _internal_metrics_tags + } + } } diff --git a/docs/reference/metrics.cue b/docs/reference/metrics.cue index 4e022f8ddb0c9..d0631140b35b3 100644 --- a/docs/reference/metrics.cue +++ b/docs/reference/metrics.cue @@ -380,16 +380,6 @@ _internal_metrics_tags: { } // Helpful metrics groupings -_internal_metrics: { - vector_config_load_errors_total: _vector_config_load_errors_total - vector_quit_total: _vector_quit_total - vector_recover_errors_total: _vector_recover_errors_total - vector_reload_errors_total: _vector_reload_errors_total - vector_reloaded_total: _vector_reloaded_total - vector_started_total: _vector_started_total - vector_stopped_total: _vector_stopped_total -} - _prometheus_metrics: { vector_events_processed_total: _vector_events_processed_total vector_http_error_response_total: _vector_http_error_response_total From 747d24e44985ebc162d00ebe12838ec316ec5f22 Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Thu, 12 Nov 2020 13:40:00 -0800 Subject: [PATCH 12/20] Remove metrics.cue file Signed-off-by: Luc Perkins --- docs/reference/components.cue | 291 +++++++++++++ docs/reference/components/kafka.cue | 4 +- .../components/sinks/aws_cloudwatch_logs.cue | 2 +- docs/reference/components/sinks/console.cue | 2 +- .../components/sinks/elasticsearch.cue | 2 +- docs/reference/components/sinks/http.cue | 2 +- .../components/sinks/prometheus_exporter.cue | 2 - .../components/sinks/sematext_metrics.cue | 2 +- .../reference/components/sinks/splunk_hec.cue | 10 +- docs/reference/components/sinks/statsd.cue | 2 +- docs/reference/components/sinks/vector.cue | 2 +- .../components/sources/apache_metrics.cue | 14 + .../sources/aws_kinesis_firehose.cue | 4 +- docs/reference/components/sources/file.cue | 20 +- .../components/sources/host_metrics.cue | 12 + docs/reference/components/sources/http.cue | 2 +- .../components/sources/internal_metrics.cue | 5 + .../reference/components/sources/journald.cue | 4 +- docs/reference/components/sources/logplex.cue | 4 +- .../components/sources/mongodb_metrics.cue | 7 +- .../components/sources/prometheus.cue | 2 - docs/reference/components/sources/socket.cue | 2 +- .../components/sources/splunk_hec.cue | 10 +- docs/reference/components/sources/statsd.cue | 6 +- docs/reference/components/sources/stdin.cue | 2 +- docs/reference/components/sources/syslog.cue | 4 +- docs/reference/components/sources/vector.cue | 2 +- .../components/transforms/ansi_stripper.cue | 2 +- ...ws_cloudwatch_logs_subscription_parser.cue | 2 +- .../transforms/aws_ec2_metadata.cue | 4 +- .../components/transforms/dedupe.cue | 2 +- .../components/transforms/grok_parser.cue | 2 +- .../components/transforms/logfmt_parser.cue | 2 +- docs/reference/components/transforms/lua.cue | 4 +- .../components/transforms/metric_to_log.cue | 2 +- .../components/transforms/reduce.cue | 2 +- .../components/transforms/regex_parser.cue | 2 +- .../reference/components/transforms/remap.cue | 2 +- .../components/transforms/sampler.cue | 2 +- .../reference/components/transforms/split.cue | 2 +- .../components/transforms/swimlanes.cue | 2 +- .../transforms/tag_cardinality_limit.cue | 4 +- .../components/transforms/tokenizer.cue | 2 +- docs/reference/metrics.cue | 391 ------------------ 44 files changed, 389 insertions(+), 461 deletions(-) delete mode 100644 docs/reference/metrics.cue diff --git a/docs/reference/components.cue b/docs/reference/components.cue index 5af6048fe5275..d821f6b9c804a 100644 --- a/docs/reference/components.cue +++ b/docs/reference/components.cue @@ -903,6 +903,297 @@ components: { vector_events_processed_total: _vector_events_processed_total vector_processed_bytes_total: _vector_processed_bytes_total + // Reusable metric definitions + _vector_auto_concurrency_averaged_rtt: { + description: "The average round-trip time (RTT) from the HTTP sink across the current window." + type: "histogram" + tags: _internal_metrics_tags + } + _vector_auto_concurrency_in_flight: { + description: "The number of outbound requests from the HTTP sink currently awaiting a response." + type: "histogram" + tags: _internal_metrics_tags + } + _vector_auto_concurrency_limit: { + description: "The concurrency limit that the auto-concurrency feature has decided on for this current window." + type: "histogram" + tags: _internal_metrics_tags + } + _vector_auto_concurrency_observed_rtt: { + description: "The observed round-trip time (RTT) for requests from this HTTP sink." + type: "histogram" + tags: _internal_metrics_tags + } + _vector_checkpoint_write_errors_total: { + description: "The total number of errors writing checkpoints." + type: "counter" + tags: _internal_metrics_tags + } + _vector_checkpoints_total: { + description: "The total number of files checkpointed." + type: "counter" + tags: _internal_metrics_tags + } + _vector_checksum_errors: { + description: "The total number of errors identifying files via checksum." + type: "counter" + tags: _internal_metrics_tags & { + file: _file + } + } + _vector_events_discarded_total: { + description: "The total number of events discarded by this component." + type: "counter" + tags: _internal_metrics_tags + } + _vector_events_processed_total: { + description: "The total number of events processed by this component." + type: "counter" + tags: _component_tags & { + file: _file + } + } + _vector_file_delete_errors: { + description: "The total number of failures to delete a file." + type: "counter" + tags: _internal_metrics_tags & { + file: _file + } + } + _vector_file_watch_errors: { + description: "The total number of errors caused by failure to watch a file." + type: "counter" + tags: _internal_metrics_tags & { + file: _file + } + } + _vector_files_added: { + description: "The total number of files Vector has found to watch." + type: "counter" + tags: _internal_metrics_tags & { + file: _file + } + } + _vector_files_deleted: { + description: "The total number of files deleted." + type: "counter" + tags: _internal_metrics_tags & { + file: _file + } + } + _vector_files_resumed: { + description: "The total number of times Vector has resumed watching a file." + type: "counter" + tags: _internal_metrics_tags & { + file: _file + } + } + _vector_files_unwatched: { + description: "The total number of times Vector has stopped watching a file." + type: "counter" + tags: _internal_metrics_tags & { + file: _file + } + } + _vector_fingerprint_read_errors: { + description: "The total number of times failing to read a file for fingerprinting." + type: "counter" + tags: _internal_metrics_tags & { + file: _file + } + } + _vector_http_bad_requests_total: { + description: "The total number of HTTP `400 Bad Request` errors encountered." + type: "counter" + tags: _internal_metrics_tags + } + _vector_http_error_response_total: { + description: "The total number of HTTP error responses for this component." + type: "counter" + tags: _internal_metrics_tags + } + _vector_http_request_errors_total: { + description: "The total number of HTTP request errors for this component." + type: "counter" + tags: _internal_metrics_tags + } + _vector_http_requests_total: { + description: "The total number of HTTP requests issued by this component." + type: "counter" + tags: _component_tags + } + _vector_memory_used: { + description: "The total memory currently being used by Vector (in bytes)." + type: "gauge" + tags: _internal_metrics_tags + } + _vector_missing_keys_total: { + description: "The total number of events dropped due to keys missing from the event." + type: "counter" + tags: _internal_metrics_tags + } + _vector_open_connections: { + description: "The number of current open connections to Vector." + type: "gauge" + tags: _internal_metrics_tags + } + _vector_parse_errors_total: { + description: "The total number of errors parsing Prometheus metrics." + type: "counter" + tags: _internal_metrics_tags + } + _vector_processed_bytes_total: { + description: "The total number of bytes processed by the component." + type: "counter" + tags: _component_tags + } + _vector_processing_errors_total: { + description: "The total number of processing errors encountered by this component." + type: "counter" + tags: _component_tags & { + error_type: _error_type + } + } + _vector_protobuf_decode_errors_total: { + description: "The total number of [Protocol Buffers](\(urls.protobuf)) errors thrown during communication between Vector instances." + type: "counter" + tags: _component_tags + } + _vector_request_duration_nanoseconds: { + description: "The request duration for this component (in nanoseconds)." + type: "histogram" + tags: _component_tags + } + _vector_request_read_errors_total: { + description: "The total number of request read errors for this component." + type: "counter" + tags: _component_tags + } + _vector_requests_completed_total: { + description: "The total number of requests completed by this component." + type: "counter" + tags: _component_tags + } + _vector_requests_received_total: { + description: "The total number of requests received by this component." + type: "counter" + tags: _component_tags + } + _vector_timestamp_parse_errors_total: { + description: "The total number of errors encountered parsing [RFC3339](\(urls.rfc_3339)) timestamps." + type: "counter" + tags: _component_tags + } + _vector_uptime_seconds: { + description: "The total number of seconds the Vector instance has been up." + type: "gauge" + tags: _component_tags + } + + // Splunk + _vector_encode_errors_total: { + description: "The total number of errors encoding [Splunk HEC](\(urls.splunk_hec_protocol)) events to JSON for this `splunk_hec` sink." + type: "counter" + tags: _component_tags + } + _vector_source_missing_keys_total: { + description: "The total number of errors rendering the template for this source." + type: "counter" + tags: _component_tags + } + _vector_sourcetype_missing_keys_total: { + description: "The total number of errors rendering the template for this sourcetype." + type: "counter" + tags: _component_tags + } + + // Vector instance metrics + _vector_config_load_errors_total: { + description: "The total number of errors loading the Vector configuration." + type: "counter" + tags: _internal_metrics_tags + } + _vector_connection_errors_total: { + description: "The total number of connection errors for this Vector instance." + type: "counter" + tags: _internal_metrics_tags + } + _vector_quit_total: { + description: "The total number of times the Vector instance has quit." + type: "counter" + tags: _internal_metrics_tags + } + _vector_recover_errors_total: { + description: "The total number of errors caused by Vector failing to recover from a failed reload." + type: "counter" + tags: _internal_metrics_tags + } + _vector_reload_errors_total: { + description: "The total number of errors encountered when reloading Vector." + type: "counter" + tags: _internal_metrics_tags + } + _vector_reloaded_total: { + description: "The total number of times the Vector instance has been reloaded." + type: "counter" + tags: _internal_metrics_tags + } + _vector_started_total: { + description: "The total number of times the Vector instance has been started." + type: "counter" + tags: _internal_metrics_tags + } + _vector_stopped_total: { + description: "The total number of times the Vector instance has been stopped." + type: "counter" + tags: _internal_metrics_tags + } + + // Windows metrics + _windows_service_does_not_exist: { + description: """ + The total number of errors raised due to the Windows service not + existing. + """ + type: "counter" + tags: _internal_metrics_tags + } + _windows_service_install: { + description: """ + The total number of times the Windows service has been installed. + """ + type: "counter" + tags: _internal_metrics_tags + } + _windows_service_restart: { + description: """ + The total number of times the Windows service has been restarted. + """ + type: "counter" + tags: _internal_metrics_tags + } + _windows_service_start: { + description: """ + The total number of times the Windows service has been started. + """ + type: "counter" + tags: _internal_metrics_tags + } + _windows_service_stop: { + description: """ + The total number of times the Windows service has been stopped. + """ + type: "counter" + tags: _internal_metrics_tags + } + _windows_service_uninstall: { + description: """ + The total number of times the Windows service has been uninstalled. + """ + type: "counter" + tags: _internal_metrics_tags + } + // Helpful tag groupings _component_tags: _internal_metrics_tags & { component_kind: _component_kind diff --git a/docs/reference/components/kafka.cue b/docs/reference/components/kafka.cue index ff01043f02c20..cb1516c121d0a 100644 --- a/docs/reference/components/kafka.cue +++ b/docs/reference/components/kafka.cue @@ -93,12 +93,12 @@ components: _kafka: { vector_consumer_offset_updates_failed_total: { description: "The total number of failures to update a Kafka consumer offset." type: "counter" - tags: _component_tags + tags: telemetry.metrics._component_tags } vector_events_failed_total: { description: "The total number of failures to read a Kafka message." type: "counter" - tags: _component_tags + tags: telemetry.metrics._component_tags } } } diff --git a/docs/reference/components/sinks/aws_cloudwatch_logs.cue b/docs/reference/components/sinks/aws_cloudwatch_logs.cue index ef5ba061fff55..866df90cead9c 100644 --- a/docs/reference/components/sinks/aws_cloudwatch_logs.cue +++ b/docs/reference/components/sinks/aws_cloudwatch_logs.cue @@ -121,6 +121,6 @@ components: sinks: aws_cloudwatch_logs: { } telemetry: metrics: { - processing_errors_total: _vector_processing_errors_total + processing_errors_total: telemetry.metrics._vector_processing_errors_total } } diff --git a/docs/reference/components/sinks/console.cue b/docs/reference/components/sinks/console.cue index 77a77a36d1be5..e2cd0769d0a56 100644 --- a/docs/reference/components/sinks/console.cue +++ b/docs/reference/components/sinks/console.cue @@ -81,6 +81,6 @@ components: sinks: console: { } telemetry: metrics: { - vector_processing_errors_total: _vector_processing_errors_total + vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total } } diff --git a/docs/reference/components/sinks/elasticsearch.cue b/docs/reference/components/sinks/elasticsearch.cue index 51a964a4d432e..927af99c205eb 100644 --- a/docs/reference/components/sinks/elasticsearch.cue +++ b/docs/reference/components/sinks/elasticsearch.cue @@ -241,6 +241,6 @@ components: sinks: elasticsearch: { } telemetry: metrics: { - vector_missing_keys_total: _vector_missing_keys_total + vector_missing_keys_total: telemetry.metrics._vector_missing_keys_total } } diff --git a/docs/reference/components/sinks/http.cue b/docs/reference/components/sinks/http.cue index b6e86f47c4306..0f1624d509e06 100644 --- a/docs/reference/components/sinks/http.cue +++ b/docs/reference/components/sinks/http.cue @@ -121,6 +121,6 @@ components: sinks: http: { } telemetry: metrics: { - vector_http_bad_requests_total: _vector_http_bad_requests_total + vector_http_bad_requests_total: telemetry.metrics._vector_http_bad_requests_total } } diff --git a/docs/reference/components/sinks/prometheus_exporter.cue b/docs/reference/components/sinks/prometheus_exporter.cue index 316f467e14d70..f33630bed7ba3 100644 --- a/docs/reference/components/sinks/prometheus_exporter.cue +++ b/docs/reference/components/sinks/prometheus_exporter.cue @@ -342,6 +342,4 @@ components: sinks: prometheus_exporter: { """ } } - - telemetry: metrics: _prometheus_metrics } diff --git a/docs/reference/components/sinks/sematext_metrics.cue b/docs/reference/components/sinks/sematext_metrics.cue index 2e25a0af149a6..c6e0fa45b9bc3 100644 --- a/docs/reference/components/sinks/sematext_metrics.cue +++ b/docs/reference/components/sinks/sematext_metrics.cue @@ -80,6 +80,6 @@ components: sinks: sematext_metrics: { } telemetry: metrics: { - vector_processing_errors_total: _vector_processing_errors_total + vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total } } diff --git a/docs/reference/components/sinks/splunk_hec.cue b/docs/reference/components/sinks/splunk_hec.cue index 31ea788d8ef24..bf407bfc219e1 100644 --- a/docs/reference/components/sinks/splunk_hec.cue +++ b/docs/reference/components/sinks/splunk_hec.cue @@ -167,10 +167,10 @@ components: sinks: splunk_hec: { } telemetry: metrics: { - vector_encode_errors_total: _vector_encode_errors_total - vector_http_request_errors_total: _vector_http_request_errors_total - vector_http_requests_total: _vector_http_requests_total - vector_source_missing_keys_total: _vector_source_missing_keys_total - vector_sourcetype_missing_keys_total: _vector_sourcetype_missing_keys_total + vector_encode_errors_total: telemetry.metrics._vector_encode_errors_total + vector_http_request_errors_total: telemetry.metrics._vector_http_request_errors_total + vector_http_requests_total: telemetry.metrics._vector_http_requests_total + vector_source_missing_keys_total: telemetry.metrics._vector_source_missing_keys_total + vector_sourcetype_missing_keys_total: telemetry.metrics._vector_sourcetype_missing_keys_total } } diff --git a/docs/reference/components/sinks/statsd.cue b/docs/reference/components/sinks/statsd.cue index 02fa2fff7d803..5b884d803ad6f 100644 --- a/docs/reference/components/sinks/statsd.cue +++ b/docs/reference/components/sinks/statsd.cue @@ -65,6 +65,6 @@ components: sinks: statsd: { } telemetry: metrics: { - vector_processing_errors_total: _vector_processing_errors_total + vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total } } diff --git a/docs/reference/components/sinks/vector.cue b/docs/reference/components/sinks/vector.cue index 8d3659195743a..8fff41c9c455b 100644 --- a/docs/reference/components/sinks/vector.cue +++ b/docs/reference/components/sinks/vector.cue @@ -86,6 +86,6 @@ components: sinks: vector: { how_it_works: components.sources.vector.how_it_works telemetry: metrics: { - vector_protobuf_decode_errors_total: _vector_protobuf_decode_errors_total + vector_protobuf_decode_errors_total: telemetry.metrics._vector_protobuf_decode_errors_total } } diff --git a/docs/reference/components/sources/apache_metrics.cue b/docs/reference/components/sources/apache_metrics.cue index 998522224732e..224c3876f4881 100644 --- a/docs/reference/components/sources/apache_metrics.cue +++ b/docs/reference/components/sources/apache_metrics.cue @@ -112,6 +112,20 @@ components: sources: apache_metrics: { } output: metrics: { + // Default Apache metrics tags + _apache_metrics_tags: { + endpoint: { + description: "The absolute path of originating file." + required: true + examples: ["http://localhost:8080/server-status?auto"] + } + host: { + description: "The hostname of the Apache HTTP server." + required: true + examples: [_values.local_host] + } + } + apache_access_total: { description: "The total number of time the Apache server has been accessed." relevant_when: "`ExtendedStatus On`" diff --git a/docs/reference/components/sources/aws_kinesis_firehose.cue b/docs/reference/components/sources/aws_kinesis_firehose.cue index 4fb9eead2b7f5..41762c731cc8e 100644 --- a/docs/reference/components/sources/aws_kinesis_firehose.cue +++ b/docs/reference/components/sources/aws_kinesis_firehose.cue @@ -189,7 +189,7 @@ components: sources: aws_kinesis_firehose: { } telemetry: metrics: { - vector_request_read_errors_total: _vector_request_read_errors_total - vector_requests_received_total: _vector_requests_received_total + vector_request_read_errors_total: telemetry.metrics._vector_request_read_errors_total + vector_requests_received_total: telemetry.metrics._vector_requests_received_total } } diff --git a/docs/reference/components/sources/file.cue b/docs/reference/components/sources/file.cue index 705fcafd0f8a8..e4c44c0fc0a24 100644 --- a/docs/reference/components/sources/file.cue +++ b/docs/reference/components/sources/file.cue @@ -517,15 +517,15 @@ components: sources: file: { } telemetry: metrics: { - vector_checkpoint_write_errors_total: _vector_checkpoint_write_errors_total - vector_checkpoints_total: _vector_checkpoints_total - vector_checksum_errors: _vector_checksum_errors - vector_file_delete_errors: _vector_file_delete_errors - vector_file_watch_errors: _vector_file_watch_errors - vector_files_added: _vector_files_added - vector_files_deleted: _vector_files_deleted - vector_files_resumed: _vector_files_resumed - vector_files_unwatched: _vector_files_unwatched - vector_fingerprint_read_errors: _vector_fingerprint_read_errors + vector_checkpoint_write_errors_total: telemetry.metrics._vector_checkpoint_write_errors_total + vector_checkpoints_total: telemetry.metrics._vector_checkpoints_total + vector_checksum_errors: telemetry.metrics._vector_checksum_errors + vector_file_delete_errors: telemetry.metrics._vector_file_delete_errors + vector_file_watch_errors: telemetry.metrics._vector_file_watch_errors + vector_files_added: telemetry.metrics._vector_files_added + vector_files_deleted: telemetry.metrics._vector_files_deleted + vector_files_resumed: telemetry.metrics._vector_files_resumed + vector_files_unwatched: telemetry.metrics._vector_files_unwatched + vector_fingerprint_read_errors: telemetry.metrics._vector_fingerprint_read_errors } } diff --git a/docs/reference/components/sources/host_metrics.cue b/docs/reference/components/sources/host_metrics.cue index 912a80788eee1..6423803724243 100644 --- a/docs/reference/components/sources/host_metrics.cue +++ b/docs/reference/components/sources/host_metrics.cue @@ -261,6 +261,18 @@ components: sources: host_metrics: { } output: metrics: { + _host_metrics_tags: { + collector: { + description: "Which collector this metric comes from." + required: true + } + host: { + description: "The hostname of the originating system." + required: true + examples: [_values.local_host] + } + } + // Host CPU host_cpu_seconds_total: { description: "The number of CPU seconds accumulated in different operating modes." diff --git a/docs/reference/components/sources/http.cue b/docs/reference/components/sources/http.cue index 1ec388069d241..0e54a25192da0 100644 --- a/docs/reference/components/sources/http.cue +++ b/docs/reference/components/sources/http.cue @@ -208,6 +208,6 @@ components: sources: http: { ] telemetry: metrics: { - vector_http_bad_requests_total: _vector_http_bad_requests_total + vector_http_bad_requests_total: telemetry.metrics._vector_http_bad_requests_total } } diff --git a/docs/reference/components/sources/internal_metrics.cue b/docs/reference/components/sources/internal_metrics.cue index 442724170eead..c01a7f2f9a2d6 100644 --- a/docs/reference/components/sources/internal_metrics.cue +++ b/docs/reference/components/sources/internal_metrics.cue @@ -55,6 +55,11 @@ components: sources: internal_metrics: { } } + vector_api_started_total: { + description: "The number of times the Vector GraphQL API has been started." + type: "counter" + tags: _internal_metrics_tags + } vector_config_load_errors_total: { description: "The total number of errors loading the Vector configuration." type: "counter" diff --git a/docs/reference/components/sources/journald.cue b/docs/reference/components/sources/journald.cue index fb6a78f6d03b1..60f881d5b37da 100644 --- a/docs/reference/components/sources/journald.cue +++ b/docs/reference/components/sources/journald.cue @@ -194,12 +194,12 @@ components: sources: journald: { vector_invalid_record_total: { description: "The total number of invalid journald records discarded." type: "counter" - tags: _component_tags + tags: telemetry.metrics._component_tags } vector_invalid_record_bytes_total: { description: "The total number of bytes from discarded journald records." type: "counter" - tags: _component_tags + tags: telemetry.metrics._component_tags } } } diff --git a/docs/reference/components/sources/logplex.cue b/docs/reference/components/sources/logplex.cue index 0864d65aeeeb5..478d02f353dba 100644 --- a/docs/reference/components/sources/logplex.cue +++ b/docs/reference/components/sources/logplex.cue @@ -99,7 +99,7 @@ components: sources: logplex: { } telemetry: metrics: { - vector_request_read_errors_total: _vector_request_read_errors_total - vector_requests_received_total: _vector_requests_received_total + vector_request_read_errors_total: telemetry.metrics._vector_request_read_errors_total + vector_requests_received_total: telemetry.metrics._vector_requests_received_total } } diff --git a/docs/reference/components/sources/mongodb_metrics.cue b/docs/reference/components/sources/mongodb_metrics.cue index 963d40d77305a..86fc44fda9db2 100644 --- a/docs/reference/components/sources/mongodb_metrics.cue +++ b/docs/reference/components/sources/mongodb_metrics.cue @@ -98,21 +98,22 @@ components: sources: mongodb_metrics: { collect_duration_nanoseconds: { description: "The duration spent collecting MongoDB metrics." type: "histogram" - tags: _internal_metrics_tags + tags: telemetry.metrics._internal_metrics_tags } collect_completed_total: { description: "The total number of MongoDB metrics collections completed." type: "counter" - tags: _internal_metrics_tags + tags: telemetry.metrics._internal_metrics_tags } request_error_total: { description: "The total number of MongoDB request errors." type: "counter" - tags: _internal_metrics_tags + tags: telemetry.metrics._internal_metrics_tags } } output: metrics: { + // Default MongoDB tags _mongodb_metrics_tags: { endpoint: { description: "The absolute path of the originating file." diff --git a/docs/reference/components/sources/prometheus.cue b/docs/reference/components/sources/prometheus.cue index 04a6bb3575f56..0c88dc84fd9a8 100644 --- a/docs/reference/components/sources/prometheus.cue +++ b/docs/reference/components/sources/prometheus.cue @@ -88,6 +88,4 @@ components: sources: prometheus: { histogram: output._passthrough_histogram summary: output._passthrough_summary } - - telemetry: metrics: _prometheus_metrics } diff --git a/docs/reference/components/sources/socket.cue b/docs/reference/components/sources/socket.cue index bab3039b7e381..ed94e640e0fba 100644 --- a/docs/reference/components/sources/socket.cue +++ b/docs/reference/components/sources/socket.cue @@ -151,6 +151,6 @@ components: sources: socket: { ] telemetry: metrics: { - vector_connection_errors_total: _vector_connection_errors_total + vector_connection_errors_total: telemetry.metrics._vector_connection_errors_total } } diff --git a/docs/reference/components/sources/splunk_hec.cue b/docs/reference/components/sources/splunk_hec.cue index 5dd1972dee280..e7c07f2efdae3 100644 --- a/docs/reference/components/sources/splunk_hec.cue +++ b/docs/reference/components/sources/splunk_hec.cue @@ -94,10 +94,10 @@ components: sources: splunk_hec: { } telemetry: metrics: { - vector_source_missing_keys_total: _vector_source_missing_keys_total - vector_sourcetype_missing_keys_total: _vector_sourcetype_missing_keys_total - vector_encode_errors_total: _vector_encode_errors_total - vector_http_request_errors_total: _vector_http_request_errors_total - vector_requests_received_total: _vector_requests_received_total + vector_source_missing_keys_total: telemetry.metrics._vector_source_missing_keys_total + vector_sourcetype_missing_keys_total: telemetry.metrics._vector_sourcetype_missing_keys_total + vector_encode_errors_total: telemetry.metrics._vector_encode_errors_total + vector_http_request_errors_total: telemetry.metrics._vector_http_request_errors_total + vector_requests_received_total: telemetry.metrics._vector_requests_received_total } } diff --git a/docs/reference/components/sources/statsd.cue b/docs/reference/components/sources/statsd.cue index 2b8ed11554943..7f594a2a37b96 100644 --- a/docs/reference/components/sources/statsd.cue +++ b/docs/reference/components/sources/statsd.cue @@ -122,16 +122,16 @@ components: sources: statsd: { } telemetry: metrics: { - vector_connection_errors_total: _vector_connection_errors_total + vector_connection_errors_total: telemetry.metrics._vector_connection_errors_total vector_invalid_record_total: { description: "The total number of invalid StatsD records discarded." type: "counter" - tags: _component_tags + tags: telemetry.metrics._component_tags } vector_invalid_record_bytes_total: { description: "The total number of bytes from StatsD journald records." type: "counter" - tags: _component_tags + tags: telemetry.metrics._component_tags } } } diff --git a/docs/reference/components/sources/stdin.cue b/docs/reference/components/sources/stdin.cue index b75974115b082..dc2600537e490 100644 --- a/docs/reference/components/sources/stdin.cue +++ b/docs/reference/components/sources/stdin.cue @@ -105,7 +105,7 @@ components: sources: stdin: { vector_stdin_reads_failed_total: { description: "The total number of errors reading from stdin." type: "counter" - tags: _component_tags + tags: telemetry.metrics._component_tags } } } diff --git a/docs/reference/components/sources/syslog.cue b/docs/reference/components/sources/syslog.cue index e18fb25008fd6..414dee7809c04 100644 --- a/docs/reference/components/sources/syslog.cue +++ b/docs/reference/components/sources/syslog.cue @@ -187,7 +187,7 @@ components: sources: syslog: { vector_connection_read_errors_total: { description: "The total number of errors reading datagram." type: "counter" - tags: _component_tags & { + tags: telemetry.metrics._component_tags & { mode: { description: "" required: true @@ -200,7 +200,7 @@ components: sources: syslog: { vector_utf8_convert_errors_total: { description: "The total number of errors converting bytes to a UTF-8 string in UDP mode." type: "counter" - tags: _component_tags & { + tags: telemetry.metrics._component_tags & { mode: { description: "The connection mode used by the component." required: true diff --git a/docs/reference/components/sources/vector.cue b/docs/reference/components/sources/vector.cue index bb5bbdb5f0954..18ff8c934290a 100644 --- a/docs/reference/components/sources/vector.cue +++ b/docs/reference/components/sources/vector.cue @@ -112,6 +112,6 @@ components: sources: vector: { } telemetry: metrics: { - vector_protobuf_decode_errors_total: _vector_protobuf_decode_errors_total + vector_protobuf_decode_errors_total: telemetry.metrics._vector_protobuf_decode_errors_total } } diff --git a/docs/reference/components/transforms/ansi_stripper.cue b/docs/reference/components/transforms/ansi_stripper.cue index d4b1181e00059..e7e4567f7fcc6 100644 --- a/docs/reference/components/transforms/ansi_stripper.cue +++ b/docs/reference/components/transforms/ansi_stripper.cue @@ -47,6 +47,6 @@ components: transforms: ansi_stripper: { } telemetry: metrics: { - vector_processing_errors_total: _vector_processing_errors_total + vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total } } diff --git a/docs/reference/components/transforms/aws_cloudwatch_logs_subscription_parser.cue b/docs/reference/components/transforms/aws_cloudwatch_logs_subscription_parser.cue index b0bdcff89713b..d05a4bdcf919a 100644 --- a/docs/reference/components/transforms/aws_cloudwatch_logs_subscription_parser.cue +++ b/docs/reference/components/transforms/aws_cloudwatch_logs_subscription_parser.cue @@ -144,6 +144,6 @@ components: transforms: aws_cloudwatch_logs_subscription_parser: { } telemetry: metrics: { - vector_processing_errors_total: _vector_processing_errors_total + vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total } } diff --git a/docs/reference/components/transforms/aws_ec2_metadata.cue b/docs/reference/components/transforms/aws_ec2_metadata.cue index 25da0a2ec96c6..ea9b26c4ad6f0 100644 --- a/docs/reference/components/transforms/aws_ec2_metadata.cue +++ b/docs/reference/components/transforms/aws_ec2_metadata.cue @@ -175,12 +175,12 @@ components: transforms: aws_ec2_metadata: { vector_metadata_refresh_failed_total: { description: "The total number of failed efforts to refresh AWS EC2 metadata." type: "counter" - tags: _component_tags + tags: telemetry.metrics._component_tags } vector_metadata_refresh_successful_total: { description: "The total number of AWS EC2 metadata refreshes." type: "counter" - tags: _component_tags + tags: telemetry.metrics._component_tags } } } diff --git a/docs/reference/components/transforms/dedupe.cue b/docs/reference/components/transforms/dedupe.cue index 93e5126ac7ca1..894e3edc08efe 100644 --- a/docs/reference/components/transforms/dedupe.cue +++ b/docs/reference/components/transforms/dedupe.cue @@ -148,6 +148,6 @@ components: transforms: dedupe: { } telemetry: metrics: { - vector_events_discarded_total: _vector_events_discarded_total + vector_events_discarded_total: telemetry.metrics._vector_events_discarded_total } } diff --git a/docs/reference/components/transforms/grok_parser.cue b/docs/reference/components/transforms/grok_parser.cue index a44e1f4442a12..4d802fd8eaf09 100644 --- a/docs/reference/components/transforms/grok_parser.cue +++ b/docs/reference/components/transforms/grok_parser.cue @@ -97,6 +97,6 @@ components: transforms: grok_parser: { } telemetry: metrics: { - vector_processing_errors_total: _vector_processing_errors_total + vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total } } diff --git a/docs/reference/components/transforms/logfmt_parser.cue b/docs/reference/components/transforms/logfmt_parser.cue index cf1ce245cbf20..dc699c64f628d 100644 --- a/docs/reference/components/transforms/logfmt_parser.cue +++ b/docs/reference/components/transforms/logfmt_parser.cue @@ -180,6 +180,6 @@ components: transforms: logfmt_parser: { } telemetry: metrics: { - vector_processing_errors_total: _vector_processing_errors_total + vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total } } diff --git a/docs/reference/components/transforms/lua.cue b/docs/reference/components/transforms/lua.cue index 76b0f4bd73267..663b1b3bde944 100644 --- a/docs/reference/components/transforms/lua.cue +++ b/docs/reference/components/transforms/lua.cue @@ -472,7 +472,7 @@ components: transforms: lua: { } telemetry: metrics: { - vector_memory_used: _vector_memory_used - vector_processing_errors_total: _vector_processing_errors_total + vector_memory_used: telemetry.metrics._vector_memory_used + vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total } } diff --git a/docs/reference/components/transforms/metric_to_log.cue b/docs/reference/components/transforms/metric_to_log.cue index 1d7ed4fe2b3e9..4a115ac0c25a1 100644 --- a/docs/reference/components/transforms/metric_to_log.cue +++ b/docs/reference/components/transforms/metric_to_log.cue @@ -95,6 +95,6 @@ components: transforms: metric_to_log: { how_it_works: {} telemetry: metrics: { - vector_processing_errors_total: _vector_processing_errors_total + vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total } } diff --git a/docs/reference/components/transforms/reduce.cue b/docs/reference/components/transforms/reduce.cue index e6581158c1a1a..2b13b2eea08d5 100644 --- a/docs/reference/components/transforms/reduce.cue +++ b/docs/reference/components/transforms/reduce.cue @@ -149,7 +149,7 @@ components: transforms: reduce: { vector_stale_events_flushed_total: { description: "The number of stale events that Vector has flushed." type: "counter" - tags: _component_tags + tags: telemetry.metrics._component_tags } } } diff --git a/docs/reference/components/transforms/regex_parser.cue b/docs/reference/components/transforms/regex_parser.cue index bc4a8a49f3c0c..e882fe6d31f22 100644 --- a/docs/reference/components/transforms/regex_parser.cue +++ b/docs/reference/components/transforms/regex_parser.cue @@ -180,6 +180,6 @@ components: transforms: regex_parser: { } telemetry: metrics: { - vector_processing_errors_total: _vector_processing_errors_total + vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total } } diff --git a/docs/reference/components/transforms/remap.cue b/docs/reference/components/transforms/remap.cue index 7e5807b2496d8..d6993d95da6c4 100644 --- a/docs/reference/components/transforms/remap.cue +++ b/docs/reference/components/transforms/remap.cue @@ -132,6 +132,6 @@ components: transforms: remap: { } telemetry: metrics: { - vector_processing_errors_total: _vector_processing_errors_total + vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total } } diff --git a/docs/reference/components/transforms/sampler.cue b/docs/reference/components/transforms/sampler.cue index 4962e7cc31303..e362f48613ab3 100644 --- a/docs/reference/components/transforms/sampler.cue +++ b/docs/reference/components/transforms/sampler.cue @@ -66,6 +66,6 @@ components: transforms: sampler: { } telemetry: metrics: { - vector_events_discarded_total: _vector_events_discarded_total + vector_events_discarded_total: telemetry.metrics._vector_events_discarded_total } } diff --git a/docs/reference/components/transforms/split.cue b/docs/reference/components/transforms/split.cue index 81477aa9173f3..9f68365911453 100644 --- a/docs/reference/components/transforms/split.cue +++ b/docs/reference/components/transforms/split.cue @@ -97,6 +97,6 @@ components: transforms: split: { ] telemetry: metrics: { - vector_processing_errors_total: _vector_processing_errors_total + vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total } } diff --git a/docs/reference/components/transforms/swimlanes.cue b/docs/reference/components/transforms/swimlanes.cue index 0777e7e559ab6..234e2ed9cb333 100644 --- a/docs/reference/components/transforms/swimlanes.cue +++ b/docs/reference/components/transforms/swimlanes.cue @@ -72,6 +72,6 @@ components: transforms: swimlanes: { ] telemetry: metrics: { - vector_events_discarded_total: _vector_events_discarded_total + vector_events_discarded_total: telemetry.metrics._vector_events_discarded_total } } diff --git a/docs/reference/components/transforms/tag_cardinality_limit.cue b/docs/reference/components/transforms/tag_cardinality_limit.cue index fa11cefab0301..a6c65f12ee68b 100644 --- a/docs/reference/components/transforms/tag_cardinality_limit.cue +++ b/docs/reference/components/transforms/tag_cardinality_limit.cue @@ -226,7 +226,7 @@ components: transforms: tag_cardinality_limit: { hitting the configured `value_limit`. """ type: "counter" - tags: _component_tags + tags: telemetry.metrics._component_tags } vector_value_limit_reached_total: { description: """ @@ -234,7 +234,7 @@ components: transforms: tag_cardinality_limit: { value limit has been reached. """ type: "counter" - tags: _component_tags + tags: telemetry.metrics._component_tags } } } diff --git a/docs/reference/components/transforms/tokenizer.cue b/docs/reference/components/transforms/tokenizer.cue index a28af4fed4ab6..b80d8801c0b47 100644 --- a/docs/reference/components/transforms/tokenizer.cue +++ b/docs/reference/components/transforms/tokenizer.cue @@ -116,6 +116,6 @@ components: transforms: tokenizer: { } telemetry: metrics: { - vector_processing_errors_total: _vector_processing_errors_total + vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total } } diff --git a/docs/reference/metrics.cue b/docs/reference/metrics.cue deleted file mode 100644 index d0631140b35b3..0000000000000 --- a/docs/reference/metrics.cue +++ /dev/null @@ -1,391 +0,0 @@ -package metadata - -// Vector internal metrics (plus misc) -_vector_api_started_total: { - description: "The number of times the Vector GraphQL API has been started." - type: "counter" - tags: _internal_metrics_tags -} -_vector_auto_concurrency_averaged_rtt: { - description: "The average round-trip time (RTT) from the HTTP sink across the current window." - type: "histogram" - tags: _internal_metrics_tags -} -_vector_auto_concurrency_in_flight: { - description: "The number of outbound requests from the HTTP sink currently awaiting a response." - type: "histogram" - tags: _internal_metrics_tags -} -_vector_auto_concurrency_limit: { - description: "The concurrency limit that the auto-concurrency feature has decided on for this current window." - type: "histogram" - tags: _internal_metrics_tags -} -_vector_auto_concurrency_observed_rtt: { - description: "The observed round-trip time (RTT) for requests from this HTTP sink." - type: "histogram" - tags: _internal_metrics_tags -} -_vector_checkpoint_write_errors_total: { - description: "The total number of errors writing checkpoints." - type: "counter" - tags: _internal_metrics_tags -} -_vector_checkpoints_total: { - description: "The total number of files checkpointed." - type: "counter" - tags: _internal_metrics_tags -} -_vector_checksum_errors: { - description: "The total number of errors identifying files via checksum." - type: "counter" - tags: _internal_metrics_tags & { - file: _file - } -} -_vector_events_discarded_total: { - description: "The total number of events discarded by this component." - type: "counter" - tags: _internal_metrics_tags -} -_vector_events_processed_total: { - description: "The total number of events processed by this component." - type: "counter" - tags: _component_tags & { - file: _file - } -} -_vector_file_delete_errors: { - description: "The total number of failures to delete a file." - type: "counter" - tags: _internal_metrics_tags & { - file: _file - } -} -_vector_file_watch_errors: { - description: "The total number of errors caused by failure to watch a file." - type: "counter" - tags: _internal_metrics_tags & { - file: _file - } -} -_vector_files_added: { - description: "The total number of files Vector has found to watch." - type: "counter" - tags: _internal_metrics_tags & { - file: _file - } -} -_vector_files_deleted: { - description: "The total number of files deleted." - type: "counter" - tags: _internal_metrics_tags & { - file: _file - } -} -_vector_files_resumed: { - description: "The total number of times Vector has resumed watching a file." - type: "counter" - tags: _internal_metrics_tags & { - file: _file - } -} -_vector_files_unwatched: { - description: "The total number of times Vector has stopped watching a file." - type: "counter" - tags: _internal_metrics_tags & { - file: _file - } -} -_vector_fingerprint_read_errors: { - description: "The total number of times failing to read a file for fingerprinting." - type: "counter" - tags: _internal_metrics_tags & { - file: _file - } -} -_vector_http_bad_requests_total: { - description: "The total number of HTTP `400 Bad Request` errors encountered." - type: "counter" - tags: _internal_metrics_tags -} -_vector_http_error_response_total: { - description: "The total number of HTTP error responses for this component." - type: "counter" - tags: _internal_metrics_tags -} -_vector_http_request_errors_total: { - description: "The total number of HTTP request errors for this component." - type: "counter" - tags: _internal_metrics_tags -} -_vector_http_requests_total: { - description: "The total number of HTTP requests issued by this component." - type: "counter" - tags: _component_tags -} -_vector_memory_used: { - description: "The total memory currently being used by Vector (in bytes)." - type: "gauge" - tags: _internal_metrics_tags -} -_vector_missing_keys_total: { - description: "The total number of events dropped due to keys missing from the event." - type: "counter" - tags: _internal_metrics_tags -} -_vector_open_connections: { - description: "The number of current open connections to Vector." - type: "gauge" - tags: _internal_metrics_tags -} -_vector_parse_errors_total: { - description: "The total number of errors parsing Prometheus metrics." - type: "counter" - tags: _internal_metrics_tags -} -_vector_processed_bytes_total: { - description: "The total number of bytes processed by the component." - type: "counter" - tags: _component_tags -} -_vector_processing_errors_total: { - description: "The total number of processing errors encountered by this component." - type: "counter" - tags: _component_tags & { - error_type: _error_type - } -} -_vector_protobuf_decode_errors_total: { - description: "The total number of [Protocol Buffers](\(urls.protobuf)) errors thrown during communication between Vector instances." - type: "counter" - tags: _component_tags -} -_vector_request_duration_nanoseconds: { - description: "The request duration for this component (in nanoseconds)." - type: "histogram" - tags: _component_tags -} -_vector_request_read_errors_total: { - description: "The total number of request read errors for this component." - type: "counter" - tags: _component_tags -} -_vector_requests_completed_total: { - description: "The total number of requests completed by this component." - type: "counter" - tags: _component_tags -} -_vector_requests_received_total: { - description: "The total number of requests received by this component." - type: "counter" - tags: _component_tags -} -_vector_timestamp_parse_errors_total: { - description: "The total number of errors encountered parsing [RFC3339](\(urls.rfc_3339)) timestamps." - type: "counter" - tags: _component_tags -} -_vector_uptime_seconds: { - description: "The total number of seconds the Vector instance has been up." - type: "gauge" - tags: _component_tags -} - -// Splunk -_vector_encode_errors_total: { - description: "The total number of errors encoding [Splunk HEC](\(urls.splunk_hec_protocol)) events to JSON for this `splunk_hec` sink." - type: "counter" - tags: _component_tags -} -_vector_source_missing_keys_total: { - description: "The total number of errors rendering the template for this source." - type: "counter" - tags: _component_tags -} -_vector_sourcetype_missing_keys_total: { - description: "The total number of errors rendering the template for this sourcetype." - type: "counter" - tags: _component_tags -} - -// Vector instance metrics -_vector_config_load_errors_total: { - description: "The total number of errors loading the Vector configuration." - type: "counter" - tags: _internal_metrics_tags -} -_vector_connection_errors_total: { - description: "The total number of connection errors for this Vector instance." - type: "counter" - tags: _internal_metrics_tags -} -_vector_quit_total: { - description: "The total number of times the Vector instance has quit." - type: "counter" - tags: _internal_metrics_tags -} -_vector_recover_errors_total: { - description: "The total number of errors caused by Vector failing to recover from a failed reload." - type: "counter" - tags: _internal_metrics_tags -} -_vector_reload_errors_total: { - description: "The total number of errors encountered when reloading Vector." - type: "counter" - tags: _internal_metrics_tags -} -_vector_reloaded_total: { - description: "The total number of times the Vector instance has been reloaded." - type: "counter" - tags: _internal_metrics_tags -} -_vector_started_total: { - description: "The total number of times the Vector instance has been started." - type: "counter" - tags: _internal_metrics_tags -} -_vector_stopped_total: { - description: "The total number of times the Vector instance has been stopped." - type: "counter" - tags: _internal_metrics_tags -} - -// Windows metrics -_windows_service_does_not_exist: { - description: """ - The total number of errors raised due to the Windows service not - existing. - """ - type: "counter" - tags: _internal_metrics_tags -} -_windows_service_install: { - description: """ - The total number of times the Windows service has been installed. - """ - type: "counter" - tags: _internal_metrics_tags -} -_windows_service_restart: { - description: """ - The total number of times the Windows service has been restarted. - """ - type: "counter" - tags: _internal_metrics_tags -} -_windows_service_start: { - description: """ - The total number of times the Windows service has been started. - """ - type: "counter" - tags: _internal_metrics_tags -} -_windows_service_stop: { - description: """ - The total number of times the Windows service has been stopped. - """ - type: "counter" - tags: _internal_metrics_tags -} -_windows_service_uninstall: { - description: """ - The total number of times the Windows service has been uninstalled. - """ - type: "counter" - tags: _internal_metrics_tags -} - -// All available tags -_collector: { - description: "Which collector this metric comes from." - required: true -} -_component_kind: { - description: "The component's kind (options are `source`, `sink`, or `transform`)." - required: true - options: ["sink", "source", "transform"] -} -_component_name: { - description: "The name of the component as specified in the Vector configuration." - required: true - examples: ["file_source", "splunk_sink"] -} -_component_type: { - description: "The type of component (source, transform, or sink)." - required: true - examples: ["file", "http", "honeycomb", "splunk_hec"] -} -_endpoint: { - description: "The absolute path of originating file." - required: true - examples: ["http://localhost:8080/server-status?auto"] -} -_error_type: { - description: "The type of the error" - required: true - options: [ - "field_missing", - "invalid_metric", - "mapping_failed", - "match_failed", - "parse_failed", - "render_error", - "type_conversion_failed", - "value_invalid", - ] -} -_file: { - description: "The file that produced the error" - required: false -} -_host: { - description: "The hostname of the originating system." - required: true - examples: [_values.local_host] -} -_instance: { - description: "The Vector instance identified by host and port." - required: true - examples: [_values.instance] -} -_job: { - description: "The name of the job producing Vector metrics." - required: true - default: "vector" -} - -// Convenient groupings of tags -_component_tags: _internal_metrics_tags & { - component_kind: _component_kind - component_name: _component_name - component_type: _component_type -} - -_apache_metrics_tags: { - endpoint: _endpoint - host: { - description: "The hostname of the Apache HTTP server." - required: true - examples: [_values.local_host] - } -} -_host_metrics_tags: { - collector: _collector - host: _host -} -_internal_metrics_tags: { - instance: _instance - job: _job -} - -// Helpful metrics groupings -_prometheus_metrics: { - vector_events_processed_total: _vector_events_processed_total - vector_http_error_response_total: _vector_http_error_response_total - vector_http_request_errors_total: _vector_http_request_errors_total - vector_parse_errors_total: _vector_parse_errors_total - vector_processed_bytes_total: _vector_processed_bytes_total - vector_request_duration_nanoseconds: _vector_request_duration_nanoseconds - vector_requests_completed_total: _vector_requests_completed_total -} From 68562b3df6a9530322b37f1c7bc1564a672f1a95 Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Thu, 12 Nov 2020 15:05:48 -0800 Subject: [PATCH 13/20] Apply explicit default namespace to all metrics Signed-off-by: Luc Perkins --- docs/reference.cue | 9 + docs/reference/components.cue | 509 +++++++------ docs/reference/components/kafka.cue | 4 +- .../components/sinks/aws_cloudwatch_logs.cue | 2 +- docs/reference/components/sinks/console.cue | 2 +- .../components/sinks/elasticsearch.cue | 2 +- docs/reference/components/sinks/http.cue | 2 +- .../components/sinks/sematext_metrics.cue | 2 +- .../reference/components/sinks/splunk_hec.cue | 10 +- docs/reference/components/sinks/statsd.cue | 2 +- docs/reference/components/sinks/vector.cue | 2 +- .../components/sources/apache_metrics.cue | 112 +-- .../sources/aws_kinesis_firehose.cue | 4 +- .../components/sources/docker_logs.cue | 57 +- docs/reference/components/sources/file.cue | 20 +- .../components/sources/host_metrics.cue | 70 +- docs/reference/components/sources/http.cue | 2 +- .../components/sources/internal_metrics.cue | 81 +- .../reference/components/sources/journald.cue | 18 +- .../components/sources/kubernetes_logs.cue | 18 +- docs/reference/components/sources/logplex.cue | 4 +- .../components/sources/mongodb_metrics.cue | 700 ++++++++++-------- docs/reference/components/sources/socket.cue | 2 +- .../components/sources/splunk_hec.cue | 10 +- docs/reference/components/sources/statsd.cue | 20 +- docs/reference/components/sources/stdin.cue | 9 +- docs/reference/components/sources/syslog.cue | 18 +- docs/reference/components/sources/vector.cue | 2 +- .../components/transforms/ansi_stripper.cue | 2 +- ...ws_cloudwatch_logs_subscription_parser.cue | 2 +- .../transforms/aws_ec2_metadata.cue | 18 +- .../components/transforms/dedupe.cue | 2 +- .../components/transforms/grok_parser.cue | 2 +- .../components/transforms/logfmt_parser.cue | 2 +- docs/reference/components/transforms/lua.cue | 4 +- .../components/transforms/metric_to_log.cue | 2 +- .../components/transforms/reduce.cue | 9 +- .../components/transforms/regex_parser.cue | 2 +- .../reference/components/transforms/remap.cue | 2 +- .../components/transforms/sampler.cue | 2 +- .../reference/components/transforms/split.cue | 2 +- .../components/transforms/swimlanes.cue | 2 +- .../transforms/tag_cardinality_limit.cue | 14 +- .../components/transforms/tokenizer.cue | 2 +- 44 files changed, 973 insertions(+), 789 deletions(-) diff --git a/docs/reference.cue b/docs/reference.cue index e728bbe86f597..0fce72f3bd129 100644 --- a/docs/reference.cue +++ b/docs/reference.cue @@ -185,6 +185,15 @@ _values: { sum: float } +#MetricOutput: [Name=string]: close({ + description: string + relevant_when?: string + tags: #MetricTags + name: Name + type: #MetricType + default_namespace: string +}) + #MetricTags: [Name=string]: close({ name: Name description: string diff --git a/docs/reference/components.cue b/docs/reference/components.cue index d821f6b9c804a..d121a18f1c457 100644 --- a/docs/reference/components.cue +++ b/docs/reference/components.cue @@ -833,7 +833,8 @@ components: { required: false } } - type: "counter" + type: "counter" + default_namespace: "vector" } _passthrough_distribution: { @@ -845,7 +846,8 @@ components: { required: false } } - type: "distribution" + type: "distribution" + default_namespace: "vector" } _passthrough_gauge: { @@ -857,7 +859,8 @@ components: { required: false } } - type: "gauge" + type: "gauge" + default_namespace: "vector" } _passthrough_histogram: { @@ -869,7 +872,8 @@ components: { required: false } } - type: "gauge" + type: "gauge" + default_namespace: "vector" } _passthrough_set: { @@ -881,7 +885,8 @@ components: { required: false } } - type: "gauge" + type: "gauge" + default_namespace: "vector" } _passthrough_summary: { @@ -893,260 +898,308 @@ components: { required: false } } - type: "gauge" + type: "gauge" + default_namespace: "vector" } } } telemetry: metrics: { // Default metrics for each component - vector_events_processed_total: _vector_events_processed_total - vector_processed_bytes_total: _vector_processed_bytes_total + events_processed_total: _events_processed_total + processed_bytes_total: _processed_bytes_total // Reusable metric definitions - _vector_auto_concurrency_averaged_rtt: { - description: "The average round-trip time (RTT) from the HTTP sink across the current window." - type: "histogram" - tags: _internal_metrics_tags - } - _vector_auto_concurrency_in_flight: { - description: "The number of outbound requests from the HTTP sink currently awaiting a response." - type: "histogram" - tags: _internal_metrics_tags - } - _vector_auto_concurrency_limit: { - description: "The concurrency limit that the auto-concurrency feature has decided on for this current window." - type: "histogram" - tags: _internal_metrics_tags - } - _vector_auto_concurrency_observed_rtt: { - description: "The observed round-trip time (RTT) for requests from this HTTP sink." - type: "histogram" - tags: _internal_metrics_tags - } - _vector_checkpoint_write_errors_total: { - description: "The total number of errors writing checkpoints." - type: "counter" - tags: _internal_metrics_tags - } - _vector_checkpoints_total: { - description: "The total number of files checkpointed." - type: "counter" - tags: _internal_metrics_tags - } - _vector_checksum_errors: { - description: "The total number of errors identifying files via checksum." - type: "counter" - tags: _internal_metrics_tags & { + _auto_concurrency_averaged_rtt: { + description: "The average round-trip time (RTT) from the HTTP sink across the current window." + type: "histogram" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _auto_concurrency_in_flight: { + description: "The number of outbound requests from the HTTP sink currently awaiting a response." + type: "histogram" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _auto_concurrency_limit: { + description: "The concurrency limit that the auto-concurrency feature has decided on for this current window." + type: "histogram" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _auto_concurrency_observed_rtt: { + description: "The observed round-trip time (RTT) for requests from this HTTP sink." + type: "histogram" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _checkpoint_write_errors_total: { + description: "The total number of errors writing checkpoints." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _checkpoints_total: { + description: "The total number of files checkpointed." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _checksum_errors: { + description: "The total number of errors identifying files via checksum." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & { file: _file } } - _vector_events_discarded_total: { - description: "The total number of events discarded by this component." - type: "counter" - tags: _internal_metrics_tags - } - _vector_events_processed_total: { - description: "The total number of events processed by this component." - type: "counter" - tags: _component_tags & { + _events_discarded_total: { + description: "The total number of events discarded by this component." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _events_processed_total: { + description: "The total number of events processed by this component." + type: "counter" + default_namespace: "vector" + tags: _component_tags & { file: _file } } - _vector_file_delete_errors: { - description: "The total number of failures to delete a file." - type: "counter" - tags: _internal_metrics_tags & { + _file_delete_errors: { + description: "The total number of failures to delete a file." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & { file: _file } } - _vector_file_watch_errors: { - description: "The total number of errors caused by failure to watch a file." - type: "counter" - tags: _internal_metrics_tags & { + _file_watch_errors: { + description: "The total number of errors caused by failure to watch a file." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & { file: _file } } - _vector_files_added: { - description: "The total number of files Vector has found to watch." - type: "counter" - tags: _internal_metrics_tags & { + _files_added: { + description: "The total number of files Vector has found to watch." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & { file: _file } } - _vector_files_deleted: { - description: "The total number of files deleted." - type: "counter" - tags: _internal_metrics_tags & { + _files_deleted: { + description: "The total number of files deleted." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & { file: _file } } - _vector_files_resumed: { - description: "The total number of times Vector has resumed watching a file." - type: "counter" - tags: _internal_metrics_tags & { + _files_resumed: { + description: "The total number of times Vector has resumed watching a file." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & { file: _file } } - _vector_files_unwatched: { - description: "The total number of times Vector has stopped watching a file." - type: "counter" - tags: _internal_metrics_tags & { + _files_unwatched: { + description: "The total number of times Vector has stopped watching a file." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & { file: _file } } - _vector_fingerprint_read_errors: { - description: "The total number of times failing to read a file for fingerprinting." - type: "counter" - tags: _internal_metrics_tags & { + _fingerprint_read_errors: { + description: "The total number of times failing to read a file for fingerprinting." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & { file: _file } } - _vector_http_bad_requests_total: { - description: "The total number of HTTP `400 Bad Request` errors encountered." - type: "counter" - tags: _internal_metrics_tags - } - _vector_http_error_response_total: { - description: "The total number of HTTP error responses for this component." - type: "counter" - tags: _internal_metrics_tags - } - _vector_http_request_errors_total: { - description: "The total number of HTTP request errors for this component." - type: "counter" - tags: _internal_metrics_tags - } - _vector_http_requests_total: { - description: "The total number of HTTP requests issued by this component." - type: "counter" - tags: _component_tags - } - _vector_memory_used: { - description: "The total memory currently being used by Vector (in bytes)." - type: "gauge" - tags: _internal_metrics_tags - } - _vector_missing_keys_total: { - description: "The total number of events dropped due to keys missing from the event." - type: "counter" - tags: _internal_metrics_tags - } - _vector_open_connections: { - description: "The number of current open connections to Vector." - type: "gauge" - tags: _internal_metrics_tags - } - _vector_parse_errors_total: { - description: "The total number of errors parsing Prometheus metrics." - type: "counter" - tags: _internal_metrics_tags - } - _vector_processed_bytes_total: { - description: "The total number of bytes processed by the component." - type: "counter" - tags: _component_tags - } - _vector_processing_errors_total: { - description: "The total number of processing errors encountered by this component." - type: "counter" - tags: _component_tags & { + _http_bad_requests_total: { + description: "The total number of HTTP `400 Bad Request` errors encountered." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _http_error_response_total: { + description: "The total number of HTTP error responses for this component." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _http_request_errors_total: { + description: "The total number of HTTP request errors for this component." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _http_requests_total: { + description: "The total number of HTTP requests issued by this component." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + _memory_used: { + description: "The total memory currently being used by Vector (in bytes)." + type: "gauge" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _missing_keys_total: { + description: "The total number of events dropped due to keys missing from the event." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _open_connections: { + description: "The number of current open connections to Vector." + type: "gauge" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _parse_errors_total: { + description: "The total number of errors parsing Prometheus metrics." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _processed_bytes_total: { + description: "The total number of bytes processed by the component." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + _processing_errors_total: { + description: "The total number of processing errors encountered by this component." + type: "counter" + default_namespace: "vector" + tags: _component_tags & { error_type: _error_type } } - _vector_protobuf_decode_errors_total: { - description: "The total number of [Protocol Buffers](\(urls.protobuf)) errors thrown during communication between Vector instances." - type: "counter" - tags: _component_tags - } - _vector_request_duration_nanoseconds: { - description: "The request duration for this component (in nanoseconds)." - type: "histogram" - tags: _component_tags - } - _vector_request_read_errors_total: { - description: "The total number of request read errors for this component." - type: "counter" - tags: _component_tags - } - _vector_requests_completed_total: { - description: "The total number of requests completed by this component." - type: "counter" - tags: _component_tags - } - _vector_requests_received_total: { - description: "The total number of requests received by this component." - type: "counter" - tags: _component_tags - } - _vector_timestamp_parse_errors_total: { - description: "The total number of errors encountered parsing [RFC3339](\(urls.rfc_3339)) timestamps." - type: "counter" - tags: _component_tags - } - _vector_uptime_seconds: { - description: "The total number of seconds the Vector instance has been up." - type: "gauge" - tags: _component_tags + _protobuf_decode_errors_total: { + description: "The total number of [Protocol Buffers](\(urls.protobuf)) errors thrown during communication between Vector instances." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + _request_duration_nanoseconds: { + description: "The request duration for this component (in nanoseconds)." + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } + _request_read_errors_total: { + description: "The total number of request read errors for this component." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + _requests_completed_total: { + description: "The total number of requests completed by this component." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + _requests_received_total: { + description: "The total number of requests received by this component." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + _timestamp_parse_errors_total: { + description: "The total number of errors encountered parsing [RFC3339](\(urls.rfc_3339)) timestamps." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + _uptime_seconds: { + description: "The total number of seconds the Vector instance has been up." + type: "gauge" + default_namespace: "vector" + tags: _component_tags } // Splunk - _vector_encode_errors_total: { - description: "The total number of errors encoding [Splunk HEC](\(urls.splunk_hec_protocol)) events to JSON for this `splunk_hec` sink." - type: "counter" - tags: _component_tags + _encode_errors_total: { + description: """ + The total number of errors encoding [Splunk HEC](\(urls.splunk_hec_protocol)) events + to JSON for this `splunk_hec` sink. + """ + type: "counter" + default_namespace: "vector" + tags: _component_tags } - _vector_source_missing_keys_total: { - description: "The total number of errors rendering the template for this source." - type: "counter" - tags: _component_tags + _source_missing_keys_total: { + description: "The total number of errors rendering the template for this source." + type: "counter" + default_namespace: "vector" + tags: _component_tags } - _vector_sourcetype_missing_keys_total: { - description: "The total number of errors rendering the template for this sourcetype." - type: "counter" - tags: _component_tags + _sourcetype_missing_keys_total: { + description: "The total number of errors rendering the template for this sourcetype." + type: "counter" + default_namespace: "vector" + tags: _component_tags } // Vector instance metrics - _vector_config_load_errors_total: { - description: "The total number of errors loading the Vector configuration." - type: "counter" - tags: _internal_metrics_tags - } - _vector_connection_errors_total: { - description: "The total number of connection errors for this Vector instance." - type: "counter" - tags: _internal_metrics_tags - } - _vector_quit_total: { - description: "The total number of times the Vector instance has quit." - type: "counter" - tags: _internal_metrics_tags - } - _vector_recover_errors_total: { - description: "The total number of errors caused by Vector failing to recover from a failed reload." - type: "counter" - tags: _internal_metrics_tags - } - _vector_reload_errors_total: { - description: "The total number of errors encountered when reloading Vector." - type: "counter" - tags: _internal_metrics_tags - } - _vector_reloaded_total: { - description: "The total number of times the Vector instance has been reloaded." - type: "counter" - tags: _internal_metrics_tags - } - _vector_started_total: { - description: "The total number of times the Vector instance has been started." - type: "counter" - tags: _internal_metrics_tags - } - _vector_stopped_total: { - description: "The total number of times the Vector instance has been stopped." - type: "counter" - tags: _internal_metrics_tags + _config_load_errors_total: { + description: "The total number of errors loading the Vector configuration." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _connection_errors_total: { + description: "The total number of connection errors for this Vector instance." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _quit_total: { + description: "The total number of times the Vector instance has quit." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _recover_errors_total: { + description: "The total number of errors caused by Vector failing to recover from a failed reload." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _reload_errors_total: { + description: "The total number of errors encountered when reloading Vector." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _reloaded_total: { + description: "The total number of times the Vector instance has been reloaded." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _started_total: { + description: "The total number of times the Vector instance has been started." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + _stopped_total: { + description: "The total number of times the Vector instance has been stopped." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags } // Windows metrics @@ -1155,43 +1208,49 @@ components: { The total number of errors raised due to the Windows service not existing. """ - type: "counter" - tags: _internal_metrics_tags + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags } _windows_service_install: { description: """ The total number of times the Windows service has been installed. """ - type: "counter" - tags: _internal_metrics_tags + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags } _windows_service_restart: { description: """ The total number of times the Windows service has been restarted. """ - type: "counter" - tags: _internal_metrics_tags + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags } _windows_service_start: { description: """ The total number of times the Windows service has been started. """ - type: "counter" - tags: _internal_metrics_tags + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags } _windows_service_stop: { description: """ The total number of times the Windows service has been stopped. """ - type: "counter" - tags: _internal_metrics_tags + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags } _windows_service_uninstall: { description: """ The total number of times the Windows service has been uninstalled. """ - type: "counter" - tags: _internal_metrics_tags + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags } // Helpful tag groupings diff --git a/docs/reference/components/kafka.cue b/docs/reference/components/kafka.cue index cb1516c121d0a..6e12e078f58de 100644 --- a/docs/reference/components/kafka.cue +++ b/docs/reference/components/kafka.cue @@ -90,12 +90,12 @@ components: _kafka: { } telemetry: metrics: { - vector_consumer_offset_updates_failed_total: { + consumer_offset_updates_failed_total: { description: "The total number of failures to update a Kafka consumer offset." type: "counter" tags: telemetry.metrics._component_tags } - vector_events_failed_total: { + events_failed_total: { description: "The total number of failures to read a Kafka message." type: "counter" tags: telemetry.metrics._component_tags diff --git a/docs/reference/components/sinks/aws_cloudwatch_logs.cue b/docs/reference/components/sinks/aws_cloudwatch_logs.cue index 866df90cead9c..d0aaaf0eb241c 100644 --- a/docs/reference/components/sinks/aws_cloudwatch_logs.cue +++ b/docs/reference/components/sinks/aws_cloudwatch_logs.cue @@ -121,6 +121,6 @@ components: sinks: aws_cloudwatch_logs: { } telemetry: metrics: { - processing_errors_total: telemetry.metrics._vector_processing_errors_total + processing_errors_total: telemetry.metrics._processing_errors_total } } diff --git a/docs/reference/components/sinks/console.cue b/docs/reference/components/sinks/console.cue index e2cd0769d0a56..2fef7d9b34349 100644 --- a/docs/reference/components/sinks/console.cue +++ b/docs/reference/components/sinks/console.cue @@ -81,6 +81,6 @@ components: sinks: console: { } telemetry: metrics: { - vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total + processing_errors_total: telemetry.metrics._processing_errors_total } } diff --git a/docs/reference/components/sinks/elasticsearch.cue b/docs/reference/components/sinks/elasticsearch.cue index 927af99c205eb..ab0523315dc63 100644 --- a/docs/reference/components/sinks/elasticsearch.cue +++ b/docs/reference/components/sinks/elasticsearch.cue @@ -241,6 +241,6 @@ components: sinks: elasticsearch: { } telemetry: metrics: { - vector_missing_keys_total: telemetry.metrics._vector_missing_keys_total + missing_keys_total: telemetry.metrics._missing_keys_total } } diff --git a/docs/reference/components/sinks/http.cue b/docs/reference/components/sinks/http.cue index 0f1624d509e06..3320f252af126 100644 --- a/docs/reference/components/sinks/http.cue +++ b/docs/reference/components/sinks/http.cue @@ -121,6 +121,6 @@ components: sinks: http: { } telemetry: metrics: { - vector_http_bad_requests_total: telemetry.metrics._vector_http_bad_requests_total + http_bad_requests_total: telemetry.metrics._http_bad_requests_total } } diff --git a/docs/reference/components/sinks/sematext_metrics.cue b/docs/reference/components/sinks/sematext_metrics.cue index c6e0fa45b9bc3..0ffbc61939e2f 100644 --- a/docs/reference/components/sinks/sematext_metrics.cue +++ b/docs/reference/components/sinks/sematext_metrics.cue @@ -80,6 +80,6 @@ components: sinks: sematext_metrics: { } telemetry: metrics: { - vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total + processing_errors_total: telemetry.metrics._processing_errors_total } } diff --git a/docs/reference/components/sinks/splunk_hec.cue b/docs/reference/components/sinks/splunk_hec.cue index bf407bfc219e1..bae876477f2cb 100644 --- a/docs/reference/components/sinks/splunk_hec.cue +++ b/docs/reference/components/sinks/splunk_hec.cue @@ -167,10 +167,10 @@ components: sinks: splunk_hec: { } telemetry: metrics: { - vector_encode_errors_total: telemetry.metrics._vector_encode_errors_total - vector_http_request_errors_total: telemetry.metrics._vector_http_request_errors_total - vector_http_requests_total: telemetry.metrics._vector_http_requests_total - vector_source_missing_keys_total: telemetry.metrics._vector_source_missing_keys_total - vector_sourcetype_missing_keys_total: telemetry.metrics._vector_sourcetype_missing_keys_total + encode_errors_total: telemetry.metrics._encode_errors_total + http_request_errors_total: telemetry.metrics._http_request_errors_total + http_requests_total: telemetry.metrics._http_requests_total + source_missing_keys_total: telemetry.metrics._source_missing_keys_total + sourcetype_missing_keys_total: telemetry.metrics._sourcetype_missing_keys_total } } diff --git a/docs/reference/components/sinks/statsd.cue b/docs/reference/components/sinks/statsd.cue index 5b884d803ad6f..2d0b23001c3d4 100644 --- a/docs/reference/components/sinks/statsd.cue +++ b/docs/reference/components/sinks/statsd.cue @@ -65,6 +65,6 @@ components: sinks: statsd: { } telemetry: metrics: { - vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total + processing_errors_total: telemetry.metrics._processing_errors_total } } diff --git a/docs/reference/components/sinks/vector.cue b/docs/reference/components/sinks/vector.cue index 8fff41c9c455b..f2d60012c764b 100644 --- a/docs/reference/components/sinks/vector.cue +++ b/docs/reference/components/sinks/vector.cue @@ -86,6 +86,6 @@ components: sinks: vector: { how_it_works: components.sources.vector.how_it_works telemetry: metrics: { - vector_protobuf_decode_errors_total: telemetry.metrics._vector_protobuf_decode_errors_total + protobuf_decode_errors_total: telemetry.metrics._protobuf_decode_errors_total } } diff --git a/docs/reference/components/sources/apache_metrics.cue b/docs/reference/components/sources/apache_metrics.cue index 224c3876f4881..a5b3dcc350380 100644 --- a/docs/reference/components/sources/apache_metrics.cue +++ b/docs/reference/components/sources/apache_metrics.cue @@ -126,16 +126,18 @@ components: sources: apache_metrics: { } } - apache_access_total: { - description: "The total number of time the Apache server has been accessed." - relevant_when: "`ExtendedStatus On`" - type: "counter" - tags: _apache_metrics_tags - } - apache_connections: { - description: "The total number of time the Apache server has been accessed." - type: "gauge" - tags: _apache_metrics_tags & { + access_total: { + description: "The total number of time the Apache server has been accessed." + relevant_when: "`ExtendedStatus On`" + type: "counter" + default_namespace: "apache" + tags: _apache_metrics_tags + } + connections: { + description: "The total number of time the Apache server has been accessed." + type: "gauge" + default_namespace: "apache" + tags: _apache_metrics_tags & { state: { description: "The state of the connection" required: true @@ -143,17 +145,19 @@ components: sources: apache_metrics: { } } } - apache_cpu_load: { - description: "The current CPU of the Apache server." - relevant_when: "`ExtendedStatus On`" - type: "gauge" - tags: _apache_metrics_tags - } - apache_cpu_seconds_total: { - description: "The CPU time of various Apache processes." - relevant_when: "`ExtendedStatus On`" - type: "counter" - tags: _apache_metrics_tags & { + cpu_load: { + description: "The current CPU of the Apache server." + relevant_when: "`ExtendedStatus On`" + type: "gauge" + default_namespace: "apache" + tags: _apache_metrics_tags + } + cpu_seconds_total: { + description: "The CPU time of various Apache processes." + relevant_when: "`ExtendedStatus On`" + type: "counter" + default_namespace: "apache" + tags: _apache_metrics_tags & { state: { description: "The state of the connection" required: true @@ -161,16 +165,18 @@ components: sources: apache_metrics: { } } } - apache_duration_seconds_total: { - description: "The amount of time the Apache server has been running." - relevant_when: "`ExtendedStatus On`" - type: "counter" - tags: _apache_metrics_tags - } - apache_scoreboard: { - description: "The amount of times various Apache server tasks have been run." - type: "gauge" - tags: _apache_metrics_tags & { + duration_seconds_total: { + description: "The amount of time the Apache server has been running." + relevant_when: "`ExtendedStatus On`" + type: "counter" + default_namespace: "apache" + tags: _apache_metrics_tags + } + scoreboard: { + description: "The amount of times various Apache server tasks have been run." + type: "gauge" + default_namespace: "apache" + tags: _apache_metrics_tags & { state: { description: "The connect state" required: true @@ -178,26 +184,30 @@ components: sources: apache_metrics: { } } } - apache_sent_bytes_total: { - description: "The amount of bytes sent by the Apache server." - relevant_when: "`ExtendedStatus On`" - type: "counter" - tags: _apache_metrics_tags - } - apache_up: { - description: "If the Apache server is up or not." - type: "gauge" - tags: _apache_metrics_tags - } - apache_uptime_seconds_total: { - description: "The amount of time the Apache server has been running." - type: "counter" - tags: _apache_metrics_tags - } - apache_workers: { - description: "Apache worker statuses." - type: "gauge" - tags: _apache_metrics_tags & { + sent_bytes_total: { + description: "The amount of bytes sent by the Apache server." + relevant_when: "`ExtendedStatus On`" + type: "counter" + default_namespace: "apache" + tags: _apache_metrics_tags + } + up: { + description: "If the Apache server is up or not." + type: "gauge" + default_namespace: "apache" + tags: _apache_metrics_tags + } + uptime_seconds_total: { + description: "The amount of time the Apache server has been running." + type: "counter" + default_namespace: "apache" + tags: _apache_metrics_tags + } + workers: { + description: "Apache worker statuses." + type: "gauge" + default_namespace: "apache" + tags: _apache_metrics_tags & { state: { description: "The state of the worker" required: true diff --git a/docs/reference/components/sources/aws_kinesis_firehose.cue b/docs/reference/components/sources/aws_kinesis_firehose.cue index 41762c731cc8e..de3e95257ccce 100644 --- a/docs/reference/components/sources/aws_kinesis_firehose.cue +++ b/docs/reference/components/sources/aws_kinesis_firehose.cue @@ -189,7 +189,7 @@ components: sources: aws_kinesis_firehose: { } telemetry: metrics: { - vector_request_read_errors_total: telemetry.metrics._vector_request_read_errors_total - vector_requests_received_total: telemetry.metrics._vector_requests_received_total + request_read_errors_total: telemetry.metrics._request_read_errors_total + requests_received_total: telemetry.metrics._requests_received_total } } diff --git a/docs/reference/components/sources/docker_logs.cue b/docs/reference/components/sources/docker_logs.cue index 45ef06ef14978..278e9c3c4330f 100644 --- a/docs/reference/components/sources/docker_logs.cue +++ b/docs/reference/components/sources/docker_logs.cue @@ -267,35 +267,44 @@ components: sources: docker_logs: { } telemetry: metrics: { - vector_communication_errors_total: { - description: "The total number of errors stemming from communication with the Docker daemon." - type: "counter" - tags: telemetry.metrics._component_tags + communication_errors_total: { + description: "The total number of errors stemming from communication with the Docker daemon." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags } - vector_container_events_processed_total: { - description: "The total number of container events processed." - type: "counter" - tags: telemetry.metrics._component_tags + container_events_processed_total: { + description: "The total number of container events processed." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags } - vector_container_metadata_fetch_errors_total: { - description: "The total number of errors caused by failure to fetch container metadata." - type: "counter" - tags: telemetry.metrics._component_tags + container_metadata_fetch_errors_total: { + description: "The total number of errors caused by failure to fetch container metadata." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags } - vector_containers_unwatched_total: { - description: "The total number of times Vector stopped watching for container logs." - type: "counter" - tags: telemetry.metrics._component_tags + containers_unwatched_total: { + description: "The total number of times Vector stopped watching for container logs." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags } - vector_containers_watched_total: { - description: "The total number of times Vector started watching for container logs." - type: "counter" - tags: telemetry.metrics._component_tags + containers_watched_total: { + description: "The total number of times Vector started watching for container logs." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags } - vector_logging_driver_errors_total: { - description: "The total number of logging driver errors encountered caused by not using either the `jsonfile` or `journald` driver." - type: "counter" - tags: telemetry.metrics._component_tags + logging_driver_errors_total: { + description: """ + The total number of logging driver errors encountered caused by not using either + the `jsonfile` or `journald` driver. + """ + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags } } } diff --git a/docs/reference/components/sources/file.cue b/docs/reference/components/sources/file.cue index e4c44c0fc0a24..4908b527a3465 100644 --- a/docs/reference/components/sources/file.cue +++ b/docs/reference/components/sources/file.cue @@ -517,15 +517,15 @@ components: sources: file: { } telemetry: metrics: { - vector_checkpoint_write_errors_total: telemetry.metrics._vector_checkpoint_write_errors_total - vector_checkpoints_total: telemetry.metrics._vector_checkpoints_total - vector_checksum_errors: telemetry.metrics._vector_checksum_errors - vector_file_delete_errors: telemetry.metrics._vector_file_delete_errors - vector_file_watch_errors: telemetry.metrics._vector_file_watch_errors - vector_files_added: telemetry.metrics._vector_files_added - vector_files_deleted: telemetry.metrics._vector_files_deleted - vector_files_resumed: telemetry.metrics._vector_files_resumed - vector_files_unwatched: telemetry.metrics._vector_files_unwatched - vector_fingerprint_read_errors: telemetry.metrics._vector_fingerprint_read_errors + checkpoint_write_errors_total: telemetry.metrics._checkpoint_write_errors_total + checkpoints_total: telemetry.metrics._checkpoints_total + checksum_errors: telemetry.metrics._checksum_errors + file_delete_errors: telemetry.metrics._file_delete_errors + file_watch_errors: telemetry.metrics._file_watch_errors + files_added: telemetry.metrics._files_added + files_deleted: telemetry.metrics._files_deleted + files_resumed: telemetry.metrics._files_resumed + files_unwatched: telemetry.metrics._files_unwatched + fingerprint_read_errors: telemetry.metrics._fingerprint_read_errors } } diff --git a/docs/reference/components/sources/host_metrics.cue b/docs/reference/components/sources/host_metrics.cue index 6423803724243..f4cd1a1306372 100644 --- a/docs/reference/components/sources/host_metrics.cue +++ b/docs/reference/components/sources/host_metrics.cue @@ -274,7 +274,7 @@ components: sources: host_metrics: { } // Host CPU - host_cpu_seconds_total: { + host_cpu_seconds_total: _host & { description: "The number of CPU seconds accumulated in different operating modes." type: "counter" tags: _host_metrics_tags & { @@ -293,52 +293,56 @@ components: sources: host_metrics: { } // Host disk - host_disk_read_bytes_total: _disk_counter & {description: "The accumulated number of bytes read in."} - host_disk_reads_completed_total: _disk_counter & {description: "The accumulated number of read operations completed."} - host_disk_written_bytes_total: _disk_counter & {description: "The accumulated number of bytes written out."} - host_disk_writes_completed_total: _disk_counter & {description: "The accumulated number of write operations completed."} + host_disk_read_bytes_total: _host & _disk_counter & {description: "The accumulated number of bytes read in."} + host_disk_reads_completed_total: _host & _disk_counter & {description: "The accumulated number of read operations completed."} + host_disk_written_bytes_total: _host & _disk_counter & {description: "The accumulated number of bytes written out."} + host_disk_writes_completed_total: _host & _disk_counter & {description: "The accumulated number of write operations completed."} // Host filesystem - host_filesystem_free_bytes: _filesystem_bytes & {description: "The number of bytes free on the named filesystem."} - host_filesystem_total_bytes: _filesystem_bytes & {description: "The total number of bytes in the named filesystem."} - host_filesystem_used_bytes: _filesystem_bytes & {description: "The number of bytes used on the named filesystem."} + host_filesystem_free_bytes: _host & _filesystem_bytes & {description: "The number of bytes free on the named filesystem."} + host_filesystem_total_bytes: _host & _filesystem_bytes & {description: "The total number of bytes in the named filesystem."} + host_filesystem_used_bytes: _host & _filesystem_bytes & {description: "The number of bytes used on the named filesystem."} // Host load - host_load1: _loadavg & {description: "System load averaged over the last 1 second."} - host_load5: _loadavg & {description: "System load averaged over the last 5 seconds."} - host_load15: _loadavg & {description: "System load averaged over the last 15 seconds."} + host_load1: _host & _loadavg & {description: "System load averaged over the last 1 second."} + host_load5: _host & _loadavg & {description: "System load averaged over the last 5 seconds."} + host_load15: _host & _loadavg & {description: "System load averaged over the last 15 seconds."} // Host memory - host_memory_active_bytes: _memory_gauge & _memory_nowin & {description: "The number of bytes of active main memory."} - host_memory_available_bytes: _memory_gauge & {description: "The number of bytes of main memory available."} - host_memory_buffers_bytes: _memory_linux & {description: "The number of bytes of main memory used by buffers."} - host_memory_cached_bytes: _memory_linux & {description: "The number of bytes of main memory used by cached blocks."} - host_memory_free_bytes: _memory_gauge & {description: "The number of bytes of main memory not used."} - host_memory_inactive_bytes: _memory_macos & {description: "The number of bytes of main memory that is not active."} - host_memory_shared_bytes: _memory_linux & {description: "The number of bytes of main memory shared between processes."} - host_memory_swap_free_bytes: _memory_gauge & {description: "The number of free bytes of swap space."} - host_memory_swapped_in_bytes_total: _memory_counter & _memory_nowin & { + host_memory_active_bytes: _host & _memory_gauge & _memory_nowin & {description: "The number of bytes of active main memory."} + host_memory_available_bytes: _host & _memory_gauge & {description: "The number of bytes of main memory available."} + host_memory_buffers_bytes: _host & _memory_linux & {description: "The number of bytes of main memory used by buffers."} + host_memory_cached_bytes: _host & _memory_linux & {description: "The number of bytes of main memory used by cached blocks."} + host_memory_free_bytes: _host & _memory_gauge & {description: "The number of bytes of main memory not used."} + host_memory_inactive_bytes: _host & _memory_macos & {description: "The number of bytes of main memory that is not active."} + host_memory_shared_bytes: _host & _memory_linux & {description: "The number of bytes of main memory shared between processes."} + host_memory_swap_free_bytes: _host & _memory_gauge & {description: "The number of free bytes of swap space."} + host_memory_swapped_in_bytes_total: _host & _memory_counter & _memory_nowin & { description: "The number of bytes that have been swapped in to main memory." } - host_memory_swapped_out_bytes_total: _memory_counter & _memory_nowin & { + host_memory_swapped_out_bytes_total: _host & _memory_counter & _memory_nowin & { description: "The number of bytes that have been swapped out from main memory." } - host_memory_swap_total_bytes: _memory_gauge & {description: "The total number of bytes of swap space."} - host_memory_swap_used_bytes: _memory_gauge & {description: "The number of used bytes of swap space."} - host_memory_total_bytes: _memory_gauge & {description: "The total number of bytes of main memory."} - host_memory_used_bytes: _memory_linux & {description: "The number of bytes of main memory used by programs or caches."} - host_memory_wired_bytes: _memory_macos & {description: "The number of wired bytes of main memory."} + host_memory_swap_total_bytes: _host & _memory_gauge & {description: "The total number of bytes of swap space."} + host_memory_swap_used_bytes: _host & _memory_gauge & {description: "The number of used bytes of swap space."} + host_memory_total_bytes: _host & _memory_gauge & {description: "The total number of bytes of main memory."} + host_memory_used_bytes: _host & _memory_linux & {description: "The number of bytes of main memory used by programs or caches."} + host_memory_wired_bytes: _host & _memory_macos & {description: "The number of wired bytes of main memory."} // Host network - host_network_receive_bytes_total: _network_gauge & {description: "The number of bytes received on this interface."} - host_network_receive_errs_total: _network_gauge & {description: "The number of errors encountered during receives on this interface."} - host_network_receive_packets_total: _network_gauge & {description: "The number of packets received on this interface."} - host_network_transmit_bytes_total: _network_gauge & {description: "The number of bytes transmitted on this interface."} - host_network_transmit_errs_total: _network_gauge & {description: "The number of errors encountered during transmits on this interface."} - host_network_transmit_packets_drop_total: _network_nomac & {description: "The number of packets dropped during transmits on this interface."} - host_network_transmit_packets_total: _network_nomac & {description: "The number of packets transmitted on this interface."} + host_network_receive_bytes_total: _host & _network_gauge & {description: "The number of bytes received on this interface."} + host_network_receive_errs_total: _host & _network_gauge & {description: "The number of errors encountered during receives on this interface."} + host_network_receive_packets_total: _host & _network_gauge & {description: "The number of packets received on this interface."} + host_network_transmit_bytes_total: _host & _network_gauge & {description: "The number of bytes transmitted on this interface."} + host_network_transmit_errs_total: _host & _network_gauge & {description: "The number of errors encountered during transmits on this interface."} + host_network_transmit_packets_drop_total: _host & _network_nomac & {description: "The number of packets dropped during transmits on this interface."} + host_network_transmit_packets_total: _host & _network_nomac & {description: "The number of packets transmitted on this interface."} // Helpers + _host: { + default_namespace: "host" + } + _disk_device: { description: "The disk device name." required: true diff --git a/docs/reference/components/sources/http.cue b/docs/reference/components/sources/http.cue index 0e54a25192da0..cfbef3a174bc3 100644 --- a/docs/reference/components/sources/http.cue +++ b/docs/reference/components/sources/http.cue @@ -208,6 +208,6 @@ components: sources: http: { ] telemetry: metrics: { - vector_http_bad_requests_total: telemetry.metrics._vector_http_bad_requests_total + http_bad_requests_total: telemetry.metrics._http_bad_requests_total } } diff --git a/docs/reference/components/sources/internal_metrics.cue b/docs/reference/components/sources/internal_metrics.cue index c01a7f2f9a2d6..fb879a45411f6 100644 --- a/docs/reference/components/sources/internal_metrics.cue +++ b/docs/reference/components/sources/internal_metrics.cue @@ -55,50 +55,59 @@ components: sources: internal_metrics: { } } - vector_api_started_total: { - description: "The number of times the Vector GraphQL API has been started." - type: "counter" - tags: _internal_metrics_tags + api_started_total: { + description: "The number of times the Vector GraphQL API has been started." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags } - vector_config_load_errors_total: { - description: "The total number of errors loading the Vector configuration." - type: "counter" - tags: _internal_metrics_tags + config_load_errors_total: { + description: "The total number of errors loading the Vector configuration." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags } - vector_connection_errors_total: { - description: "The total number of connection errors for this Vector instance." - type: "counter" - tags: _internal_metrics_tags + connection_errors_total: { + description: "The total number of connection errors for this Vector instance." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags } - vector_quit_total: { - description: "The total number of times the Vector instance has quit." - type: "counter" - tags: _internal_metrics_tags + quit_total: { + description: "The total number of times the Vector instance has quit." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags } - vector_recover_errors_total: { - description: "The total number of errors caused by Vector failing to recover from a failed reload." - type: "counter" - tags: _internal_metrics_tags + recover_errors_total: { + description: "The total number of errors caused by Vector failing to recover from a failed reload." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags } - vector_reload_errors_total: { - description: "The total number of errors encountered when reloading Vector." - type: "counter" - tags: _internal_metrics_tags + reload_errors_total: { + description: "The total number of errors encountered when reloading Vector." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags } - vector_reloaded_total: { - description: "The total number of times the Vector instance has been reloaded." - type: "counter" - tags: _internal_metrics_tags + reloaded_total: { + description: "The total number of times the Vector instance has been reloaded." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags } - vector_started_total: { - description: "The total number of times the Vector instance has been started." - type: "counter" - tags: _internal_metrics_tags + started_total: { + description: "The total number of times the Vector instance has been started." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags } - vector_stopped_total: { - description: "The total number of times the Vector instance has been stopped." - type: "counter" - tags: _internal_metrics_tags + stopped_total: { + description: "The total number of times the Vector instance has been stopped." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags } } } diff --git a/docs/reference/components/sources/journald.cue b/docs/reference/components/sources/journald.cue index 60f881d5b37da..66b870d25e50b 100644 --- a/docs/reference/components/sources/journald.cue +++ b/docs/reference/components/sources/journald.cue @@ -191,15 +191,17 @@ components: sources: journald: { } telemetry: metrics: { - vector_invalid_record_total: { - description: "The total number of invalid journald records discarded." - type: "counter" - tags: telemetry.metrics._component_tags + invalid_record_total: { + description: "The total number of invalid journald records discarded." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags } - vector_invalid_record_bytes_total: { - description: "The total number of bytes from discarded journald records." - type: "counter" - tags: telemetry.metrics._component_tags + invalid_record_bytes_total: { + description: "The total number of bytes from discarded journald records." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags } } } diff --git a/docs/reference/components/sources/kubernetes_logs.cue b/docs/reference/components/sources/kubernetes_logs.cue index f5c92420dfb25..061ba36809d4d 100644 --- a/docs/reference/components/sources/kubernetes_logs.cue +++ b/docs/reference/components/sources/kubernetes_logs.cue @@ -301,16 +301,18 @@ components: sources: kubernetes_logs: { } telemetry: metrics: { - vector_k8s_docker_format_parse_failures_total: { - description: "The total number of failures to parse a message as a JSON object." - type: "counter" - tags: telemetry.metrics._component_tags + docker_format_parse_failures_total: { + description: "The total number of failures to parse a message as a JSON object." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags } - vector_k8s_event_annotation_failures_total: { - description: "The total number of failures to annotate Vector events with Kubernetes Pod metadata." - type: "counter" - tags: telemetry.metrics._component_tags + event_annotation_failures_total: { + description: "The total number of failures to annotate Vector events with Kubernetes Pod metadata." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags } } } diff --git a/docs/reference/components/sources/logplex.cue b/docs/reference/components/sources/logplex.cue index 478d02f353dba..0a447b6b32431 100644 --- a/docs/reference/components/sources/logplex.cue +++ b/docs/reference/components/sources/logplex.cue @@ -99,7 +99,7 @@ components: sources: logplex: { } telemetry: metrics: { - vector_request_read_errors_total: telemetry.metrics._vector_request_read_errors_total - vector_requests_received_total: telemetry.metrics._vector_requests_received_total + request_read_errors_total: telemetry.metrics._request_read_errors_total + requests_received_total: telemetry.metrics._requests_received_total } } diff --git a/docs/reference/components/sources/mongodb_metrics.cue b/docs/reference/components/sources/mongodb_metrics.cue index 86fc44fda9db2..f7c0e6b48e8d4 100644 --- a/docs/reference/components/sources/mongodb_metrics.cue +++ b/docs/reference/components/sources/mongodb_metrics.cue @@ -96,19 +96,22 @@ components: sources: mongodb_metrics: { telemetry: metrics: { collect_duration_nanoseconds: { - description: "The duration spent collecting MongoDB metrics." - type: "histogram" - tags: telemetry.metrics._internal_metrics_tags + description: "The duration spent collecting MongoDB metrics." + type: "histogram" + default_namespace: "vector" + tags: telemetry.metrics._internal_metrics_tags } collect_completed_total: { - description: "The total number of MongoDB metrics collections completed." - type: "counter" - tags: telemetry.metrics._internal_metrics_tags + description: "The total number of MongoDB metrics collections completed." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._internal_metrics_tags } request_error_total: { - description: "The total number of MongoDB request errors." - type: "counter" - tags: telemetry.metrics._internal_metrics_tags + description: "The total number of MongoDB request errors." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._internal_metrics_tags } } @@ -127,10 +130,11 @@ components: sources: mongodb_metrics: { } } - mongodb_assets_total: { - description: "Number of assertions raised since the MongoDB process started." - type: "counter" - tags: _mongodb_metrics_tags & { + assets_total: { + description: "Number of assertions raised since the MongoDB process started." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "The assertion type" required: true @@ -138,15 +142,17 @@ components: sources: mongodb_metrics: { } } } - mongodb_bson_parse_error_total: { - description: "The total number of BSON parsing errors." - type: "counter" - tags: _mongodb_metrics_tags + bson_parse_error_total: { + description: "The total number of BSON parsing errors." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags } - mongodb_connections: { - description: "Number of connections in some state." - type: "gauge" - tags: _mongodb_metrics_tags & { + connections: { + description: "Number of connections in some state." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { state: { description: "The connection state" required: true @@ -154,36 +160,42 @@ components: sources: mongodb_metrics: { } } } - mongodb_extra_info_heap_usage_bytes: { - description: "The total size in bytes of heap space used by the database process." - relevant_when: "Unix/Linux" - type: "gauge" - tags: _mongodb_metrics_tags - } - mongodb_extra_info_page_faults: { - description: "The total number of page faults." - type: "gauge" - tags: _mongodb_metrics_tags - } - mongodb_instance_local_time: { - description: "The ISODate representing the current time, according to the server, in UTC." - type: "gauge" - tags: _mongodb_metrics_tags - } - mongodb_instance_uptime_estimate_seconds_total: { - description: "The uptime in seconds as calculated from MongoDB’s internal course-grained time keeping system." - type: "gauge" - tags: _mongodb_metrics_tags - } - mongodb_instance_uptime_seconds_total: { - description: "The number of seconds that the current MongoDB process has been active." - type: "gauge" - tags: _mongodb_metrics_tags - } - mongodb_memory: { - description: "Current memory unsage." - type: "gauge" - tags: _mongodb_metrics_tags & { + extra_info_heap_usage_bytes: { + description: "The total size in bytes of heap space used by the database process." + relevant_when: "Unix/Linux" + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + extra_info_page_faults: { + description: "The total number of page faults." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + instance_local_time: { + description: "The ISODate representing the current time, according to the server, in UTC." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + instance_uptime_estimate_seconds_total: { + description: "The uptime in seconds as calculated from MongoDB’s internal course-grained time keeping system." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + instance_uptime_seconds_total: { + description: "The number of seconds that the current MongoDB process has been active." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + memory: { + description: "Current memory unsage." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Memory type" required: true @@ -191,10 +203,11 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_global_lock_active_clients: { - description: "Number of connected clients and the read and write operations performed by these clients." - type: "gauge" - tags: _mongodb_metrics_tags & { + mongod_global_lock_active_clients: { + description: "Number of connected clients and the read and write operations performed by these clients." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Number type." required: true @@ -202,10 +215,11 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_global_lock_current_queue: { - description: "Number of operations queued because of a lock." - type: "gauge" - tags: _mongodb_metrics_tags & { + mongod_global_lock_current_queue: { + description: "Number of operations queued because of a lock." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Number type." required: true @@ -213,15 +227,17 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_global_lock_total_time_seconds: { - description: "The time since the database last started and created the globalLock. This is roughly equivalent to total server uptime." - type: "counter" - tags: _mongodb_metrics_tags + mongod_global_lock_total_time_seconds: { + description: "The time since the database last started and created the globalLock. This is roughly equivalent to total server uptime." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags } - mongodb_mongod_locks_time_acquiring_global_seconds_total: { - description: "Amount of time that any database has spent waiting for the global lock." - type: "counter" - tags: _mongodb_metrics_tags & { + mongod_locks_time_acquiring_global_seconds_total: { + description: "Amount of time that any database has spent waiting for the global lock." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Lock type." required: true @@ -234,10 +250,11 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_metrics_cursor_open: { - description: "Number of cursors." - type: "gauge" - tags: _mongodb_metrics_tags & { + mongod_metrics_cursor_open: { + description: "Number of cursors." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { state: { description: "Cursor state." required: true @@ -245,15 +262,17 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_metrics_cursor_timed_out_total: { - description: "The total number of cursors that have timed out since the server process started." - type: "counter" - tags: _mongodb_metrics_tags + mongod_metrics_cursor_timed_out_total: { + description: "The total number of cursors that have timed out since the server process started." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags } - mongodb_mongod_metrics_document_total: { - description: "Document access and modification patterns." - type: "counter" - tags: _mongodb_metrics_tags & { + mongod_metrics_document_total: { + description: "Document access and modification patterns." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { state: { description: "Document state." required: true @@ -261,25 +280,29 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_metrics_get_last_error_wtime_num: { - description: "The total number of getLastError operations with a specified write concern." - type: "gauge" - tags: _mongodb_metrics_tags + mongod_metrics_get_last_error_wtime_num: { + description: "The total number of getLastError operations with a specified write concern." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags } - mongodb_mongod_metrics_get_last_error_wtime_seconds_total: { - description: "The total amount of time that the mongod has spent performing getLastError operations." - type: "counter" - tags: _mongodb_metrics_tags + mongod_metrics_get_last_error_wtime_seconds_total: { + description: "The total amount of time that the mongod has spent performing getLastError operations." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags } - mongodb_mongod_metrics_get_last_error_wtimeouts_total: { - description: "The number of times that write concern operations have timed out as a result of the wtimeout threshold to getLastError." - type: "counter" - tags: _mongodb_metrics_tags + mongod_metrics_get_last_error_wtimeouts_total: { + description: "The number of times that write concern operations have timed out as a result of the wtimeout threshold to getLastError." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags } - mongodb_mongod_metrics_operation_total: { - description: "Update and query operations that MongoDB handles using special operation types." - type: "counter" - tags: _mongodb_metrics_tags & { + mongod_metrics_operation_total: { + description: "Update and query operations that MongoDB handles using special operation types." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Operation type." required: true @@ -287,10 +310,11 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_metrics_query_executor_total: { - description: "Data from query execution system." - type: "counter" - tags: _mongodb_metrics_tags & { + mongod_metrics_query_executor_total: { + description: "Data from query execution system." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { state: { description: "Query state." required: true @@ -298,45 +322,53 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_metrics_record_moves_total: { - description: "Moves reports the total number of times documents move within the on-disk representation of the MongoDB data set. Documents move as a result of operations that increase the size of the document beyond their allocated record size." - type: "counter" - tags: _mongodb_metrics_tags - } - mongodb_mongod_metrics_repl_apply_batches_num_total: { - description: "The total number of batches applied across all databases." - type: "counter" - tags: _mongodb_metrics_tags - } - mongodb_mongod_metrics_repl_apply_batches_seconds_total: { - description: "The total amount of time the mongod has spent applying operations from the oplog." - type: "counter" - tags: _mongodb_metrics_tags - } - mongodb_mongod_metrics_repl_apply_ops_total: { - description: "The total number of oplog operations applied." - type: "counter" - tags: _mongodb_metrics_tags - } - mongodb_mongod_metrics_repl_buffer_count: { - description: "The current number of operations in the oplog buffer." - type: "counter" - tags: _mongodb_metrics_tags - } - mongodb_mongod_metrics_repl_buffer_max_size_bytes_total: { - description: "The maximum size of the buffer." - type: "counter" - tags: _mongodb_metrics_tags - } - mongodb_mongod_metrics_repl_buffer_size_bytes: { - description: "The current size of the contents of the oplog buffer." - type: "counter" - tags: _mongodb_metrics_tags - } - mongodb_mongod_metrics_repl_executor_queue: { - description: "Number of queued operations in the replication executor." - type: "gauge" - tags: _mongodb_metrics_tags & { + mongod_metrics_record_moves_total: { + description: "Moves reports the total number of times documents move within the on-disk representation of the MongoDB data set. Documents move as a result of operations that increase the size of the document beyond their allocated record size." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + mongod_metrics_repl_apply_batches_num_total: { + description: "The total number of batches applied across all databases." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + mongod_metrics_repl_apply_batches_seconds_total: { + description: "The total amount of time the mongod has spent applying operations from the oplog." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + mongod_metrics_repl_apply_ops_total: { + description: "The total number of oplog operations applied." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + mongod_metrics_repl_buffer_count: { + description: "The current number of operations in the oplog buffer." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + mongod_metrics_repl_buffer_max_size_bytes_total: { + description: "The maximum size of the buffer." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + mongod_metrics_repl_buffer_size_bytes: { + description: "The current size of the contents of the oplog buffer." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + mongod_metrics_repl_executor_queue: { + description: "Number of queued operations in the replication executor." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Queue type." required: true @@ -344,50 +376,59 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_metrics_repl_executor_unsignaled_events: { - description: "Number of unsignaled events in the replication executor." - type: "gauge" - tags: _mongodb_metrics_tags - } - mongodb_mongod_metrics_repl_network_bytes_total: { - description: "The total amount of data read from the replication sync source." - type: "counter" - tags: _mongodb_metrics_tags - } - mongodb_mongod_metrics_repl_network_getmores_num_total: { - description: "The total number of getmore operations, which are operations that request an additional set of operations from the replication sync source." - type: "counter" - tags: _mongodb_metrics_tags - } - mongodb_mongod_metrics_repl_network_getmores_seconds_total: { - description: "The total amount of time required to collect data from getmore operations." - type: "counter" - tags: _mongodb_metrics_tags - } - mongodb_mongod_metrics_repl_network_ops_total: { - description: "The total number of operations read from the replication source." - type: "counter" - tags: _mongodb_metrics_tags - } - mongodb_mongod_metrics_repl_network_readers_created_total: { - description: "The total number of oplog query processes created." - type: "counter" - tags: _mongodb_metrics_tags - } - mongodb_mongod_metrics_ttl_deleted_documents_total: { - description: "The total number of documents deleted from collections with a ttl index." - type: "counter" - tags: _mongodb_metrics_tags - } - mongodb_mongod_metrics_ttl_passes_total: { - description: "The number of times the background process removes documents from collections with a ttl index." - type: "counter" - tags: _mongodb_metrics_tags - } - mongodb_mongod_op_latencies_histogram: { - description: "Latency statistics." - type: "gauge" - tags: _mongodb_metrics_tags & { + mongod_metrics_repl_executor_unsignaled_events: { + description: "Number of unsignaled events in the replication executor." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + mongod_metrics_repl_network_bytes_total: { + description: "The total amount of data read from the replication sync source." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + mongod_metrics_repl_network_getmores_num_total: { + description: "The total number of getmore operations, which are operations that request an additional set of operations from the replication sync source." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + mongod_metrics_repl_network_getmores_seconds_total: { + description: "The total amount of time required to collect data from getmore operations." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + mongod_metrics_repl_network_ops_total: { + description: "The total number of operations read from the replication source." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + mongod_metrics_repl_network_readers_created_total: { + description: "The total number of oplog query processes created." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + mongod_metrics_ttl_deleted_documents_total: { + description: "The total number of documents deleted from collections with a ttl index." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + mongod_metrics_ttl_passes_total: { + description: "The number of times the background process removes documents from collections with a ttl index." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags + } + mongod_op_latencies_histogram: { + description: "Latency statistics." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Latency type." required: true @@ -400,10 +441,11 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_op_latencies_latency: { - description: "A 64-bit integer giving the total combined latency in microseconds." - type: "gauge" - tags: _mongodb_metrics_tags & { + mongod_op_latencies_latency: { + description: "A 64-bit integer giving the total combined latency in microseconds." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Latency type." required: true @@ -411,10 +453,11 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_op_latencies_ops_total: { - description: "A 64-bit integer giving the total number of operations performed on the collection since startup." - type: "gauge" - tags: _mongodb_metrics_tags & { + mongod_op_latencies_ops_total: { + description: "A 64-bit integer giving the total number of operations performed on the collection since startup." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Latency type." required: true @@ -422,10 +465,11 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_storage_engine: { - description: "The name of the current storage engine." - type: "gauge" - tags: _mongodb_metrics_tags & { + mongod_storage_engine: { + description: "The name of the current storage engine." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { engine: { description: "Engine name." required: true @@ -433,11 +477,12 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_wiredtiger_blockmanager_blocks_total: { - description: "Statistics on the block manager operations." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags & { + mongod_wiredtiger_blockmanager_blocks_total: { + description: "Statistics on the block manager operations." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Operation type." required: true @@ -445,11 +490,12 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_wiredtiger_blockmanager_bytes_total: { - description: "Statistics on the block manager operations." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags & { + mongod_wiredtiger_blockmanager_bytes_total: { + description: "Statistics on the block manager operations." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Operation type." required: true @@ -457,11 +503,12 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_wiredtiger_cache_bytes: { - description: "Statistics on the cache and page evictions from the cache." - relevant_when: "Storage engine is `wiredTiger`." - type: "gauge" - tags: _mongodb_metrics_tags & { + mongod_wiredtiger_cache_bytes: { + description: "Statistics on the cache and page evictions from the cache." + relevant_when: "Storage engine is `wiredTiger`." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Statistics type." required: true @@ -469,11 +516,12 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_wiredtiger_cache_bytes_total: { - description: "Statistics on the cache and page evictions from the cache." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags & { + mongod_wiredtiger_cache_bytes_total: { + description: "Statistics on the cache and page evictions from the cache." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Statistics type." required: true @@ -481,11 +529,12 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_wiredtiger_cache_evicted_total: { - description: "Statistics on the cache and page evictions from the cache." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags & { + mongod_wiredtiger_cache_evicted_total: { + description: "Statistics on the cache and page evictions from the cache." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Statistics type." required: true @@ -493,21 +542,24 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_wiredtiger_cache_max_bytes: { - description: "Maximum cache size." - type: "gauge" - tags: _mongodb_metrics_tags + mongod_wiredtiger_cache_max_bytes: { + description: "Maximum cache size." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags } - mongodb_mongod_wiredtiger_cache_overhead_percent: { - description: "Percentage overhead." - type: "gauge" - tags: _mongodb_metrics_tags + mongod_wiredtiger_cache_overhead_percent: { + description: "Percentage overhead." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags } - mongodb_mongod_wiredtiger_cache_pages: { - description: "Pages in the cache." - relevant_when: "Storage engine is `wiredTiger`." - type: "gauge" - tags: _mongodb_metrics_tags & { + mongod_wiredtiger_cache_pages: { + description: "Pages in the cache." + relevant_when: "Storage engine is `wiredTiger`." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Pages type." required: true @@ -515,11 +567,12 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_wiredtiger_cache_pages_total: { - description: "Pages in the cache." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags & { + mongod_wiredtiger_cache_pages_total: { + description: "Pages in the cache." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Pages type." required: true @@ -527,11 +580,12 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_wiredtiger_concurrent_transactions_available_tickets: { - description: "Information on the number of concurrent of read and write transactions allowed into the WiredTiger storage engine" - relevant_when: "Storage engine is `wiredTiger`." - type: "gauge" - tags: _mongodb_metrics_tags & { + mongod_wiredtiger_concurrent_transactions_available_tickets: { + description: "Information on the number of concurrent of read and write transactions allowed into the WiredTiger storage engine" + relevant_when: "Storage engine is `wiredTiger`." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Transactions type." required: true @@ -539,11 +593,12 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_wiredtiger_concurrent_transactions_out_tickets: { - description: "Information on the number of concurrent of read and write transactions allowed into the WiredTiger storage engine" - relevant_when: "Storage engine is `wiredTiger`." - type: "gauge" - tags: _mongodb_metrics_tags & { + mongod_wiredtiger_concurrent_transactions_out_tickets: { + description: "Information on the number of concurrent of read and write transactions allowed into the WiredTiger storage engine" + relevant_when: "Storage engine is `wiredTiger`." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Transactions type." required: true @@ -551,11 +606,12 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_wiredtiger_concurrent_transactions_total_tickets: { - description: "Information on the number of concurrent of read and write transactions allowed into the WiredTiger storage engine" - relevant_when: "Storage engine is `wiredTiger`." - type: "gauge" - tags: _mongodb_metrics_tags & { + mongod_wiredtiger_concurrent_transactions_total_tickets: { + description: "Information on the number of concurrent of read and write transactions allowed into the WiredTiger storage engine" + relevant_when: "Storage engine is `wiredTiger`." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Transactions type." required: true @@ -563,11 +619,12 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_wiredtiger_log_bytes_total: { - description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags & { + mongod_wiredtiger_log_bytes_total: { + description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Bytes type." required: true @@ -575,11 +632,12 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_wiredtiger_log_operations_total: { - description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags & { + mongod_wiredtiger_log_operations_total: { + description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Operations type." required: true @@ -587,11 +645,12 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_wiredtiger_log_records_scanned_total: { - description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags & { + mongod_wiredtiger_log_records_scanned_total: { + description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Scanned records type." required: true @@ -599,23 +658,26 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_wiredtiger_log_records_total: { - description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags + mongod_wiredtiger_log_records_total: { + description: "Statistics on WiredTiger’s write ahead log (i.e. the journal)." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags } - mongodb_mongod_wiredtiger_session_open_sessions: { - description: "Open session count." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags + mongod_wiredtiger_session_open_sessions: { + description: "Open session count." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags } - mongodb_mongod_wiredtiger_transactions_checkpoint_seconds: { - description: "Statistics on transaction checkpoints and operations." - relevant_when: "Storage engine is `wiredTiger`." - type: "gauge" - tags: _mongodb_metrics_tags & { + mongod_wiredtiger_transactions_checkpoint_seconds: { + description: "Statistics on transaction checkpoints and operations." + relevant_when: "Storage engine is `wiredTiger`." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Checkpoint type." required: true @@ -623,23 +685,26 @@ components: sources: mongodb_metrics: { } } } - mongodb_mongod_wiredtiger_transactions_checkpoint_seconds_total: { - description: "Statistics on transaction checkpoints and operations." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags + mongod_wiredtiger_transactions_checkpoint_seconds_total: { + description: "Statistics on transaction checkpoints and operations." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags } - mongodb_mongod_wiredtiger_transactions_running_checkpoints: { - description: "Statistics on transaction checkpoints and operations." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags + mongod_wiredtiger_transactions_running_checkpoints: { + description: "Statistics on transaction checkpoints and operations." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags } - mongodb_mongod_wiredtiger_transactions_total: { - description: "Statistics on transaction checkpoints and operations." - relevant_when: "Storage engine is `wiredTiger`." - type: "counter" - tags: _mongodb_metrics_tags & { + mongod_wiredtiger_transactions_total: { + description: "Statistics on transaction checkpoints and operations." + relevant_when: "Storage engine is `wiredTiger`." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Transactions type." required: true @@ -647,10 +712,11 @@ components: sources: mongodb_metrics: { } } } - mongodb_network_bytes_total: { - description: "The number of bytes that reflects the amount of network traffic." - type: "counter" - tags: _mongodb_metrics_tags & { + network_bytes_total: { + description: "The number of bytes that reflects the amount of network traffic." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { state: { description: "Bytes state." required: true @@ -658,15 +724,17 @@ components: sources: mongodb_metrics: { } } } - mongodb_network_metrics_num_requests_total: { - description: "The total number of distinct requests that the server has received." - type: "counter" - tags: _mongodb_metrics_tags + network_metrics_num_requests_total: { + description: "The total number of distinct requests that the server has received." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags } mongodb_op_counters_repl_total: { - description: "Database replication operations by type since the mongod instance last started." - type: "counter" - tags: _mongodb_metrics_tags & { + description: "Database replication operations by type since the mongod instance last started." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Counter type." required: true @@ -675,9 +743,10 @@ components: sources: mongodb_metrics: { } } mongodb_op_counters_total: { - description: "Database operations by type since the mongod instance last started." - type: "counter" - tags: _mongodb_metrics_tags & { + description: "Database operations by type since the mongod instance last started." + type: "counter" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags & { type: { description: "Counter type." required: true @@ -685,10 +754,11 @@ components: sources: mongodb_metrics: { } } } - mongodb_up: { - description: "If the MongoDB server is up or not." - type: "gauge" - tags: _mongodb_metrics_tags + up: { + description: "If the MongoDB server is up or not." + type: "gauge" + default_namespace: "mongodb" + tags: _mongodb_metrics_tags } } } diff --git a/docs/reference/components/sources/socket.cue b/docs/reference/components/sources/socket.cue index ed94e640e0fba..673b2d5a79ae0 100644 --- a/docs/reference/components/sources/socket.cue +++ b/docs/reference/components/sources/socket.cue @@ -151,6 +151,6 @@ components: sources: socket: { ] telemetry: metrics: { - vector_connection_errors_total: telemetry.metrics._vector_connection_errors_total + connection_errors_total: telemetry.metrics._connection_errors_total } } diff --git a/docs/reference/components/sources/splunk_hec.cue b/docs/reference/components/sources/splunk_hec.cue index e7c07f2efdae3..c96d0efc965d0 100644 --- a/docs/reference/components/sources/splunk_hec.cue +++ b/docs/reference/components/sources/splunk_hec.cue @@ -94,10 +94,10 @@ components: sources: splunk_hec: { } telemetry: metrics: { - vector_source_missing_keys_total: telemetry.metrics._vector_source_missing_keys_total - vector_sourcetype_missing_keys_total: telemetry.metrics._vector_sourcetype_missing_keys_total - vector_encode_errors_total: telemetry.metrics._vector_encode_errors_total - vector_http_request_errors_total: telemetry.metrics._vector_http_request_errors_total - vector_requests_received_total: telemetry.metrics._vector_requests_received_total + source_missing_keys_total: telemetry.metrics._source_missing_keys_total + sourcetype_missing_keys_total: telemetry.metrics._sourcetype_missing_keys_total + encode_errors_total: telemetry.metrics._encode_errors_total + http_request_errors_total: telemetry.metrics._http_request_errors_total + requests_received_total: telemetry.metrics._requests_received_total } } diff --git a/docs/reference/components/sources/statsd.cue b/docs/reference/components/sources/statsd.cue index 7f594a2a37b96..1ebe7a699daac 100644 --- a/docs/reference/components/sources/statsd.cue +++ b/docs/reference/components/sources/statsd.cue @@ -122,16 +122,18 @@ components: sources: statsd: { } telemetry: metrics: { - vector_connection_errors_total: telemetry.metrics._vector_connection_errors_total - vector_invalid_record_total: { - description: "The total number of invalid StatsD records discarded." - type: "counter" - tags: telemetry.metrics._component_tags + connection_errors_total: telemetry.metrics._connection_errors_total + invalid_record_total: { + description: "The total number of invalid StatsD records discarded." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags } - vector_invalid_record_bytes_total: { - description: "The total number of bytes from StatsD journald records." - type: "counter" - tags: telemetry.metrics._component_tags + invalid_record_bytes_total: { + description: "The total number of bytes from StatsD journald records." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags } } } diff --git a/docs/reference/components/sources/stdin.cue b/docs/reference/components/sources/stdin.cue index dc2600537e490..33a7dc1229313 100644 --- a/docs/reference/components/sources/stdin.cue +++ b/docs/reference/components/sources/stdin.cue @@ -102,10 +102,11 @@ components: sources: stdin: { } telemetry: metrics: { - vector_stdin_reads_failed_total: { - description: "The total number of errors reading from stdin." - type: "counter" - tags: telemetry.metrics._component_tags + stdin_reads_failed_total: { + description: "The total number of errors reading from stdin." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags } } } diff --git a/docs/reference/components/sources/syslog.cue b/docs/reference/components/sources/syslog.cue index 414dee7809c04..d801775046190 100644 --- a/docs/reference/components/sources/syslog.cue +++ b/docs/reference/components/sources/syslog.cue @@ -184,10 +184,11 @@ components: sources: syslog: { } telemetry: metrics: { - vector_connection_read_errors_total: { - description: "The total number of errors reading datagram." - type: "counter" - tags: telemetry.metrics._component_tags & { + connection_read_errors_total: { + description: "The total number of errors reading datagram." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags & { mode: { description: "" required: true @@ -197,10 +198,11 @@ components: sources: syslog: { } } } - vector_utf8_convert_errors_total: { - description: "The total number of errors converting bytes to a UTF-8 string in UDP mode." - type: "counter" - tags: telemetry.metrics._component_tags & { + utf8_convert_errors_total: { + description: "The total number of errors converting bytes to a UTF-8 string in UDP mode." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags & { mode: { description: "The connection mode used by the component." required: true diff --git a/docs/reference/components/sources/vector.cue b/docs/reference/components/sources/vector.cue index 18ff8c934290a..381ecc0f13d4e 100644 --- a/docs/reference/components/sources/vector.cue +++ b/docs/reference/components/sources/vector.cue @@ -112,6 +112,6 @@ components: sources: vector: { } telemetry: metrics: { - vector_protobuf_decode_errors_total: telemetry.metrics._vector_protobuf_decode_errors_total + protobuf_decode_errors_total: telemetry.metrics._protobuf_decode_errors_total } } diff --git a/docs/reference/components/transforms/ansi_stripper.cue b/docs/reference/components/transforms/ansi_stripper.cue index e7e4567f7fcc6..25eb48445eefc 100644 --- a/docs/reference/components/transforms/ansi_stripper.cue +++ b/docs/reference/components/transforms/ansi_stripper.cue @@ -47,6 +47,6 @@ components: transforms: ansi_stripper: { } telemetry: metrics: { - vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total + processing_errors_total: telemetry.metrics._processing_errors_total } } diff --git a/docs/reference/components/transforms/aws_cloudwatch_logs_subscription_parser.cue b/docs/reference/components/transforms/aws_cloudwatch_logs_subscription_parser.cue index d05a4bdcf919a..f1505df60dc86 100644 --- a/docs/reference/components/transforms/aws_cloudwatch_logs_subscription_parser.cue +++ b/docs/reference/components/transforms/aws_cloudwatch_logs_subscription_parser.cue @@ -144,6 +144,6 @@ components: transforms: aws_cloudwatch_logs_subscription_parser: { } telemetry: metrics: { - vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total + processing_errors_total: telemetry.metrics._processing_errors_total } } diff --git a/docs/reference/components/transforms/aws_ec2_metadata.cue b/docs/reference/components/transforms/aws_ec2_metadata.cue index ea9b26c4ad6f0..65d0f8bdfed31 100644 --- a/docs/reference/components/transforms/aws_ec2_metadata.cue +++ b/docs/reference/components/transforms/aws_ec2_metadata.cue @@ -172,15 +172,17 @@ components: transforms: aws_ec2_metadata: { } telemetry: metrics: { - vector_metadata_refresh_failed_total: { - description: "The total number of failed efforts to refresh AWS EC2 metadata." - type: "counter" - tags: telemetry.metrics._component_tags + metadata_refresh_failed_total: { + description: "The total number of failed efforts to refresh AWS EC2 metadata." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags } - vector_metadata_refresh_successful_total: { - description: "The total number of AWS EC2 metadata refreshes." - type: "counter" - tags: telemetry.metrics._component_tags + metadata_refresh_successful_total: { + description: "The total number of AWS EC2 metadata refreshes." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags } } } diff --git a/docs/reference/components/transforms/dedupe.cue b/docs/reference/components/transforms/dedupe.cue index 894e3edc08efe..244e37e322d35 100644 --- a/docs/reference/components/transforms/dedupe.cue +++ b/docs/reference/components/transforms/dedupe.cue @@ -148,6 +148,6 @@ components: transforms: dedupe: { } telemetry: metrics: { - vector_events_discarded_total: telemetry.metrics._vector_events_discarded_total + events_discarded_total: telemetry.metrics._events_discarded_total } } diff --git a/docs/reference/components/transforms/grok_parser.cue b/docs/reference/components/transforms/grok_parser.cue index 4d802fd8eaf09..eb6664c82f5e3 100644 --- a/docs/reference/components/transforms/grok_parser.cue +++ b/docs/reference/components/transforms/grok_parser.cue @@ -97,6 +97,6 @@ components: transforms: grok_parser: { } telemetry: metrics: { - vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total + processing_errors_total: telemetry.metrics._processing_errors_total } } diff --git a/docs/reference/components/transforms/logfmt_parser.cue b/docs/reference/components/transforms/logfmt_parser.cue index dc699c64f628d..8e243f9baafbc 100644 --- a/docs/reference/components/transforms/logfmt_parser.cue +++ b/docs/reference/components/transforms/logfmt_parser.cue @@ -180,6 +180,6 @@ components: transforms: logfmt_parser: { } telemetry: metrics: { - vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total + processing_errors_total: telemetry.metrics._processing_errors_total } } diff --git a/docs/reference/components/transforms/lua.cue b/docs/reference/components/transforms/lua.cue index 663b1b3bde944..46e83fd8e776e 100644 --- a/docs/reference/components/transforms/lua.cue +++ b/docs/reference/components/transforms/lua.cue @@ -472,7 +472,7 @@ components: transforms: lua: { } telemetry: metrics: { - vector_memory_used: telemetry.metrics._vector_memory_used - vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total + memory_used: telemetry.metrics._memory_used + processing_errors_total: telemetry.metrics._processing_errors_total } } diff --git a/docs/reference/components/transforms/metric_to_log.cue b/docs/reference/components/transforms/metric_to_log.cue index 4a115ac0c25a1..8fe00007d9213 100644 --- a/docs/reference/components/transforms/metric_to_log.cue +++ b/docs/reference/components/transforms/metric_to_log.cue @@ -95,6 +95,6 @@ components: transforms: metric_to_log: { how_it_works: {} telemetry: metrics: { - vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total + processing_errors_total: telemetry.metrics._processing_errors_total } } diff --git a/docs/reference/components/transforms/reduce.cue b/docs/reference/components/transforms/reduce.cue index 2b13b2eea08d5..7faf8034088de 100644 --- a/docs/reference/components/transforms/reduce.cue +++ b/docs/reference/components/transforms/reduce.cue @@ -146,10 +146,11 @@ components: transforms: reduce: { ] telemetry: metrics: { - vector_stale_events_flushed_total: { - description: "The number of stale events that Vector has flushed." - type: "counter" - tags: telemetry.metrics._component_tags + stale_events_flushed_total: { + description: "The number of stale events that Vector has flushed." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags } } } diff --git a/docs/reference/components/transforms/regex_parser.cue b/docs/reference/components/transforms/regex_parser.cue index e882fe6d31f22..15c72913c6eba 100644 --- a/docs/reference/components/transforms/regex_parser.cue +++ b/docs/reference/components/transforms/regex_parser.cue @@ -180,6 +180,6 @@ components: transforms: regex_parser: { } telemetry: metrics: { - vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total + processing_errors_total: telemetry.metrics._processing_errors_total } } diff --git a/docs/reference/components/transforms/remap.cue b/docs/reference/components/transforms/remap.cue index d6993d95da6c4..ce0837f966f7b 100644 --- a/docs/reference/components/transforms/remap.cue +++ b/docs/reference/components/transforms/remap.cue @@ -132,6 +132,6 @@ components: transforms: remap: { } telemetry: metrics: { - vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total + processing_errors_total: telemetry.metrics._processing_errors_total } } diff --git a/docs/reference/components/transforms/sampler.cue b/docs/reference/components/transforms/sampler.cue index e362f48613ab3..523b7323452be 100644 --- a/docs/reference/components/transforms/sampler.cue +++ b/docs/reference/components/transforms/sampler.cue @@ -66,6 +66,6 @@ components: transforms: sampler: { } telemetry: metrics: { - vector_events_discarded_total: telemetry.metrics._vector_events_discarded_total + events_discarded_total: telemetry.metrics._events_discarded_total } } diff --git a/docs/reference/components/transforms/split.cue b/docs/reference/components/transforms/split.cue index 9f68365911453..dab2adb3d6040 100644 --- a/docs/reference/components/transforms/split.cue +++ b/docs/reference/components/transforms/split.cue @@ -97,6 +97,6 @@ components: transforms: split: { ] telemetry: metrics: { - vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total + processing_errors_total: telemetry.metrics._processing_errors_total } } diff --git a/docs/reference/components/transforms/swimlanes.cue b/docs/reference/components/transforms/swimlanes.cue index 234e2ed9cb333..faad9ee701007 100644 --- a/docs/reference/components/transforms/swimlanes.cue +++ b/docs/reference/components/transforms/swimlanes.cue @@ -72,6 +72,6 @@ components: transforms: swimlanes: { ] telemetry: metrics: { - vector_events_discarded_total: telemetry.metrics._vector_events_discarded_total + events_discarded_total: telemetry.metrics._events_discarded_total } } diff --git a/docs/reference/components/transforms/tag_cardinality_limit.cue b/docs/reference/components/transforms/tag_cardinality_limit.cue index a6c65f12ee68b..1a640bc915bfc 100644 --- a/docs/reference/components/transforms/tag_cardinality_limit.cue +++ b/docs/reference/components/transforms/tag_cardinality_limit.cue @@ -220,21 +220,23 @@ components: transforms: tag_cardinality_limit: { } telemetry: metrics: { - vector_tag_value_limit_exceeded_total: { + tag_value_limit_exceeded_total: { description: """ The total number of events discarded because the tag has been rejected after hitting the configured `value_limit`. """ - type: "counter" - tags: telemetry.metrics._component_tags + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags } - vector_value_limit_reached_total: { + value_limit_reached_total: { description: """ The total number of times new values for a key have been rejected because the value limit has been reached. """ - type: "counter" - tags: telemetry.metrics._component_tags + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags } } } diff --git a/docs/reference/components/transforms/tokenizer.cue b/docs/reference/components/transforms/tokenizer.cue index b80d8801c0b47..b5fd23af5fee9 100644 --- a/docs/reference/components/transforms/tokenizer.cue +++ b/docs/reference/components/transforms/tokenizer.cue @@ -116,6 +116,6 @@ components: transforms: tokenizer: { } telemetry: metrics: { - vector_processing_errors_total: telemetry.metrics._vector_processing_errors_total + processing_errors_total: telemetry.metrics._processing_errors_total } } From 7d0371388deb817afc6609d6004b4c979b80673a Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Thu, 12 Nov 2020 15:14:58 -0800 Subject: [PATCH 14/20] Refactor Splunk common metrics into separate file Signed-off-by: Luc Perkins --- docs/reference/components.cue | 23 ---------------- .../reference/components/sinks/splunk_hec.cue | 7 ++--- .../components/sources/splunk_hec.cue | 7 ++--- docs/reference/components/splunk.cue | 27 +++++++++++++++++++ 4 files changed, 31 insertions(+), 33 deletions(-) create mode 100644 docs/reference/components/splunk.cue diff --git a/docs/reference/components.cue b/docs/reference/components.cue index d121a18f1c457..e2839fe7e42c9 100644 --- a/docs/reference/components.cue +++ b/docs/reference/components.cue @@ -1129,29 +1129,6 @@ components: { tags: _component_tags } - // Splunk - _encode_errors_total: { - description: """ - The total number of errors encoding [Splunk HEC](\(urls.splunk_hec_protocol)) events - to JSON for this `splunk_hec` sink. - """ - type: "counter" - default_namespace: "vector" - tags: _component_tags - } - _source_missing_keys_total: { - description: "The total number of errors rendering the template for this source." - type: "counter" - default_namespace: "vector" - tags: _component_tags - } - _sourcetype_missing_keys_total: { - description: "The total number of errors rendering the template for this sourcetype." - type: "counter" - default_namespace: "vector" - tags: _component_tags - } - // Vector instance metrics _config_load_errors_total: { description: "The total number of errors loading the Vector configuration." diff --git a/docs/reference/components/sinks/splunk_hec.cue b/docs/reference/components/sinks/splunk_hec.cue index bae876477f2cb..15fe74350d5d9 100644 --- a/docs/reference/components/sinks/splunk_hec.cue +++ b/docs/reference/components/sinks/splunk_hec.cue @@ -167,10 +167,7 @@ components: sinks: splunk_hec: { } telemetry: metrics: { - encode_errors_total: telemetry.metrics._encode_errors_total - http_request_errors_total: telemetry.metrics._http_request_errors_total - http_requests_total: telemetry.metrics._http_requests_total - source_missing_keys_total: telemetry.metrics._source_missing_keys_total - sourcetype_missing_keys_total: telemetry.metrics._sourcetype_missing_keys_total + http_request_errors_total: telemetry.metrics._http_request_errors_total + http_requests_total: telemetry.metrics._http_requests_total } } diff --git a/docs/reference/components/sources/splunk_hec.cue b/docs/reference/components/sources/splunk_hec.cue index c96d0efc965d0..b48f1122ef8fc 100644 --- a/docs/reference/components/sources/splunk_hec.cue +++ b/docs/reference/components/sources/splunk_hec.cue @@ -94,10 +94,7 @@ components: sources: splunk_hec: { } telemetry: metrics: { - source_missing_keys_total: telemetry.metrics._source_missing_keys_total - sourcetype_missing_keys_total: telemetry.metrics._sourcetype_missing_keys_total - encode_errors_total: telemetry.metrics._encode_errors_total - http_request_errors_total: telemetry.metrics._http_request_errors_total - requests_received_total: telemetry.metrics._requests_received_total + http_request_errors_total: telemetry.metrics._http_request_errors_total + requests_received_total: telemetry.metrics._requests_received_total } } diff --git a/docs/reference/components/splunk.cue b/docs/reference/components/splunk.cue new file mode 100644 index 0000000000000..83c5bb6bc0f15 --- /dev/null +++ b/docs/reference/components/splunk.cue @@ -0,0 +1,27 @@ +package metadata + +components: _splunk: { + telemetry: metrics: { + encode_errors_total: { + description: """ + The total number of errors encoding [Splunk HEC](\(urls.splunk_hec_protocol)) events + to JSON for this `splunk_hec` sink. + """ + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags + } + source_missing_keys_total: { + description: "The total number of errors rendering the template for this source." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags + } + sourcetype_missing_keys_total: { + description: "The total number of errors rendering the template for this sourcetype." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags + } + } +} From 3b4c7ba47b245a65812dcc38b16969b469dc06d2 Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Mon, 16 Nov 2020 12:29:06 -0800 Subject: [PATCH 15/20] Address feedback on metric descriptions Signed-off-by: Luc Perkins --- docs/reference/components.cue | 40 +------------------ .../components/sources/docker_logs.cue | 2 +- docs/reference/components/sources/statsd.cue | 2 +- 3 files changed, 4 insertions(+), 40 deletions(-) diff --git a/docs/reference/components.cue b/docs/reference/components.cue index e2839fe7e42c9..6e05dd1363e89 100644 --- a/docs/reference/components.cue +++ b/docs/reference/components.cue @@ -977,7 +977,7 @@ components: { } } _file_watch_errors: { - description: "The total number of errors caused by failure to watch a file." + description: "The total number of errors encountered when watching files." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags & { @@ -1017,7 +1017,7 @@ components: { } } _fingerprint_read_errors: { - description: "The total number of times failing to read a file for fingerprinting." + description: "The total number of times Vector failed to read a file for fingerprinting." type: "counter" default_namespace: "vector" tags: _internal_metrics_tags & { @@ -1142,42 +1142,6 @@ components: { default_namespace: "vector" tags: _internal_metrics_tags } - _quit_total: { - description: "The total number of times the Vector instance has quit." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _recover_errors_total: { - description: "The total number of errors caused by Vector failing to recover from a failed reload." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _reload_errors_total: { - description: "The total number of errors encountered when reloading Vector." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _reloaded_total: { - description: "The total number of times the Vector instance has been reloaded." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _started_total: { - description: "The total number of times the Vector instance has been started." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _stopped_total: { - description: "The total number of times the Vector instance has been stopped." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } // Windows metrics _windows_service_does_not_exist: { diff --git a/docs/reference/components/sources/docker_logs.cue b/docs/reference/components/sources/docker_logs.cue index 278e9c3c4330f..2474863c76b84 100644 --- a/docs/reference/components/sources/docker_logs.cue +++ b/docs/reference/components/sources/docker_logs.cue @@ -280,7 +280,7 @@ components: sources: docker_logs: { tags: telemetry.metrics._component_tags } container_metadata_fetch_errors_total: { - description: "The total number of errors caused by failure to fetch container metadata." + description: "The total number of errors encountered when fetching container metadata." type: "counter" default_namespace: "vector" tags: telemetry.metrics._component_tags diff --git a/docs/reference/components/sources/statsd.cue b/docs/reference/components/sources/statsd.cue index 1ebe7a699daac..111747cddf835 100644 --- a/docs/reference/components/sources/statsd.cue +++ b/docs/reference/components/sources/statsd.cue @@ -124,7 +124,7 @@ components: sources: statsd: { telemetry: metrics: { connection_errors_total: telemetry.metrics._connection_errors_total invalid_record_total: { - description: "The total number of invalid StatsD records discarded." + description: "The total number of discarded invalid StatsD records." type: "counter" default_namespace: "vector" tags: telemetry.metrics._component_tags From dca04642d36a04772c99e3665e0b83b7e9d5749b Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Tue, 17 Nov 2020 15:04:04 -0800 Subject: [PATCH 16/20] Add default namespace to AWS ECS metrics Signed-off-by: Luc Perkins --- docs/reference.cue | 9 -- docs/reference/components.cue | 11 +- .../components/sources/aws_ecs_metrics.cue | 132 +++++++++--------- 3 files changed, 74 insertions(+), 78 deletions(-) diff --git a/docs/reference.cue b/docs/reference.cue index 0fce72f3bd129..e728bbe86f597 100644 --- a/docs/reference.cue +++ b/docs/reference.cue @@ -185,15 +185,6 @@ _values: { sum: float } -#MetricOutput: [Name=string]: close({ - description: string - relevant_when?: string - tags: #MetricTags - name: Name - type: #MetricType - default_namespace: string -}) - #MetricTags: [Name=string]: close({ name: Name description: string diff --git a/docs/reference/components.cue b/docs/reference/components.cue index 6e05dd1363e89..71b564a0084ed 100644 --- a/docs/reference/components.cue +++ b/docs/reference/components.cue @@ -354,11 +354,12 @@ components: { } #MetricOutput: [Name=string]: close({ - description: string - relevant_when?: string - tags: #MetricTags - name: Name - type: #MetricType + description: string + relevant_when?: string + tags: #MetricTags + name: Name + type: #MetricType + default_namespace: string }) #Output: { diff --git a/docs/reference/components/sources/aws_ecs_metrics.cue b/docs/reference/components/sources/aws_ecs_metrics.cue index 85d494377c351..5984950b4e2c8 100644 --- a/docs/reference/components/sources/aws_ecs_metrics.cue +++ b/docs/reference/components/sources/aws_ecs_metrics.cue @@ -103,6 +103,10 @@ components: sources: aws_ecs_metrics: { } output: metrics: { + _awsecs: { + default_namespace: "awsecs" + } + _tags: { container_id: { description: "The identifier of the ECS container." @@ -142,25 +146,25 @@ components: sources: aws_ecs_metrics: { } } - blkio_recursive_io_merged_total: _blkio_counter & {description: "Total number of bios/requests merged into requests."} - blkio_recursive_io_queued_total: _blkio_counter & {description: "Total number of requests queued up at any given instant."} - blkio_recursive_io_service_bytes_total: _blkio_counter & {description: "Number of bytes transferred to/from the disk."} - blkio_recursive_io_service_time_seconds_total: _blkio_counter & {description: "Total amount of time in seconds between request dispatch and request completion for the IOs done."} - blkio_recursive_io_serviced_total: _blkio_counter & {description: "Number of IOs completed to/from the disk."} - blkio_recursive_io_time_seconds_total: _blkio_counter & {description: "Disk time allocated per device in seconds."} - blkio_recursive_io_wait_time_seconds_total: _blkio_counter & {description: "Total amount of time in seconds the IOs spent waiting in the scheduler queues for service."} - blkio_recursive_sectors_total: _blkio_counter & {description: "Number of sectors transferred to/from disk."} - - cpu_online_cpus: _gauge & {description: "Number of CPU cores."} - cpu_usage_system_jiffies_total: _counter & {description: "Jiffies of CPU time used by the system."} - cpu_usage_usermode_jiffies_total: _counter & {description: "Jiffies of CPU time spent in user mode by the container."} - cpu_usage_kernelmode_jiffies_total: _counter & {description: "Jiffies of CPU time spent in kernel mode by the container."} - cpu_usage_total_jiffies_total: _counter & {description: "Jiffies of CPU time used by the container."} - cpu_throttling_periods_total: _counter & {description: "Number of periods."} - cpu_throttled_periods_total: _counter & {description: "Number of periods throttled."} - cpu_throttled_time_seconds_total: _counter & {description: "Throttling time in seconds."} - - cpu_usage_percpu_jiffies_total: { + blkio_recursive_io_merged_total: _awsecs & _blkio_counter & {description: "Total number of bios/requests merged into requests."} + blkio_recursive_io_queued_total: _awsecs & _blkio_counter & {description: "Total number of requests queued up at any given instant."} + blkio_recursive_io_service_bytes_total: _awsecs & _blkio_counter & {description: "Number of bytes transferred to/from the disk."} + blkio_recursive_io_service_time_seconds_total: _awsecs & _blkio_counter & {description: "Total amount of time in seconds between request dispatch and request completion for the IOs done."} + blkio_recursive_io_serviced_total: _awsecs & _blkio_counter & {description: "Number of IOs completed to/from the disk."} + blkio_recursive_io_time_seconds_total: _awsecs & _blkio_counter & {description: "Disk time allocated per device in seconds."} + blkio_recursive_io_wait_time_seconds_total: _awsecs & _blkio_counter & {description: "Total amount of time in seconds the IOs spent waiting in the scheduler queues for service."} + blkio_recursive_sectors_total: _awsecs & _blkio_counter & {description: "Number of sectors transferred to/from disk."} + + cpu_online_cpus: _awsecs & _gauge & {description: "Number of CPU cores."} + cpu_usage_system_jiffies_total: _awsecs & _counter & {description: "Jiffies of CPU time used by the system."} + cpu_usage_usermode_jiffies_total: _awsecs & _counter & {description: "Jiffies of CPU time spent in user mode by the container."} + cpu_usage_kernelmode_jiffies_total: _awsecs & _counter & {description: "Jiffies of CPU time spent in kernel mode by the container."} + cpu_usage_total_jiffies_total: _awsecs & _counter & {description: "Jiffies of CPU time used by the container."} + cpu_throttling_periods_total: _awsecs & _counter & {description: "Number of periods."} + cpu_throttled_periods_total: _awsecs & _counter & {description: "Number of periods throttled."} + cpu_throttled_time_seconds_total: _awsecs & _counter & {description: "Throttling time in seconds."} + + cpu_usage_percpu_jiffies_total: _awsecs & { description: "Jiffies of CPU time used by the container, per CPU core." type: "counter" tags: _tags & { @@ -172,43 +176,43 @@ components: sources: aws_ecs_metrics: { } } - memory_used_bytes: _gauge & {description: "Memory used by the container, in bytes."} - memory_max_used_bytes: _gauge & {description: "Maximum measured memory usage of the container, in bytes."} - memory_limit_bytes: _gauge & {description: "Memory usage limit of the container, in bytes."} - memory_active_anonymous_bytes: _gauge & {description: "Amount of memory that has been identified as active by the kernel. Anonymous memory is memory that is not linked to disk pages."} - memory_active_file_bytes: _gauge & {description: "Amount of active file cache memory. Cache memory = active_file + inactive_file + tmpfs."} - memory_cache_bytes: _gauge & {description: "The amount of memory used by the processes of this cgroup that can be associated with a block on a block device. Also accounts for memory used by tmpfs."} - memory_dirty_bytes: _gauge & {description: "The amount of memory waiting to get written to disk."} - memory_inactive_anonymous_bytes: _gauge & {description: "Amount of memory that has been identified as inactive by the kernel."} - memory_inactive_file_bytes: _gauge & {description: "Amount of inactive file cache memory."} - memory_mapped_file_bytes: _gauge & {description: "Indicates the amount of memory mapped by the processes in the cgroup. It doesn’t give you information about how much memory is used; it rather tells you how it is used."} - memory_page_faults_total: _counter & {description: "Number of times that a process of the cgroup triggered a page fault."} - memory_major_faults_total: _counter & {description: "Number of times that a process of the cgroup triggered a major page fault."} - memory_page_charged_total: _counter & {description: "Number of charging events to the memory cgroup. Charging events happen each time a page is accounted as either mapped anon page(RSS) or cache page to the cgroup."} - memory_page_uncharged_total: _counter & {description: "Number of uncharging events to the memory cgroup. Uncharging events happen each time a page is unaccounted from the cgroup."} - memory_rss_bytes: _gauge & {description: "The amount of memory that doesn’t correspond to anything on disk: stacks, heaps, and anonymous memory maps."} - memory_rss_hugepages_bytes: _gauge & {description: "Amount of memory due to anonymous transparent hugepages."} - memory_unevictable_bytes: _gauge & {description: "The amount of memory that cannot be reclaimed."} - memory_writeback_bytes: _gauge & {description: "The amount of memory from file/anon cache that are queued for syncing to the disk."} - memory_total_active_anonymous_bytes: _gauge & {description: "Total amount of memory that has been identified as active by the kernel."} - memory_total_active_file_bytes: _gauge & {description: "Total amount of active file cache memory."} - memory_total_cache_bytes: _gauge & {description: "Total amount of memory used by the processes of this cgroup that can be associated with a block on a block device."} - memory_total_dirty_bytes: _gauge & {description: "Total amount of memory waiting to get written to disk."} - memory_total_inactive_anonymous_bytes: _gauge & {description: "Total amount of memory that has been identified as inactive by the kernel."} - memory_total_inactive_file_bytes: _gauge & {description: "Total amount of inactive file cache memory."} - memory_total_mapped_file_bytes: _gauge & {description: "Total amount of memory mapped by the processes in the cgroup."} - memory_total_page_faults_total: _counter & {description: "Total number of page faults."} - memory_total_major_faults_total: _counter & {description: "Total number of major page faults."} - memory_total_page_charged_total: _counter & {description: "Total number of charging events."} - memory_total_page_uncharged_total: _counter & {description: "Total number of uncharging events."} - memory_total_rss_bytes: _gauge & {description: "Total amount of memory that doesn’t correspond to anything on disk: stacks, heaps, and anonymous memory maps."} - memory_total_rss_hugepages_bytes: _gauge & {description: "Total amount of memory due to anonymous transparent hugepages."} - memory_total_unevictable_bytes: _gauge & {description: "Total amount of memory that can not be reclaimed."} - memory_total_writeback_bytes: _gauge & {description: "Total amount of memory from file/anon cache that are queued for syncing to the disk."} - memory_hierarchical_memory_limit_bytes: _gauge & {description: "The memory limit in place by the hierarchy cgroup."} - memory_hierarchical_memsw_limit_bytes: _gauge & {description: "The memory + swap limit in place by the hierarchy cgroup."} - - _network_counter: { + memory_used_bytes: _awsecs & _gauge & {description: "Memory used by the container, in bytes."} + memory_max_used_bytes: _awsecs & _gauge & {description: "Maximum measured memory usage of the container, in bytes."} + memory_limit_bytes: _awsecs & _gauge & {description: "Memory usage limit of the container, in bytes."} + memory_active_anonymous_bytes: _awsecs & _gauge & {description: "Amount of memory that has been identified as active by the kernel. Anonymous memory is memory that is not linked to disk pages."} + memory_active_file_bytes: _awsecs & _gauge & {description: "Amount of active file cache memory. Cache memory = active_file + inactive_file + tmpfs."} + memory_cache_bytes: _awsecs & _awsecs & _gauge & {description: "The amount of memory used by the processes of this cgroup that can be associated with a block on a block device. Also accounts for memory used by tmpfs."} + memory_dirty_bytes: _awsecs & _gauge & {description: "The amount of memory waiting to get written to disk."} + memory_inactive_anonymous_bytes: _awsecs & _gauge & {description: "Amount of memory that has been identified as inactive by the kernel."} + memory_inactive_file_bytes: _awsecs & _gauge & {description: "Amount of inactive file cache memory."} + memory_mapped_file_bytes: _awsecs & _gauge & {description: "Indicates the amount of memory mapped by the processes in the cgroup. It doesn’t give you information about how much memory is used; it rather tells you how it is used."} + memory_page_faults_total: _awsecs & _counter & {description: "Number of times that a process of the cgroup triggered a page fault."} + memory_major_faults_total: _awsecs & _counter & {description: "Number of times that a process of the cgroup triggered a major page fault."} + memory_page_charged_total: _awsecs & _counter & {description: "Number of charging events to the memory cgroup. Charging events happen each time a page is accounted as either mapped anon page(RSS) or cache page to the cgroup."} + memory_page_uncharged_total: _awsecs & _counter & {description: "Number of uncharging events to the memory cgroup. Uncharging events happen each time a page is unaccounted from the cgroup."} + memory_rss_bytes: _awsecs & _gauge & {description: "The amount of memory that doesn’t correspond to anything on disk: stacks, heaps, and anonymous memory maps."} + memory_rss_hugepages_bytes: _awsecs & _gauge & {description: "Amount of memory due to anonymous transparent hugepages."} + memory_unevictable_bytes: _awsecs & _gauge & {description: "The amount of memory that cannot be reclaimed."} + memory_writeback_bytes: _awsecs & _gauge & {description: "The amount of memory from file/anon cache that are queued for syncing to the disk."} + memory_total_active_anonymous_bytes: _awsecs & _gauge & {description: "Total amount of memory that has been identified as active by the kernel."} + memory_total_active_file_bytes: _awsecs & _gauge & {description: "Total amount of active file cache memory."} + memory_total_cache_bytes: _awsecs & _gauge & {description: "Total amount of memory used by the processes of this cgroup that can be associated with a block on a block device."} + memory_total_dirty_bytes: _awsecs & _gauge & {description: "Total amount of memory waiting to get written to disk."} + memory_total_inactive_anonymous_bytes: _awsecs & _gauge & {description: "Total amount of memory that has been identified as inactive by the kernel."} + memory_total_inactive_file_bytes: _awsecs & _gauge & {description: "Total amount of inactive file cache memory."} + memory_total_mapped_file_bytes: _awsecs & _gauge & {description: "Total amount of memory mapped by the processes in the cgroup."} + memory_total_page_faults_total: _awsecs & _counter & {description: "Total number of page faults."} + memory_total_major_faults_total: _awsecs & _counter & {description: "Total number of major page faults."} + memory_total_page_charged_total: _awsecs & _counter & {description: "Total number of charging events."} + memory_total_page_uncharged_total: _awsecs & _counter & {description: "Total number of uncharging events."} + memory_total_rss_bytes: _awsecs & _gauge & {description: "Total amount of memory that doesn’t correspond to anything on disk: stacks, heaps, and anonymous memory maps."} + memory_total_rss_hugepages_bytes: _awsecs & _gauge & {description: "Total amount of memory due to anonymous transparent hugepages."} + memory_total_unevictable_bytes: _awsecs & _gauge & {description: "Total amount of memory that can not be reclaimed."} + memory_total_writeback_bytes: _awsecs & _gauge & {description: "Total amount of memory from file/anon cache that are queued for syncing to the disk."} + memory_hierarchical_memory_limit_bytes: _awsecs & _gauge & {description: "The memory limit in place by the hierarchy cgroup."} + memory_hierarchical_memsw_limit_bytes: _awsecs & _gauge & {description: "The memory + swap limit in place by the hierarchy cgroup."} + + _network_counter: _awsecs & { type: "counter" tags: _tags & { device: { @@ -219,13 +223,13 @@ components: sources: aws_ecs_metrics: { } } - network_receive_bytes_total: _network_counter & {description: "Bytes received by the container via the network interface."} - network_receive_packets_total: _network_counter & {description: "Number of packets received by the container via the network interface."} - network_receive_packets_drop_total: _network_counter & {description: "Number of inbound packets dropped by the container."} - network_receive_errs_total: _network_counter & {description: "Errors receiving packets."} - network_transmit_bytes_total: _network_counter & {description: "Bytes sent by the container via the network interface."} - network_transmit_packets_total: _network_counter & {description: "Number of packets sent by the container via the network interface."} - network_transmit_packets_drop_total: _network_counter & {description: "Number of outbound packets dropped by the container."} - network_transmit_errs_total: _network_counter & {description: "Errors sending packets."} + network_receive_bytes_total: _awsecs & _network_counter & {description: "Bytes received by the container via the network interface."} + network_receive_packets_total: _awsecs & _network_counter & {description: "Number of packets received by the container via the network interface."} + network_receive_packets_drop_total: _awsecs & _network_counter & {description: "Number of inbound packets dropped by the container."} + network_receive_errs_total: _awsecs & _network_counter & {description: "Errors receiving packets."} + network_transmit_bytes_total: _awsecs & _network_counter & {description: "Bytes sent by the container via the network interface."} + network_transmit_packets_total: _awsecs & _network_counter & {description: "Number of packets sent by the container via the network interface."} + network_transmit_packets_drop_total: _awsecs & _network_counter & {description: "Number of outbound packets dropped by the container."} + network_transmit_errs_total: _awsecs & _network_counter & {description: "Errors sending packets."} } } From 4f2b014f12abc945eece37a6880054914e50f5d2 Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Wed, 18 Nov 2020 10:14:37 -0800 Subject: [PATCH 17/20] Add _total suffix to Windows metrics Signed-off-by: Luc Perkins --- docs/reference/components.cue | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/reference/components.cue b/docs/reference/components.cue index 71b564a0084ed..0f171b9052701 100644 --- a/docs/reference/components.cue +++ b/docs/reference/components.cue @@ -1145,7 +1145,7 @@ components: { } // Windows metrics - _windows_service_does_not_exist: { + _windows_service_does_not_exist_total: { description: """ The total number of errors raised due to the Windows service not existing. @@ -1154,7 +1154,7 @@ components: { default_namespace: "vector" tags: _internal_metrics_tags } - _windows_service_install: { + _windows_service_install_total: { description: """ The total number of times the Windows service has been installed. """ @@ -1162,7 +1162,7 @@ components: { default_namespace: "vector" tags: _internal_metrics_tags } - _windows_service_restart: { + _windows_service_restart_total: { description: """ The total number of times the Windows service has been restarted. """ @@ -1170,7 +1170,7 @@ components: { default_namespace: "vector" tags: _internal_metrics_tags } - _windows_service_start: { + _windows_service_start_total: { description: """ The total number of times the Windows service has been started. """ @@ -1178,7 +1178,7 @@ components: { default_namespace: "vector" tags: _internal_metrics_tags } - _windows_service_stop: { + _windows_service_stop_total: { description: """ The total number of times the Windows service has been stopped. """ @@ -1186,7 +1186,7 @@ components: { default_namespace: "vector" tags: _internal_metrics_tags } - _windows_service_uninstall: { + _windows_service_uninstall_total: { description: """ The total number of times the Windows service has been uninstalled. """ From 8b527445bb02698ec92ae060a8339ae7b4f7a658 Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Wed, 18 Nov 2020 10:36:59 -0800 Subject: [PATCH 18/20] Add new options for the error_type tag Signed-off-by: Luc Perkins --- docs/reference/components.cue | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/reference/components.cue b/docs/reference/components.cue index 0f171b9052701..97f013d90b2c7 100644 --- a/docs/reference/components.cue +++ b/docs/reference/components.cue @@ -1243,6 +1243,8 @@ components: { "parse_failed", "render_error", "type_conversion_failed", + "type_field_does_not_exist", + "type_ip_address_parse_error", "value_invalid", ] } From 9169269aa042c325d4b85182b581f681d0cf19a1 Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Wed, 18 Nov 2020 10:56:31 -0800 Subject: [PATCH 19/20] Add AWS S3+SQS metrics to docs Signed-off-by: Luc Perkins --- docs/reference/components/sources/aws_s3.cue | 67 ++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/docs/reference/components/sources/aws_s3.cue b/docs/reference/components/sources/aws_s3.cue index f869292a21a58..7e015c22b49b3 100644 --- a/docs/reference/components/sources/aws_s3.cue +++ b/docs/reference/components/sources/aws_s3.cue @@ -239,4 +239,71 @@ components: sources: aws_s3: { """ } } + + telemetry: metrics: { + sqs_message_delete_failed_total: { + description: "The total number of failures to delete SQS messages." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags + } + + sqs_message_delete_succeeded_total: { + description: "The total number of successful deletions of SQS messages." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags + } + + sqs_message_processing_failed_total: { + description: "The total number of failures to process SQS messages." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags + } + + sqs_message_processing_succeeded_total: { + description: "The total number of SQS messages successfully processed." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags + } + + sqs_message_receive_failed_total: { + description: "The total number of failures to receive SQS messages." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags + } + + sqs_message_receive_succeeded_total: { + description: "The total number of times successfully receiving SQS messages." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags + } + + sqs_message_received_messages_total: { + description: "The total number of received SQS messages." + type: "counter" + default_namespace: "vector" + tags: telemetry.metrics._component_tags + } + + sqs_s3_event_record_ignored_total: { + description: "The total number of times an S3 record in an SQS message was ignored (for an event that was not `ObjectCreated`)." + type: "counter" + default_namespace: "vector" + + tags: telemetry.metrics._component_tags & { + ignore_type: { + description: "The reason for ignoring the S3 record" + required: true + options: [ + "invalid_event_kind", + ] + } + } + } + } } From 974bdd8fd2851f88f9c672403f274ca8c0242c95 Mon Sep 17 00:00:00 2001 From: Luc Perkins Date: Thu, 19 Nov 2020 15:06:54 -0800 Subject: [PATCH 20/20] Refactor all vector namespace metrics into internal_metrics.cue Signed-off-by: Luc Perkins --- docs/reference/components.cue | 362 +---------- docs/reference/components/kafka.cue | 12 +- .../components/sinks/aws_cloudwatch_logs.cue | 2 +- docs/reference/components/sinks/console.cue | 2 +- .../components/sinks/elasticsearch.cue | 2 +- docs/reference/components/sinks/http.cue | 2 +- .../components/sinks/sematext_metrics.cue | 2 +- .../reference/components/sinks/splunk_hec.cue | 4 +- docs/reference/components/sinks/statsd.cue | 2 +- docs/reference/components/sinks/vector.cue | 2 +- .../sources/aws_kinesis_firehose.cue | 4 +- docs/reference/components/sources/aws_s3.cue | 72 +-- .../components/sources/docker_logs.cue | 45 +- docs/reference/components/sources/file.cue | 20 +- docs/reference/components/sources/http.cue | 2 +- .../components/sources/internal_metrics.cue | 567 ++++++++++++++++++ .../reference/components/sources/journald.cue | 14 +- .../components/sources/kubernetes_logs.cue | 15 +- docs/reference/components/sources/logplex.cue | 4 +- .../components/sources/mongodb_metrics.cue | 21 +- docs/reference/components/sources/socket.cue | 2 +- .../components/sources/splunk_hec.cue | 4 +- docs/reference/components/sources/statsd.cue | 16 +- docs/reference/components/sources/stdin.cue | 7 +- docs/reference/components/sources/syslog.cue | 30 +- docs/reference/components/sources/vector.cue | 2 +- .../components/transforms/ansi_stripper.cue | 2 +- ...ws_cloudwatch_logs_subscription_parser.cue | 2 +- .../transforms/aws_ec2_metadata.cue | 14 +- .../components/transforms/dedupe.cue | 2 +- .../components/transforms/grok_parser.cue | 2 +- .../components/transforms/logfmt_parser.cue | 2 +- docs/reference/components/transforms/lua.cue | 4 +- .../components/transforms/metric_to_log.cue | 2 +- .../components/transforms/reduce.cue | 7 +- .../components/transforms/regex_parser.cue | 2 +- .../reference/components/transforms/remap.cue | 2 +- .../components/transforms/sampler.cue | 2 +- .../reference/components/transforms/split.cue | 2 +- .../components/transforms/swimlanes.cue | 2 +- .../transforms/tag_cardinality_limit.cue | 20 +- .../components/transforms/tokenizer.cue | 2 +- 42 files changed, 645 insertions(+), 641 deletions(-) diff --git a/docs/reference/components.cue b/docs/reference/components.cue index 97f013d90b2c7..93b562fb56e1f 100644 --- a/docs/reference/components.cue +++ b/docs/reference/components.cue @@ -907,366 +907,8 @@ components: { telemetry: metrics: { // Default metrics for each component - events_processed_total: _events_processed_total - processed_bytes_total: _processed_bytes_total - - // Reusable metric definitions - _auto_concurrency_averaged_rtt: { - description: "The average round-trip time (RTT) from the HTTP sink across the current window." - type: "histogram" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _auto_concurrency_in_flight: { - description: "The number of outbound requests from the HTTP sink currently awaiting a response." - type: "histogram" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _auto_concurrency_limit: { - description: "The concurrency limit that the auto-concurrency feature has decided on for this current window." - type: "histogram" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _auto_concurrency_observed_rtt: { - description: "The observed round-trip time (RTT) for requests from this HTTP sink." - type: "histogram" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _checkpoint_write_errors_total: { - description: "The total number of errors writing checkpoints." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _checkpoints_total: { - description: "The total number of files checkpointed." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _checksum_errors: { - description: "The total number of errors identifying files via checksum." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags & { - file: _file - } - } - _events_discarded_total: { - description: "The total number of events discarded by this component." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _events_processed_total: { - description: "The total number of events processed by this component." - type: "counter" - default_namespace: "vector" - tags: _component_tags & { - file: _file - } - } - _file_delete_errors: { - description: "The total number of failures to delete a file." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags & { - file: _file - } - } - _file_watch_errors: { - description: "The total number of errors encountered when watching files." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags & { - file: _file - } - } - _files_added: { - description: "The total number of files Vector has found to watch." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags & { - file: _file - } - } - _files_deleted: { - description: "The total number of files deleted." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags & { - file: _file - } - } - _files_resumed: { - description: "The total number of times Vector has resumed watching a file." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags & { - file: _file - } - } - _files_unwatched: { - description: "The total number of times Vector has stopped watching a file." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags & { - file: _file - } - } - _fingerprint_read_errors: { - description: "The total number of times Vector failed to read a file for fingerprinting." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags & { - file: _file - } - } - _http_bad_requests_total: { - description: "The total number of HTTP `400 Bad Request` errors encountered." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _http_error_response_total: { - description: "The total number of HTTP error responses for this component." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _http_request_errors_total: { - description: "The total number of HTTP request errors for this component." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _http_requests_total: { - description: "The total number of HTTP requests issued by this component." - type: "counter" - default_namespace: "vector" - tags: _component_tags - } - _memory_used: { - description: "The total memory currently being used by Vector (in bytes)." - type: "gauge" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _missing_keys_total: { - description: "The total number of events dropped due to keys missing from the event." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _open_connections: { - description: "The number of current open connections to Vector." - type: "gauge" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _parse_errors_total: { - description: "The total number of errors parsing Prometheus metrics." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _processed_bytes_total: { - description: "The total number of bytes processed by the component." - type: "counter" - default_namespace: "vector" - tags: _component_tags - } - _processing_errors_total: { - description: "The total number of processing errors encountered by this component." - type: "counter" - default_namespace: "vector" - tags: _component_tags & { - error_type: _error_type - } - } - _protobuf_decode_errors_total: { - description: "The total number of [Protocol Buffers](\(urls.protobuf)) errors thrown during communication between Vector instances." - type: "counter" - default_namespace: "vector" - tags: _component_tags - } - _request_duration_nanoseconds: { - description: "The request duration for this component (in nanoseconds)." - type: "histogram" - default_namespace: "vector" - tags: _component_tags - } - _request_read_errors_total: { - description: "The total number of request read errors for this component." - type: "counter" - default_namespace: "vector" - tags: _component_tags - } - _requests_completed_total: { - description: "The total number of requests completed by this component." - type: "counter" - default_namespace: "vector" - tags: _component_tags - } - _requests_received_total: { - description: "The total number of requests received by this component." - type: "counter" - default_namespace: "vector" - tags: _component_tags - } - _timestamp_parse_errors_total: { - description: "The total number of errors encountered parsing [RFC3339](\(urls.rfc_3339)) timestamps." - type: "counter" - default_namespace: "vector" - tags: _component_tags - } - _uptime_seconds: { - description: "The total number of seconds the Vector instance has been up." - type: "gauge" - default_namespace: "vector" - tags: _component_tags - } - - // Vector instance metrics - _config_load_errors_total: { - description: "The total number of errors loading the Vector configuration." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _connection_errors_total: { - description: "The total number of connection errors for this Vector instance." - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - - // Windows metrics - _windows_service_does_not_exist_total: { - description: """ - The total number of errors raised due to the Windows service not - existing. - """ - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _windows_service_install_total: { - description: """ - The total number of times the Windows service has been installed. - """ - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _windows_service_restart_total: { - description: """ - The total number of times the Windows service has been restarted. - """ - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _windows_service_start_total: { - description: """ - The total number of times the Windows service has been started. - """ - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _windows_service_stop_total: { - description: """ - The total number of times the Windows service has been stopped. - """ - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - _windows_service_uninstall_total: { - description: """ - The total number of times the Windows service has been uninstalled. - """ - type: "counter" - default_namespace: "vector" - tags: _internal_metrics_tags - } - - // Helpful tag groupings - _component_tags: _internal_metrics_tags & { - component_kind: _component_kind - component_name: _component_name - component_type: _component_type - } - - _internal_metrics_tags: { - instance: _instance - job: _job - } - - // All available tags - _collector: { - description: "Which collector this metric comes from." - required: true - } - _component_kind: { - description: "The component's kind (options are `source`, `sink`, or `transform`)." - required: true - options: ["sink", "source", "transform"] - } - _component_name: { - description: "The name of the component as specified in the Vector configuration." - required: true - examples: ["file_source", "splunk_sink"] - } - _component_type: { - description: "The type of component (source, transform, or sink)." - required: true - examples: ["file", "http", "honeycomb", "splunk_hec"] - } - _endpoint: { - description: "The absolute path of originating file." - required: true - examples: ["http://localhost:8080/server-status?auto"] - } - _error_type: { - description: "The type of the error" - required: true - options: [ - "field_missing", - "invalid_metric", - "mapping_failed", - "match_failed", - "parse_failed", - "render_error", - "type_conversion_failed", - "type_field_does_not_exist", - "type_ip_address_parse_error", - "value_invalid", - ] - } - _file: { - description: "The file that produced the error" - required: false - } - _host: { - description: "The hostname of the originating system." - required: true - examples: [_values.local_host] - } - _instance: { - description: "The Vector instance identified by host and port." - required: true - examples: [_values.instance] - } - _job: { - description: "The name of the job producing Vector metrics." - required: true - default: "vector" - } + events_processed_total: components.sources.internal_metrics.output.metrics.events_processed_total + processed_bytes_total: components.sources.internal_metrics.output.metrics.processed_bytes_total } }} } diff --git a/docs/reference/components/kafka.cue b/docs/reference/components/kafka.cue index 6e12e078f58de..64adb233a5fe7 100644 --- a/docs/reference/components/kafka.cue +++ b/docs/reference/components/kafka.cue @@ -90,15 +90,7 @@ components: _kafka: { } telemetry: metrics: { - consumer_offset_updates_failed_total: { - description: "The total number of failures to update a Kafka consumer offset." - type: "counter" - tags: telemetry.metrics._component_tags - } - events_failed_total: { - description: "The total number of failures to read a Kafka message." - type: "counter" - tags: telemetry.metrics._component_tags - } + consumer_offset_updates_failed_total: components.sources.internal_metrics.output.metrics.consumer_offset_updates_failed_total + events_failed_total: components.sources.internal_metrics.output.metrics.events_failed_total } } diff --git a/docs/reference/components/sinks/aws_cloudwatch_logs.cue b/docs/reference/components/sinks/aws_cloudwatch_logs.cue index d0aaaf0eb241c..5c27382889ad7 100644 --- a/docs/reference/components/sinks/aws_cloudwatch_logs.cue +++ b/docs/reference/components/sinks/aws_cloudwatch_logs.cue @@ -121,6 +121,6 @@ components: sinks: aws_cloudwatch_logs: { } telemetry: metrics: { - processing_errors_total: telemetry.metrics._processing_errors_total + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total } } diff --git a/docs/reference/components/sinks/console.cue b/docs/reference/components/sinks/console.cue index 2fef7d9b34349..579a9c15ebc4b 100644 --- a/docs/reference/components/sinks/console.cue +++ b/docs/reference/components/sinks/console.cue @@ -81,6 +81,6 @@ components: sinks: console: { } telemetry: metrics: { - processing_errors_total: telemetry.metrics._processing_errors_total + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total } } diff --git a/docs/reference/components/sinks/elasticsearch.cue b/docs/reference/components/sinks/elasticsearch.cue index ab0523315dc63..f4ac3bdce9a9c 100644 --- a/docs/reference/components/sinks/elasticsearch.cue +++ b/docs/reference/components/sinks/elasticsearch.cue @@ -241,6 +241,6 @@ components: sinks: elasticsearch: { } telemetry: metrics: { - missing_keys_total: telemetry.metrics._missing_keys_total + missing_keys_total: components.sources.internal_metrics.output.metrics.missing_keys_total } } diff --git a/docs/reference/components/sinks/http.cue b/docs/reference/components/sinks/http.cue index 3320f252af126..cbd63dd251aae 100644 --- a/docs/reference/components/sinks/http.cue +++ b/docs/reference/components/sinks/http.cue @@ -121,6 +121,6 @@ components: sinks: http: { } telemetry: metrics: { - http_bad_requests_total: telemetry.metrics._http_bad_requests_total + http_bad_requests_total: components.sources.internal_metrics.output.metrics.http_bad_requests_total } } diff --git a/docs/reference/components/sinks/sematext_metrics.cue b/docs/reference/components/sinks/sematext_metrics.cue index 0ffbc61939e2f..380bb16e0703a 100644 --- a/docs/reference/components/sinks/sematext_metrics.cue +++ b/docs/reference/components/sinks/sematext_metrics.cue @@ -80,6 +80,6 @@ components: sinks: sematext_metrics: { } telemetry: metrics: { - processing_errors_total: telemetry.metrics._processing_errors_total + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total } } diff --git a/docs/reference/components/sinks/splunk_hec.cue b/docs/reference/components/sinks/splunk_hec.cue index 15fe74350d5d9..4350a6924b262 100644 --- a/docs/reference/components/sinks/splunk_hec.cue +++ b/docs/reference/components/sinks/splunk_hec.cue @@ -167,7 +167,7 @@ components: sinks: splunk_hec: { } telemetry: metrics: { - http_request_errors_total: telemetry.metrics._http_request_errors_total - http_requests_total: telemetry.metrics._http_requests_total + http_request_errors_total: components.sources.internal_metrics.output.metrics.http_request_errors_total + http_requests_total: components.sources.internal_metrics.output.metrics.http_requests_total } } diff --git a/docs/reference/components/sinks/statsd.cue b/docs/reference/components/sinks/statsd.cue index 2d0b23001c3d4..1e5b6678e90e1 100644 --- a/docs/reference/components/sinks/statsd.cue +++ b/docs/reference/components/sinks/statsd.cue @@ -65,6 +65,6 @@ components: sinks: statsd: { } telemetry: metrics: { - processing_errors_total: telemetry.metrics._processing_errors_total + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total } } diff --git a/docs/reference/components/sinks/vector.cue b/docs/reference/components/sinks/vector.cue index f2d60012c764b..801c9bd426099 100644 --- a/docs/reference/components/sinks/vector.cue +++ b/docs/reference/components/sinks/vector.cue @@ -86,6 +86,6 @@ components: sinks: vector: { how_it_works: components.sources.vector.how_it_works telemetry: metrics: { - protobuf_decode_errors_total: telemetry.metrics._protobuf_decode_errors_total + protobuf_decode_errors_total: components.sources.internal_metrics.output.metrics.protobuf_decode_errors_total } } diff --git a/docs/reference/components/sources/aws_kinesis_firehose.cue b/docs/reference/components/sources/aws_kinesis_firehose.cue index de3e95257ccce..891cab7bbd278 100644 --- a/docs/reference/components/sources/aws_kinesis_firehose.cue +++ b/docs/reference/components/sources/aws_kinesis_firehose.cue @@ -189,7 +189,7 @@ components: sources: aws_kinesis_firehose: { } telemetry: metrics: { - request_read_errors_total: telemetry.metrics._request_read_errors_total - requests_received_total: telemetry.metrics._requests_received_total + request_read_errors_total: components.sources.internal_metrics.output.metrics.request_read_errors_total + requests_received_total: components.sources.internal_metrics.output.metrics.requests_received_total } } diff --git a/docs/reference/components/sources/aws_s3.cue b/docs/reference/components/sources/aws_s3.cue index 7e015c22b49b3..a5502fdd10808 100644 --- a/docs/reference/components/sources/aws_s3.cue +++ b/docs/reference/components/sources/aws_s3.cue @@ -241,69 +241,13 @@ components: sources: aws_s3: { } telemetry: metrics: { - sqs_message_delete_failed_total: { - description: "The total number of failures to delete SQS messages." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } - - sqs_message_delete_succeeded_total: { - description: "The total number of successful deletions of SQS messages." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } - - sqs_message_processing_failed_total: { - description: "The total number of failures to process SQS messages." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } - - sqs_message_processing_succeeded_total: { - description: "The total number of SQS messages successfully processed." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } - - sqs_message_receive_failed_total: { - description: "The total number of failures to receive SQS messages." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } - - sqs_message_receive_succeeded_total: { - description: "The total number of times successfully receiving SQS messages." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } - - sqs_message_received_messages_total: { - description: "The total number of received SQS messages." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } - - sqs_s3_event_record_ignored_total: { - description: "The total number of times an S3 record in an SQS message was ignored (for an event that was not `ObjectCreated`)." - type: "counter" - default_namespace: "vector" - - tags: telemetry.metrics._component_tags & { - ignore_type: { - description: "The reason for ignoring the S3 record" - required: true - options: [ - "invalid_event_kind", - ] - } - } - } + sqs_message_delete_failed_total: components.sources.internal_metrics.output.metrics.sqs_message_delete_failed_total + sqs_message_delete_succeeded_total: components.sources.internal_metrics.output.metrics.sqs_message_delete_succeeded_total + sqs_message_processing_failed_total: components.sources.internal_metrics.output.metrics.sqs_message_processing_failed_total + sqs_message_processing_succeeded_total: components.sources.internal_metrics.output.metrics.sqs_message_processing_succeeded_total + sqs_message_receive_failed_total: components.sources.internal_metrics.output.metrics.sqs_message_receive_failed_total + sqs_message_receive_succeeded_total: components.sources.internal_metrics.output.metrics.sqs_message_receive_succeeded_total + sqs_message_received_messages_total: components.sources.internal_metrics.output.metrics.sqs_message_received_messages_total + sqs_s3_event_record_ignored_total: components.sources.internal_metrics.output.metrics.sqs_s3_event_record_ignored_total } } diff --git a/docs/reference/components/sources/docker_logs.cue b/docs/reference/components/sources/docker_logs.cue index 2474863c76b84..260e146cac84c 100644 --- a/docs/reference/components/sources/docker_logs.cue +++ b/docs/reference/components/sources/docker_logs.cue @@ -267,44 +267,11 @@ components: sources: docker_logs: { } telemetry: metrics: { - communication_errors_total: { - description: "The total number of errors stemming from communication with the Docker daemon." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } - container_events_processed_total: { - description: "The total number of container events processed." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } - container_metadata_fetch_errors_total: { - description: "The total number of errors encountered when fetching container metadata." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } - containers_unwatched_total: { - description: "The total number of times Vector stopped watching for container logs." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } - containers_watched_total: { - description: "The total number of times Vector started watching for container logs." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } - logging_driver_errors_total: { - description: """ - The total number of logging driver errors encountered caused by not using either - the `jsonfile` or `journald` driver. - """ - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } + communication_errors_total: components.sources.internal_metrics.output.metrics.communication_errors_total + container_events_processed_total: components.sources.internal_metrics.output.metrics.container_events_processed_total + container_metadata_fetch_errors_total: components.sources.internal_metrics.output.metrics.container_metadata_fetch_errors_total + containers_unwatched_total: components.sources.internal_metrics.output.metrics.containers_unwatched_total + containers_watched_total: components.sources.internal_metrics.output.metrics.containers_watched_total + logging_driver_errors_total: components.sources.internal_metrics.output.metrics.logging_driver_errors_total } } diff --git a/docs/reference/components/sources/file.cue b/docs/reference/components/sources/file.cue index 4908b527a3465..6678f41c36a7e 100644 --- a/docs/reference/components/sources/file.cue +++ b/docs/reference/components/sources/file.cue @@ -517,15 +517,15 @@ components: sources: file: { } telemetry: metrics: { - checkpoint_write_errors_total: telemetry.metrics._checkpoint_write_errors_total - checkpoints_total: telemetry.metrics._checkpoints_total - checksum_errors: telemetry.metrics._checksum_errors - file_delete_errors: telemetry.metrics._file_delete_errors - file_watch_errors: telemetry.metrics._file_watch_errors - files_added: telemetry.metrics._files_added - files_deleted: telemetry.metrics._files_deleted - files_resumed: telemetry.metrics._files_resumed - files_unwatched: telemetry.metrics._files_unwatched - fingerprint_read_errors: telemetry.metrics._fingerprint_read_errors + checkpoint_write_errors_total: components.sources.internal_metrics.output.metrics.checkpoint_write_errors_total + checkpoints_total: components.sources.internal_metrics.output.metrics.checkpoints_total + checksum_errors: components.sources.internal_metrics.output.metrics.checksum_errors + file_delete_errors: components.sources.internal_metrics.output.metrics.file_delete_errors + file_watch_errors: components.sources.internal_metrics.output.metrics.file_watch_errors + files_added: components.sources.internal_metrics.output.metrics.files_added + files_deleted: components.sources.internal_metrics.output.metrics.files_deleted + files_resumed: components.sources.internal_metrics.output.metrics.files_resumed + files_unwatched: components.sources.internal_metrics.output.metrics.files_unwatched + fingerprint_read_errors: components.sources.internal_metrics.output.metrics.fingerprint_read_errors } } diff --git a/docs/reference/components/sources/http.cue b/docs/reference/components/sources/http.cue index cfbef3a174bc3..97f5773777040 100644 --- a/docs/reference/components/sources/http.cue +++ b/docs/reference/components/sources/http.cue @@ -208,6 +208,6 @@ components: sources: http: { ] telemetry: metrics: { - http_bad_requests_total: telemetry.metrics._http_bad_requests_total + http_bad_requests_total: components.sources.internal_metrics.output.metrics.http_bad_requests_total } } diff --git a/docs/reference/components/sources/internal_metrics.cue b/docs/reference/components/sources/internal_metrics.cue index fb879a45411f6..ac7367ff8cf0c 100644 --- a/docs/reference/components/sources/internal_metrics.cue +++ b/docs/reference/components/sources/internal_metrics.cue @@ -55,6 +55,7 @@ components: sources: internal_metrics: { } } + // Instance-level "process" metrics api_started_total: { description: "The number of times the Vector GraphQL API has been started." type: "counter" @@ -109,5 +110,571 @@ components: sources: internal_metrics: { default_namespace: "vector" tags: _internal_metrics_tags } + + // Metrics emitted by one or more components + // Reusable metric definitions + auto_concurrency_averaged_rtt: { + description: "The average round-trip time (RTT) from the HTTP sink across the current window." + type: "histogram" + default_namespace: "vector" + tags: _internal_metrics_tags + } + auto_concurrency_in_flight: { + description: "The number of outbound requests from the HTTP sink currently awaiting a response." + type: "histogram" + default_namespace: "vector" + tags: _internal_metrics_tags + } + auto_concurrency_limit: { + description: "The concurrency limit that the auto-concurrency feature has decided on for this current window." + type: "histogram" + default_namespace: "vector" + tags: _internal_metrics_tags + } + auto_concurrency_observed_rtt: { + description: "The observed round-trip time (RTT) for requests from this HTTP sink." + type: "histogram" + default_namespace: "vector" + tags: _internal_metrics_tags + } + checkpoint_write_errors_total: { + description: "The total number of errors writing checkpoints." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + checkpoints_total: { + description: "The total number of files checkpointed." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + checksum_errors: { + description: "The total number of errors identifying files via checksum." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & { + file: _file + } + } + collect_completed_total: { + description: "The total number of MongoDB metrics collections completed." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + collect_duration_nanoseconds: { + description: "The duration spent collecting MongoDB metrics." + type: "histogram" + default_namespace: "vector" + tags: _internal_metrics_tags + } + communication_errors_total: { + description: "The total number of errors stemming from communication with the Docker daemon." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + connection_read_errors_total: { + description: "The total number of errors reading datagram." + type: "counter" + default_namespace: "vector" + tags: _component_tags & { + mode: { + description: "" + required: true + options: { + udp: "User Datagram Protocol" + } + } + } + } + consumer_offset_updates_failed_total: { + description: "The total number of failures to update a Kafka consumer offset." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + container_events_processed_total: { + description: "The total number of container events processed." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + container_metadata_fetch_errors_total: { + description: "The total number of errors encountered when fetching container metadata." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + containers_unwatched_total: { + description: "The total number of times Vector stopped watching for container logs." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + containers_watched_total: { + description: "The total number of times Vector started watching for container logs." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + docker_format_parse_failures_total: { + description: "The total number of failures to parse a message as a JSON object." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + event_annotation_failures_total: { + description: "The total number of failures to annotate Vector events with Kubernetes Pod metadata." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + events_discarded_total: { + description: "The total number of events discarded by this component." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + events_failed_total: { + description: "The total number of failures to read a Kafka message." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + events_processed_total: { + description: "The total number of events processed by this component." + type: "counter" + default_namespace: "vector" + tags: _component_tags & { + file: _file + } + } + file_delete_errors: { + description: "The total number of failures to delete a file." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & { + file: _file + } + } + file_watch_errors: { + description: "The total number of errors encountered when watching files." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & { + file: _file + } + } + files_added: { + description: "The total number of files Vector has found to watch." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & { + file: _file + } + } + files_deleted: { + description: "The total number of files deleted." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & { + file: _file + } + } + files_resumed: { + description: "The total number of times Vector has resumed watching a file." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & { + file: _file + } + } + files_unwatched: { + description: "The total number of times Vector has stopped watching a file." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & { + file: _file + } + } + fingerprint_read_errors: { + description: "The total number of times Vector failed to read a file for fingerprinting." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags & { + file: _file + } + } + http_bad_requests_total: { + description: "The total number of HTTP `400 Bad Request` errors encountered." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + http_error_response_total: { + description: "The total number of HTTP error responses for this component." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + http_request_errors_total: { + description: "The total number of HTTP request errors for this component." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + http_requests_total: { + description: "The total number of HTTP requests issued by this component." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + invalid_record_total: { + description: "The total number of invalid records that have been discarded." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + invalid_record_bytes_total: { + description: "The total number of bytes from invalid records that have been discarded." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + logging_driver_errors_total: { + description: """ + The total number of logging driver errors encountered caused by not using either + the `jsonfile` or `journald` driver. + """ + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + memory_used: { + description: "The total memory currently being used by Vector (in bytes)." + type: "gauge" + default_namespace: "vector" + tags: _internal_metrics_tags + } + metadata_refresh_failed_total: { + description: "The total number of failed efforts to refresh AWS EC2 metadata." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + metadata_refresh_successful_total: { + description: "The total number of AWS EC2 metadata refreshes." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + missing_keys_total: { + description: "The total number of events dropped due to keys missing from the event." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + open_connections: { + description: "The number of current open connections to Vector." + type: "gauge" + default_namespace: "vector" + tags: _internal_metrics_tags + } + parse_errors_total: { + description: "The total number of errors parsing Prometheus metrics." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + processed_bytes_total: { + description: "The total number of bytes processed by the component." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + processing_errors_total: { + description: "The total number of processing errors encountered by this component." + type: "counter" + default_namespace: "vector" + tags: _component_tags & { + error_type: _error_type + } + } + protobuf_decode_errors_total: { + description: "The total number of [Protocol Buffers](\(urls.protobuf)) errors thrown during communication between Vector instances." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + request_duration_nanoseconds: { + description: "The request duration for this component (in nanoseconds)." + type: "histogram" + default_namespace: "vector" + tags: _component_tags + } + request_error_total: { + description: "The total number of MongoDB request errors." + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + request_read_errors_total: { + description: "The total number of request read errors for this component." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + requests_completed_total: { + description: "The total number of requests completed by this component." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + requests_received_total: { + description: "The total number of requests received by this component." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + sqs_message_delete_failed_total: { + description: "The total number of failures to delete SQS messages." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + sqs_message_delete_succeeded_total: { + description: "The total number of successful deletions of SQS messages." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + sqs_message_processing_failed_total: { + description: "The total number of failures to process SQS messages." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + sqs_message_processing_succeeded_total: { + description: "The total number of SQS messages successfully processed." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + + sqs_message_receive_failed_total: { + description: "The total number of failures to receive SQS messages." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + sqs_message_receive_succeeded_total: { + description: "The total number of times successfully receiving SQS messages." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + sqs_message_received_messages_total: { + description: "The total number of received SQS messages." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + sqs_s3_event_record_ignored_total: { + description: "The total number of times an S3 record in an SQS message was ignored (for an event that was not `ObjectCreated`)." + type: "counter" + default_namespace: "vector" + + tags: _component_tags & { + ignore_type: { + description: "The reason for ignoring the S3 record" + required: true + options: [ + "invalid_event_kind", + ] + } + } + } + stale_events_flushed_total: { + description: "The number of stale events that Vector has flushed." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + stdin_reads_failed_total: { + description: "The total number of errors reading from stdin." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + tag_value_limit_exceeded_total: { + description: """ + The total number of events discarded because the tag has been rejected after + hitting the configured `value_limit`. + """ + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + timestamp_parse_errors_total: { + description: "The total number of errors encountered parsing [RFC3339](\(urls.rfc_3339)) timestamps." + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + uptime_seconds: { + description: "The total number of seconds the Vector instance has been up." + type: "gauge" + default_namespace: "vector" + tags: _component_tags + } + utf8_convert_errors_total: { + description: "The total number of errors converting bytes to a UTF-8 string in UDP mode." + type: "counter" + default_namespace: "vector" + tags: _component_tags & { + mode: { + description: "The connection mode used by the component." + required: true + options: { + udp: "User Datagram Protocol" + } + } + } + } + value_limit_reached_total: { + description: """ + The total number of times new values for a key have been rejected because the + value limit has been reached. + """ + type: "counter" + default_namespace: "vector" + tags: _component_tags + } + + // Windows metrics + windows_service_does_not_exist_total: { + description: """ + The total number of errors raised due to the Windows service not + existing. + """ + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + windows_service_install_total: { + description: """ + The total number of times the Windows service has been installed. + """ + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + windows_service_restart_total: { + description: """ + The total number of times the Windows service has been restarted. + """ + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + windows_service_start_total: { + description: """ + The total number of times the Windows service has been started. + """ + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + windows_service_stop_total: { + description: """ + The total number of times the Windows service has been stopped. + """ + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + windows_service_uninstall_total: { + description: """ + The total number of times the Windows service has been uninstalled. + """ + type: "counter" + default_namespace: "vector" + tags: _internal_metrics_tags + } + + // Helpful tag groupings + _component_tags: _internal_metrics_tags & { + component_kind: _component_kind + component_name: _component_name + component_type: _component_type + } + + _internal_metrics_tags: { + instance: _instance + job: _job + } + + // All available tags + _collector: { + description: "Which collector this metric comes from." + required: true + } + _component_kind: { + description: "The component's kind (options are `source`, `sink`, or `transform`)." + required: true + options: ["sink", "source", "transform"] + } + _component_name: { + description: "The name of the component as specified in the Vector configuration." + required: true + examples: ["file_source", "splunk_sink"] + } + _component_type: { + description: "The type of component (source, transform, or sink)." + required: true + examples: ["file", "http", "honeycomb", "splunk_hec"] + } + _endpoint: { + description: "The absolute path of originating file." + required: true + examples: ["http://localhost:8080/server-status?auto"] + } + _error_type: { + description: "The type of the error" + required: true + options: [ + "field_missing", + "invalid_metric", + "mapping_failed", + "match_failed", + "parse_failed", + "render_error", + "type_conversion_failed", + "type_field_does_not_exist", + "type_ip_address_parse_error", + "value_invalid", + ] + } + _file: { + description: "The file that produced the error" + required: false + } + _host: { + description: "The hostname of the originating system." + required: true + examples: [_values.local_host] + } + _instance: { + description: "The Vector instance identified by host and port." + required: true + examples: [_values.instance] + } + _job: { + description: "The name of the job producing Vector metrics." + required: true + default: "vector" + } } } diff --git a/docs/reference/components/sources/journald.cue b/docs/reference/components/sources/journald.cue index 66b870d25e50b..01f1da261f4e4 100644 --- a/docs/reference/components/sources/journald.cue +++ b/docs/reference/components/sources/journald.cue @@ -191,17 +191,7 @@ components: sources: journald: { } telemetry: metrics: { - invalid_record_total: { - description: "The total number of invalid journald records discarded." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } - invalid_record_bytes_total: { - description: "The total number of bytes from discarded journald records." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } + invalid_record_total: components.sources.internal_metrics.output.metrics.invalid_record_total + invalid_record_bytes_total: components.sources.internal_metrics.output.metrics.invalid_record_bytes_total } } diff --git a/docs/reference/components/sources/kubernetes_logs.cue b/docs/reference/components/sources/kubernetes_logs.cue index 061ba36809d4d..ca6e848ca2306 100644 --- a/docs/reference/components/sources/kubernetes_logs.cue +++ b/docs/reference/components/sources/kubernetes_logs.cue @@ -301,18 +301,7 @@ components: sources: kubernetes_logs: { } telemetry: metrics: { - docker_format_parse_failures_total: { - description: "The total number of failures to parse a message as a JSON object." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } - - event_annotation_failures_total: { - description: "The total number of failures to annotate Vector events with Kubernetes Pod metadata." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } + docker_format_parse_failures_total: components.sources.internal_metrics.output.metrics.docker_format_parse_failures_total + event_annotation_failures_total: components.sources.internal_metrics.output.metrics.event_annotation_failures_total } } diff --git a/docs/reference/components/sources/logplex.cue b/docs/reference/components/sources/logplex.cue index 0a447b6b32431..03dd32bc6cf72 100644 --- a/docs/reference/components/sources/logplex.cue +++ b/docs/reference/components/sources/logplex.cue @@ -99,7 +99,7 @@ components: sources: logplex: { } telemetry: metrics: { - request_read_errors_total: telemetry.metrics._request_read_errors_total - requests_received_total: telemetry.metrics._requests_received_total + request_read_errors_total: components.sources.internal_metrics.output.metrics.request_read_errors_total + requests_received_total: components.sources.internal_metrics.output.metrics.requests_received_total } } diff --git a/docs/reference/components/sources/mongodb_metrics.cue b/docs/reference/components/sources/mongodb_metrics.cue index f7c0e6b48e8d4..7d8b514231574 100644 --- a/docs/reference/components/sources/mongodb_metrics.cue +++ b/docs/reference/components/sources/mongodb_metrics.cue @@ -95,24 +95,9 @@ components: sources: mongodb_metrics: { } telemetry: metrics: { - collect_duration_nanoseconds: { - description: "The duration spent collecting MongoDB metrics." - type: "histogram" - default_namespace: "vector" - tags: telemetry.metrics._internal_metrics_tags - } - collect_completed_total: { - description: "The total number of MongoDB metrics collections completed." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._internal_metrics_tags - } - request_error_total: { - description: "The total number of MongoDB request errors." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._internal_metrics_tags - } + collect_duration_nanoseconds: components.sources.internal_metrics.output.metrics.collect_duration_nanoseconds + collect_completed_total: components.sources.internal_metrics.output.metrics.collect_completed_total + request_error_total: components.sources.internal_metrics.output.metrics.request_error_total } output: metrics: { diff --git a/docs/reference/components/sources/socket.cue b/docs/reference/components/sources/socket.cue index 673b2d5a79ae0..2e7d220488ce3 100644 --- a/docs/reference/components/sources/socket.cue +++ b/docs/reference/components/sources/socket.cue @@ -151,6 +151,6 @@ components: sources: socket: { ] telemetry: metrics: { - connection_errors_total: telemetry.metrics._connection_errors_total + connection_errors_total: components.sources.internal_metrics.output.metrics.connection_errors_total } } diff --git a/docs/reference/components/sources/splunk_hec.cue b/docs/reference/components/sources/splunk_hec.cue index b48f1122ef8fc..c4d60938e42a3 100644 --- a/docs/reference/components/sources/splunk_hec.cue +++ b/docs/reference/components/sources/splunk_hec.cue @@ -94,7 +94,7 @@ components: sources: splunk_hec: { } telemetry: metrics: { - http_request_errors_total: telemetry.metrics._http_request_errors_total - requests_received_total: telemetry.metrics._requests_received_total + http_request_errors_total: components.sources.internal_metrics.output.metrics.http_request_errors_total + requests_received_total: components.sources.internal_metrics.output.metrics.requests_received_total } } diff --git a/docs/reference/components/sources/statsd.cue b/docs/reference/components/sources/statsd.cue index 111747cddf835..8ae463bc6aaab 100644 --- a/docs/reference/components/sources/statsd.cue +++ b/docs/reference/components/sources/statsd.cue @@ -122,18 +122,8 @@ components: sources: statsd: { } telemetry: metrics: { - connection_errors_total: telemetry.metrics._connection_errors_total - invalid_record_total: { - description: "The total number of discarded invalid StatsD records." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } - invalid_record_bytes_total: { - description: "The total number of bytes from StatsD journald records." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } + connection_errors_total: components.sources.internal_metrics.output.metrics.connection_errors_total + invalid_record_total: components.sources.internal_metrics.output.metrics.invalid_record_total + invalid_record_bytes_total: components.sources.internal_metrics.output.metrics.invalid_record_bytes_total } } diff --git a/docs/reference/components/sources/stdin.cue b/docs/reference/components/sources/stdin.cue index 33a7dc1229313..a9568b2beefff 100644 --- a/docs/reference/components/sources/stdin.cue +++ b/docs/reference/components/sources/stdin.cue @@ -102,11 +102,6 @@ components: sources: stdin: { } telemetry: metrics: { - stdin_reads_failed_total: { - description: "The total number of errors reading from stdin." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } + stdin_reads_failed_total: components.sources.internal_metrics.output.metrics.stdin_reads_failed_total } } diff --git a/docs/reference/components/sources/syslog.cue b/docs/reference/components/sources/syslog.cue index d801775046190..e313900d79bab 100644 --- a/docs/reference/components/sources/syslog.cue +++ b/docs/reference/components/sources/syslog.cue @@ -184,33 +184,7 @@ components: sources: syslog: { } telemetry: metrics: { - connection_read_errors_total: { - description: "The total number of errors reading datagram." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags & { - mode: { - description: "" - required: true - options: { - udp: "User Datagram Protocol" - } - } - } - } - utf8_convert_errors_total: { - description: "The total number of errors converting bytes to a UTF-8 string in UDP mode." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags & { - mode: { - description: "The connection mode used by the component." - required: true - options: { - udp: "User Datagram Protocol" - } - } - } - } + connection_read_errors_total: components.sources.internal_metrics.output.metrics.connection_read_errors_total + utf8_convert_errors_total: components.sources.internal_metrics.output.metrics.utf8_convert_errors_total } } diff --git a/docs/reference/components/sources/vector.cue b/docs/reference/components/sources/vector.cue index 381ecc0f13d4e..3ff70bdcf4932 100644 --- a/docs/reference/components/sources/vector.cue +++ b/docs/reference/components/sources/vector.cue @@ -112,6 +112,6 @@ components: sources: vector: { } telemetry: metrics: { - protobuf_decode_errors_total: telemetry.metrics._protobuf_decode_errors_total + protobuf_decode_errors_total: components.sources.internal_metrics.output.metrics.protobuf_decode_errors_total } } diff --git a/docs/reference/components/transforms/ansi_stripper.cue b/docs/reference/components/transforms/ansi_stripper.cue index 25eb48445eefc..690f2250d17f7 100644 --- a/docs/reference/components/transforms/ansi_stripper.cue +++ b/docs/reference/components/transforms/ansi_stripper.cue @@ -47,6 +47,6 @@ components: transforms: ansi_stripper: { } telemetry: metrics: { - processing_errors_total: telemetry.metrics._processing_errors_total + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total } } diff --git a/docs/reference/components/transforms/aws_cloudwatch_logs_subscription_parser.cue b/docs/reference/components/transforms/aws_cloudwatch_logs_subscription_parser.cue index f1505df60dc86..6489d0baaab68 100644 --- a/docs/reference/components/transforms/aws_cloudwatch_logs_subscription_parser.cue +++ b/docs/reference/components/transforms/aws_cloudwatch_logs_subscription_parser.cue @@ -144,6 +144,6 @@ components: transforms: aws_cloudwatch_logs_subscription_parser: { } telemetry: metrics: { - processing_errors_total: telemetry.metrics._processing_errors_total + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total } } diff --git a/docs/reference/components/transforms/aws_ec2_metadata.cue b/docs/reference/components/transforms/aws_ec2_metadata.cue index 65d0f8bdfed31..7b4f9e83480c9 100644 --- a/docs/reference/components/transforms/aws_ec2_metadata.cue +++ b/docs/reference/components/transforms/aws_ec2_metadata.cue @@ -172,17 +172,7 @@ components: transforms: aws_ec2_metadata: { } telemetry: metrics: { - metadata_refresh_failed_total: { - description: "The total number of failed efforts to refresh AWS EC2 metadata." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } - metadata_refresh_successful_total: { - description: "The total number of AWS EC2 metadata refreshes." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } + metadata_refresh_failed_total: components.sources.internal_metrics.output.metrics.metadata_refresh_failed_total + metadata_refresh_successful_total: components.sources.internal_metrics.output.metrics.metadata_refresh_successful_total } } diff --git a/docs/reference/components/transforms/dedupe.cue b/docs/reference/components/transforms/dedupe.cue index 244e37e322d35..3b53bf7938ccc 100644 --- a/docs/reference/components/transforms/dedupe.cue +++ b/docs/reference/components/transforms/dedupe.cue @@ -148,6 +148,6 @@ components: transforms: dedupe: { } telemetry: metrics: { - events_discarded_total: telemetry.metrics._events_discarded_total + events_discarded_total: components.sources.internal_metrics.output.metrics.events_discarded_total } } diff --git a/docs/reference/components/transforms/grok_parser.cue b/docs/reference/components/transforms/grok_parser.cue index eb6664c82f5e3..2f99ea2e096fd 100644 --- a/docs/reference/components/transforms/grok_parser.cue +++ b/docs/reference/components/transforms/grok_parser.cue @@ -97,6 +97,6 @@ components: transforms: grok_parser: { } telemetry: metrics: { - processing_errors_total: telemetry.metrics._processing_errors_total + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total } } diff --git a/docs/reference/components/transforms/logfmt_parser.cue b/docs/reference/components/transforms/logfmt_parser.cue index 8e243f9baafbc..06e5eb0abdb0b 100644 --- a/docs/reference/components/transforms/logfmt_parser.cue +++ b/docs/reference/components/transforms/logfmt_parser.cue @@ -180,6 +180,6 @@ components: transforms: logfmt_parser: { } telemetry: metrics: { - processing_errors_total: telemetry.metrics._processing_errors_total + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total } } diff --git a/docs/reference/components/transforms/lua.cue b/docs/reference/components/transforms/lua.cue index 46e83fd8e776e..07ad1fca20e46 100644 --- a/docs/reference/components/transforms/lua.cue +++ b/docs/reference/components/transforms/lua.cue @@ -472,7 +472,7 @@ components: transforms: lua: { } telemetry: metrics: { - memory_used: telemetry.metrics._memory_used - processing_errors_total: telemetry.metrics._processing_errors_total + memory_used: components.sources.internal_metrics.output.metrics.memory_used + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total } } diff --git a/docs/reference/components/transforms/metric_to_log.cue b/docs/reference/components/transforms/metric_to_log.cue index 8fe00007d9213..d494446ed82d6 100644 --- a/docs/reference/components/transforms/metric_to_log.cue +++ b/docs/reference/components/transforms/metric_to_log.cue @@ -95,6 +95,6 @@ components: transforms: metric_to_log: { how_it_works: {} telemetry: metrics: { - processing_errors_total: telemetry.metrics._processing_errors_total + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total } } diff --git a/docs/reference/components/transforms/reduce.cue b/docs/reference/components/transforms/reduce.cue index 7faf8034088de..5bae46c028409 100644 --- a/docs/reference/components/transforms/reduce.cue +++ b/docs/reference/components/transforms/reduce.cue @@ -146,11 +146,6 @@ components: transforms: reduce: { ] telemetry: metrics: { - stale_events_flushed_total: { - description: "The number of stale events that Vector has flushed." - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } + stale_events_flushed_total: components.sources.internal_metrics.output.metrics.stale_events_flushed_total } } diff --git a/docs/reference/components/transforms/regex_parser.cue b/docs/reference/components/transforms/regex_parser.cue index 15c72913c6eba..a44ee53f137ba 100644 --- a/docs/reference/components/transforms/regex_parser.cue +++ b/docs/reference/components/transforms/regex_parser.cue @@ -180,6 +180,6 @@ components: transforms: regex_parser: { } telemetry: metrics: { - processing_errors_total: telemetry.metrics._processing_errors_total + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total } } diff --git a/docs/reference/components/transforms/remap.cue b/docs/reference/components/transforms/remap.cue index ce0837f966f7b..91197d70045b3 100644 --- a/docs/reference/components/transforms/remap.cue +++ b/docs/reference/components/transforms/remap.cue @@ -132,6 +132,6 @@ components: transforms: remap: { } telemetry: metrics: { - processing_errors_total: telemetry.metrics._processing_errors_total + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total } } diff --git a/docs/reference/components/transforms/sampler.cue b/docs/reference/components/transforms/sampler.cue index 523b7323452be..0614a2aeef2ba 100644 --- a/docs/reference/components/transforms/sampler.cue +++ b/docs/reference/components/transforms/sampler.cue @@ -66,6 +66,6 @@ components: transforms: sampler: { } telemetry: metrics: { - events_discarded_total: telemetry.metrics._events_discarded_total + events_discarded_total: components.sources.internal_metrics.output.metrics.events_discarded_total } } diff --git a/docs/reference/components/transforms/split.cue b/docs/reference/components/transforms/split.cue index dab2adb3d6040..366a5003393a7 100644 --- a/docs/reference/components/transforms/split.cue +++ b/docs/reference/components/transforms/split.cue @@ -97,6 +97,6 @@ components: transforms: split: { ] telemetry: metrics: { - processing_errors_total: telemetry.metrics._processing_errors_total + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total } } diff --git a/docs/reference/components/transforms/swimlanes.cue b/docs/reference/components/transforms/swimlanes.cue index faad9ee701007..3079c9c22c895 100644 --- a/docs/reference/components/transforms/swimlanes.cue +++ b/docs/reference/components/transforms/swimlanes.cue @@ -72,6 +72,6 @@ components: transforms: swimlanes: { ] telemetry: metrics: { - events_discarded_total: telemetry.metrics._events_discarded_total + events_discarded_total: components.sources.internal_metrics.output.metrics.events_discarded_total } } diff --git a/docs/reference/components/transforms/tag_cardinality_limit.cue b/docs/reference/components/transforms/tag_cardinality_limit.cue index 1a640bc915bfc..766f525b57beb 100644 --- a/docs/reference/components/transforms/tag_cardinality_limit.cue +++ b/docs/reference/components/transforms/tag_cardinality_limit.cue @@ -220,23 +220,7 @@ components: transforms: tag_cardinality_limit: { } telemetry: metrics: { - tag_value_limit_exceeded_total: { - description: """ - The total number of events discarded because the tag has been rejected after - hitting the configured `value_limit`. - """ - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } - value_limit_reached_total: { - description: """ - The total number of times new values for a key have been rejected because the - value limit has been reached. - """ - type: "counter" - default_namespace: "vector" - tags: telemetry.metrics._component_tags - } + tag_value_limit_exceeded_total: components.sources.internal_metrics.output.metrics.tag_value_limit_exceeded_total + value_limit_reached_total: components.sources.internal_metrics.output.metrics.value_limit_reached_total } } diff --git a/docs/reference/components/transforms/tokenizer.cue b/docs/reference/components/transforms/tokenizer.cue index b5fd23af5fee9..3ba7aac2ecb81 100644 --- a/docs/reference/components/transforms/tokenizer.cue +++ b/docs/reference/components/transforms/tokenizer.cue @@ -116,6 +116,6 @@ components: transforms: tokenizer: { } telemetry: metrics: { - processing_errors_total: telemetry.metrics._processing_errors_total + processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total } }