enhancement(external docs): Add Cue docs for component-specific metri…

…cs (vectordotdev#4820) * Begin transferring metrics Signed-off-by: Luc Perkins <luc@timber.io> * Metrics docs for sinks Signed-off-by: Luc Perkins <luc@timber.io> * Add source metrics and refactor metrics.cue Signed-off-by: Luc Perkins <luc@timber.io> * Add metrics for transforms Signed-off-by: Luc Perkins <luc@timber.io> * Fill in remaining TODO descriptions Signed-off-by: Luc Perkins <luc@timber.io> * Add internal_metrics source Signed-off-by: Luc Perkins <luc@timber.io> * Fix service/aggregator discrepancy Signed-off-by: Luc Perkins <luc@timber.io> * Move Apache, MongoDB, and host metrics out of metrics.cue Signed-off-by: Luc Perkins <luc@timber.io> * Move Docker metrics out of metrics.cue Signed-off-by: Luc Perkins <luc@timber.io> * Move k8s metrics out of metrics.cue Signed-off-by: Luc Perkins <luc@timber.io> * Migrate internal metrics out of metrics.cue Signed-off-by: Luc Perkins <luc@timber.io> * Remove metrics.cue file Signed-off-by: Luc Perkins <luc@timber.io> * Apply explicit default namespace to all metrics Signed-off-by: Luc Perkins <luc@timber.io> * Refactor Splunk common metrics into separate file Signed-off-by: Luc Perkins <luc@timber.io> * Address feedback on metric descriptions Signed-off-by: Luc Perkins <luc@timber.io> * Add default namespace to AWS ECS metrics Signed-off-by: Luc Perkins <luc@timber.io> * Add _total suffix to Windows metrics Signed-off-by: Luc Perkins <luc@timber.io> * Add new options for the error_type tag Signed-off-by: Luc Perkins <luc@timber.io> * Add AWS S3+SQS metrics to docs Signed-off-by: Luc Perkins <luc@timber.io> * Refactor all vector namespace metrics into internal_metrics.cue Signed-off-by: Luc Perkins <luc@timber.io> Signed-off-by: Brian Menges <brian.menges@anaplan.com>
jacobbraaten · Dec 9, 2020 · 4aed50b · 4aed50b
1 parent b1a19be
commit 4aed50b
Show file tree

Hide file tree

Showing 49 changed files with 1,452 additions and 694 deletions.
diff --git a/docs/reference.cue b/docs/reference.cue
@@ -44,7 +44,7 @@ _values: {
 // * `sidecar` - Vector is installed alongside each process it is
 //   monitoring. Therefore, there might be multiple Vector processes
 //   on the host.
-// * `service` - Vector receives data from one or more upstream
+// * `aggregator` - Vector receives data from one or more upstream
 //   sources, typically over a network protocol.
 #DeploymentRole: "aggregator" | "daemon" | "sidecar"
 

diff --git a/docs/reference/components.cue b/docs/reference/components.cue
@@ -354,11 +354,12 @@ components: {
 	}
 
 	#MetricOutput: [Name=string]: close({
-		description:    string
-		relevant_when?: string
-		tags:           #MetricTags
-		name:           Name
-		type:           #MetricType
+		description:       string
+		relevant_when?:    string
+		tags:              #MetricTags
+		name:              Name
+		type:              #MetricType
+		default_namespace: string
 	})
 
 	#Output: {
@@ -833,7 +834,8 @@ components: {
 							required: false
 						}
 					}
-					type: "counter"
+					type:              "counter"
+					default_namespace: "vector"
 				}
 
 				_passthrough_distribution: {
@@ -845,7 +847,8 @@ components: {
 							required: false
 						}
 					}
-					type: "distribution"
+					type:              "distribution"
+					default_namespace: "vector"
 				}
 
 				_passthrough_gauge: {
@@ -857,7 +860,8 @@ components: {
 							required: false
 						}
 					}
-					type: "gauge"
+					type:              "gauge"
+					default_namespace: "vector"
 				}
 
 				_passthrough_histogram: {
@@ -869,7 +873,8 @@ components: {
 							required: false
 						}
 					}
-					type: "gauge"
+					type:              "gauge"
+					default_namespace: "vector"
 				}
 
 				_passthrough_set: {
@@ -881,7 +886,8 @@ components: {
 							required: false
 						}
 					}
-					type: "gauge"
+					type:              "gauge"
+					default_namespace: "vector"
 				}
 
 				_passthrough_summary: {
@@ -893,14 +899,16 @@ components: {
 							required: false
 						}
 					}
-					type: "gauge"
+					type:              "gauge"
+					default_namespace: "vector"
 				}
 			}
 		}
 
 		telemetry: metrics: {
 			// Default metrics for each component
-			vector_events_processed_total: _vector_events_processed_total
+			events_processed_total: components.sources.internal_metrics.output.metrics.events_processed_total
+			processed_bytes_total:  components.sources.internal_metrics.output.metrics.processed_bytes_total
 		}
 	}}
 }
diff --git a/docs/reference/components/kafka.cue b/docs/reference/components/kafka.cue
@@ -88,4 +88,9 @@ components: _kafka: {
 				"""
 		}
 	}
+
+	telemetry: metrics: {
+		consumer_offset_updates_failed_total: components.sources.internal_metrics.output.metrics.consumer_offset_updates_failed_total
+		events_failed_total:                  components.sources.internal_metrics.output.metrics.events_failed_total
+	}
 }
diff --git a/docs/reference/components/sinks/aws_cloudwatch_logs.cue b/docs/reference/components/sinks/aws_cloudwatch_logs.cue
@@ -119,4 +119,8 @@ components: sinks: aws_cloudwatch_logs: {
 		logs:    true
 		metrics: null
 	}
+
+	telemetry: metrics: {
+		processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total
+	}
 }
diff --git a/docs/reference/components/sinks/console.cue b/docs/reference/components/sinks/console.cue
@@ -81,7 +81,6 @@ components: sinks: console: {
 	}
 
 	telemetry: metrics: {
-		vector_processed_bytes_total:   _vector_processed_bytes_total
-		vector_processing_errors_total: _vector_processing_errors_total
+		processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total
 	}
 }
diff --git a/docs/reference/components/sinks/elasticsearch.cue b/docs/reference/components/sinks/elasticsearch.cue
@@ -239,4 +239,8 @@ components: sinks: elasticsearch: {
 					"""
 		}
 	}
+
+	telemetry: metrics: {
+		missing_keys_total: components.sources.internal_metrics.output.metrics.missing_keys_total
+	}
 }
diff --git a/docs/reference/components/sinks/http.cue b/docs/reference/components/sinks/http.cue
@@ -119,4 +119,8 @@ components: sinks: http: {
 		logs:    true
 		metrics: null
 	}
+
+	telemetry: metrics: {
+		http_bad_requests_total: components.sources.internal_metrics.output.metrics.http_bad_requests_total
+	}
 }
diff --git a/docs/reference/components/sinks/sematext_metrics.cue b/docs/reference/components/sinks/sematext_metrics.cue
@@ -78,4 +78,8 @@ components: sinks: sematext_metrics: {
 			summary:      false
 		}
 	}
+
+	telemetry: metrics: {
+		processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total
+	}
 }
diff --git a/docs/reference/components/sinks/splunk_hec.cue b/docs/reference/components/sinks/splunk_hec.cue
@@ -165,4 +165,9 @@ components: sinks: splunk_hec: {
 		logs:    true
 		metrics: null
 	}
+
+	telemetry: metrics: {
+		http_request_errors_total: components.sources.internal_metrics.output.metrics.http_request_errors_total
+		http_requests_total:       components.sources.internal_metrics.output.metrics.http_requests_total
+	}
 }
diff --git a/docs/reference/components/sinks/statsd.cue b/docs/reference/components/sinks/statsd.cue
@@ -63,4 +63,8 @@ components: sinks: statsd: {
 			}
 		}
 	}
+
+	telemetry: metrics: {
+		processing_errors_total: components.sources.internal_metrics.output.metrics.processing_errors_total
+	}
 }
diff --git a/docs/reference/components/sinks/vector.cue b/docs/reference/components/sinks/vector.cue
@@ -84,4 +84,8 @@ components: sinks: vector: {
 	}
 
 	how_it_works: components.sources.vector.how_it_works
+
+	telemetry: metrics: {
+		protobuf_decode_errors_total: components.sources.internal_metrics.output.metrics.protobuf_decode_errors_total
+	}
 }
diff --git a/docs/reference/components/sources/apache_metrics.cue b/docs/reference/components/sources/apache_metrics.cue
@@ -112,31 +112,32 @@ components: sources: apache_metrics: {
 	}
 
 	output: metrics: {
-		_endpoint: {
-			description: "The absolute path of originating file."
-			required:    true
-			examples: ["http://localhost:8080/server-status?auto"]
-		}
-		_host: {
-			description: "The hostname of the Apache HTTP server"
-			required:    true
-			examples: [_values.local_host]
+		// Default Apache metrics tags
+		_apache_metrics_tags: {
+			endpoint: {
+				description: "The absolute path of originating file."
+				required:    true
+				examples: ["http://localhost:8080/server-status?auto"]
+			}
+			host: {
+				description: "The hostname of the Apache HTTP server."
+				required:    true
+				examples: [_values.local_host]
+			}
 		}
+
 		access_total: {
-			description:   "The total number of time the Apache server has been accessed."
-			relevant_when: "`ExtendedStatus On`"
-			type:          "counter"
-			tags: {
-				endpoint: _endpoint
-				host:     _host
-			}
+			description:       "The total number of time the Apache server has been accessed."
+			relevant_when:     "`ExtendedStatus On`"
+			type:              "counter"
+			default_namespace: "apache"
+			tags:              _apache_metrics_tags
 		}
 		connections: {
-			description: "The total number of time the Apache server has been accessed."
-			type:        "gauge"
-			tags: {
-				endpoint: _endpoint
-				host:     _host
+			description:       "The total number of time the Apache server has been accessed."
+			type:              "gauge"
+			default_namespace: "apache"
+			tags:              _apache_metrics_tags & {
 				state: {
 					description: "The state of the connection"
 					required:    true
@@ -145,43 +146,37 @@ components: sources: apache_metrics: {
 			}
 		}
 		cpu_load: {
-			description:   "The current CPU of the Apache server."
-			relevant_when: "`ExtendedStatus On`"
-			type:          "gauge"
-			tags: {
-				endpoint: _endpoint
-				host:     _host
-			}
+			description:       "The current CPU of the Apache server."
+			relevant_when:     "`ExtendedStatus On`"
+			type:              "gauge"
+			default_namespace: "apache"
+			tags:              _apache_metrics_tags
 		}
 		cpu_seconds_total: {
-			description:   "The CPU time of various Apache processes."
-			relevant_when: "`ExtendedStatus On`"
-			type:          "counter"
-			tags: {
-				endpoint: _endpoint
-				host:     _host
-				type: {
+			description:       "The CPU time of various Apache processes."
+			relevant_when:     "`ExtendedStatus On`"
+			type:              "counter"
+			default_namespace: "apache"
+			tags:              _apache_metrics_tags & {
+				state: {
 					description: "The state of the connection"
 					required:    true
 					examples: ["children_system", "children_user", "system", "user"]
 				}
 			}
 		}
 		duration_seconds_total: {
-			description:   "The amount of time the Apache server has been running."
-			relevant_when: "`ExtendedStatus On`"
-			type:          "counter"
-			tags: {
-				endpoint: _endpoint
-				host:     _host
-			}
+			description:       "The amount of time the Apache server has been running."
+			relevant_when:     "`ExtendedStatus On`"
+			type:              "counter"
+			default_namespace: "apache"
+			tags:              _apache_metrics_tags
 		}
 		scoreboard: {
-			description: "The amount of times various Apache server tasks have been run."
-			type:        "gauge"
-			tags: {
-				endpoint: _endpoint
-				host:     _host
+			description:       "The amount of times various Apache server tasks have been run."
+			type:              "gauge"
+			default_namespace: "apache"
+			tags:              _apache_metrics_tags & {
 				state: {
 					description: "The connect state"
 					required:    true
@@ -190,43 +185,36 @@ components: sources: apache_metrics: {
 			}
 		}
 		sent_bytes_total: {
-			description:   "The amount of bytes sent by the Apache server."
-			relevant_when: "`ExtendedStatus On`"
-			type:          "counter"
-			tags: {
-				endpoint: _endpoint
-				host:     _host
-			}
+			description:       "The amount of bytes sent by the Apache server."
+			relevant_when:     "`ExtendedStatus On`"
+			type:              "counter"
+			default_namespace: "apache"
+			tags:              _apache_metrics_tags
+		}
+		up: {
+			description:       "If the Apache server is up or not."
+			type:              "gauge"
+			default_namespace: "apache"
+			tags:              _apache_metrics_tags
 		}
 		uptime_seconds_total: {
-			description: "The amount of time the Apache server has been running."
-			type:        "counter"
-			tags: {
-				endpoint: _endpoint
-				host:     _host
-			}
+			description:       "The amount of time the Apache server has been running."
+			type:              "counter"
+			default_namespace: "apache"
+			tags:              _apache_metrics_tags
 		}
 		workers: {
-			description: "Apache worker statuses."
-			type:        "gauge"
-			tags: {
-				endpoint: _endpoint
-				host:     _host
+			description:       "Apache worker statuses."
+			type:              "gauge"
+			default_namespace: "apache"
+			tags:              _apache_metrics_tags & {
 				state: {
 					description: "The state of the worker"
 					required:    true
 					examples: ["busy", "idle"]
 				}
 			}
 		}
-		up: {
-			description: "If the Apache server is up or not."
-			type:        "gauge"
-			tags: {
-				endpoint: _endpoint
-				host:     _host
-			}
-		}
 	}
 
 	how_it_works: {}