custom-resources.yaml.tmpl

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: grafana-agent
  namespace: grafana-agent
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: grafana-agent
rules:
- apiGroups:
  - ""
  resources:
  - nodes
  - nodes/proxy
  - nodes/metrics
  - services
  - endpoints
  - pods
  - events
  verbs:
  - get
  - list
  - watch
- apiGroups:
  - networking.k8s.io
  resources:
  - ingresses
  verbs:
  - get
  - list
  - watch
- nonResourceURLs:
  - /metrics
  - /metrics/cadvisor
  verbs:
  - get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: grafana-agent
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: grafana-agent
subjects:
- kind: ServiceAccount
  name: grafana-agent
  namespace: grafana-agent
---
apiVersion: monitoring.grafana.com/v1alpha1
kind: GrafanaAgent
metadata:
  name: grafana-agent
  namespace: grafana-agent
  labels:
    app: grafana-agent
spec:
  image: grafana/agent:v${agent_version}
  containers:
  - name: grafana-agent
    resources:
      requests:
        memory: 100Mi
        cpu: 20m
  - name: config-reloader
    resources:
      requests:
        memory: 100Mi
        cpu: 20m
  logLevel: debug
  serviceAccountName: grafana-agent
  priorityClassName: system-node-critical
%{ if length(tolerations) > 0 ~}
  tolerations:
  %{~ for entry in tolerations ~}
  - effect: ${entry.effect}
    operator: ${entry.operator}
    %{~ if entry.key != null ~}
    key: ${entry.key}
    %{~ endif ~}
    %{~ if entry.value != null ~}
    value: ${entry.value}
    %{~ endif ~}
%{ endfor ~}
%{ endif ~}
  metrics:
    instanceSelector:
      matchLabels:
        agent: grafana-agent-metrics
%{~ if length(external_labels) > 0 }
    externalLabels:
%{ for key, value in external_labels ~}
      ${key}: ${value}
%{ endfor ~}
%{ endif ~}
  logs:
    instanceSelector:
      matchLabels:
        agent: grafana-agent-logs
  integrations:
    selector:
      matchLabels:
        agent: grafana-agent-integrations
---
apiVersion: monitoring.grafana.com/v1alpha1
kind: MetricsInstance
metadata:
  name: primary
  namespace: grafana-agent
  labels:
    agent: grafana-agent-metrics
spec:
  remoteWrite:
  - url: ${metrics_url}
    basicAuth:
      username:
        name: ${metrics_secret}
        key: ${metrics_username_key}
      password:
        name: ${metrics_secret}
        key: ${metrics_password_key}

  # Supply an empty namespace selector to look in all namespaces. Remove
  # this to only look in the same namespace as the MetricsInstance CR
  serviceMonitorNamespaceSelector: { }
  serviceMonitorSelector:
    matchLabels:
      instance: primary

  # Supply an empty namespace selector to look in all namespaces. Remove
  # this to only look in the same namespace as the MetricsInstance CR.
  podMonitorNamespaceSelector: { }
  podMonitorSelector:
    matchLabels:
      instance: primary

  # Supply an empty namespace selector to look in all namespaces. Remove
  # this to only look in the same namespace as the MetricsInstance CR.
  probeNamespaceSelector: { }
  probeSelector:
    matchLabels:
      instance: primary
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  labels:
    instance: primary
  name: kubelet-monitor
  namespace: default
spec:
  endpoints:
  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    honorLabels: true
    interval: 60s
    metricRelabelings:
    - action: keep
      regex: container_network_transmit_packets_total|kube_horizontalpodautoscaler_spec_min_replicas|kube_pod_container_resource_limits|kubelet_pod_worker_duration_seconds_bucket|storage_operation_errors_total|kube_statefulset_status_replicas_updated|container_memory_working_set_bytes|kubelet_pod_start_duration_seconds_count|kube_statefulset_status_observed_generation|container_cpu_usage_seconds_total|kube_pod_owner|cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits|cluster:namespace:pod_memory:active:kube_pod_container_resource_limits|node_namespace_pod_container:container_memory_swap|container_fs_writes_total|kubelet_running_pods|kubelet_volume_stats_capacity_bytes|kube_replicaset_owner|kube_deployment_spec_replicas|kube_statefulset_metadata_generation|node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile|kubelet_certificate_manager_client_ttl_seconds|container_fs_reads_bytes_total|kubelet_cgroup_manager_duration_seconds_count|kubelet_running_container_count|container_fs_writes_bytes_total|container_network_receive_packets_dropped_total|kube_node_status_allocatable|go_goroutines|kubelet_node_name|kubelet_volume_stats_inodes|namespace_memory:kube_pod_container_resource_limits:sum|container_cpu_cfs_periods_total|kubelet_runtime_operations_errors_total|kubelet_running_pod_count|rest_client_requests_total|kube_job_spec_completions|namespace_cpu:kube_pod_container_resource_requests:sum|kube_node_info|namespace_workload_pod|kubernetes_build_info|machine_memory_bytes|container_network_receive_packets_total|kubelet_runtime_operations_total|kubelet_volume_stats_available_bytes|kube_daemonset_status_number_available|kube_pod_container_status_waiting_reason|up|storage_operation_duration_seconds_count|kube_daemonset_status_current_number_scheduled|container_network_receive_bytes_total|cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests|container_memory_swap|volume_manager_total_volumes|container_memory_rss|kube_deployment_status_observed_generation|kube_deployment_metadata_generation|kube_pod_info|kube_node_status_condition|kubelet_pleg_relist_duration_seconds_bucket|kubelet_volume_stats_inodes_used|kube_horizontalpodautoscaler_status_desired_replicas|kube_statefulset_status_current_revision|namespace_memory:kube_pod_container_resource_requests:sum|kube_resourcequota|process_cpu_seconds_total|kubelet_node_config_error|kubelet_cgroup_manager_duration_seconds_bucket|kubelet_pod_worker_duration_seconds_count|kube_horizontalpodautoscaler_status_current_replicas|kube_node_spec_taint|process_resident_memory_bytes|kube_statefulset_status_replicas_ready|node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate|node_namespace_pod_container:container_memory_cache|kubelet_pleg_relist_interval_seconds_bucket|kubelet_certificate_manager_server_ttl_seconds|kubelet_server_expiration_renew_errors|container_network_transmit_bytes_total|container_network_transmit_packets_dropped_total|cluster:namespace:pod_memory:active:kube_pod_container_resource_requests|kubelet_running_containers|kube_statefulset_replicas|kube_job_status_succeeded|kube_deployment_status_replicas_updated|namespace_workload_pod:kube_pod_owner:relabel|container_fs_reads_total|container_memory_cache|kube_node_status_capacity|kubelet_certificate_manager_client_expiration_renew_errors|kube_job_failed|namespace_cpu:kube_pod_container_resource_limits:sum|node_namespace_pod_container:container_memory_working_set_bytes|kube_namespace_status_phase|kube_statefulset_status_replicas|kube_horizontalpodautoscaler_spec_max_replicas|kube_daemonset_status_desired_number_scheduled|kubelet_pleg_relist_duration_seconds_count|kube_daemonset_status_updated_number_scheduled|kube_daemonset_status_number_misscheduled|kube_deployment_status_replicas_available|node_namespace_pod_container:container_memory_rss|kube_pod_container_resource_requests|container_cpu_cfs_throttled_periods_total|kube_statefulset_status_update_revision|kube_pod_status_phase
      sourceLabels:
      - __name__
    - action: replace
      targetLabel: job
      replacement: integrations/kubernetes/kubelet
    - action: replace
      targetLabel: cluster
      replacement: ${cluster_name}
    port: https-metrics
    relabelings:
    - sourceLabels:
      - __metrics_path__
      targetLabel: metrics_path
    - action: replace
      targetLabel: job
      replacement: integrations/kubernetes/kubelet
    scheme: https
    tlsConfig:
      insecureSkipVerify: true
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      app.kubernetes.io/name: kubelet
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  labels:
    instance: primary
  name: cadvisor-monitor
  namespace: default
spec:
  endpoints:
  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
    honorLabels: true
    honorTimestamps: false
    interval: 60s
    metricRelabelings:
    - action: keep
      regex: container_network_transmit_packets_total|kube_horizontalpodautoscaler_spec_min_replicas|kube_pod_container_resource_limits|kubelet_pod_worker_duration_seconds_bucket|storage_operation_errors_total|kube_statefulset_status_replicas_updated|container_memory_working_set_bytes|kubelet_pod_start_duration_seconds_count|kube_statefulset_status_observed_generation|container_cpu_usage_seconds_total|kube_pod_owner|cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits|cluster:namespace:pod_memory:active:kube_pod_container_resource_limits|node_namespace_pod_container:container_memory_swap|container_fs_writes_total|kubelet_running_pods|kubelet_volume_stats_capacity_bytes|kube_replicaset_owner|kube_deployment_spec_replicas|kube_statefulset_metadata_generation|node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile|kubelet_certificate_manager_client_ttl_seconds|container_fs_reads_bytes_total|kubelet_cgroup_manager_duration_seconds_count|kubelet_running_container_count|container_fs_writes_bytes_total|container_network_receive_packets_dropped_total|kube_node_status_allocatable|go_goroutines|kubelet_node_name|kubelet_volume_stats_inodes|namespace_memory:kube_pod_container_resource_limits:sum|container_cpu_cfs_periods_total|kubelet_runtime_operations_errors_total|kubelet_running_pod_count|rest_client_requests_total|kube_job_spec_completions|namespace_cpu:kube_pod_container_resource_requests:sum|kube_node_info|namespace_workload_pod|kubernetes_build_info|machine_memory_bytes|container_network_receive_packets_total|kubelet_runtime_operations_total|kubelet_volume_stats_available_bytes|kube_daemonset_status_number_available|kube_pod_container_status_waiting_reason|up|storage_operation_duration_seconds_count|kube_daemonset_status_current_number_scheduled|container_network_receive_bytes_total|cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests|container_memory_swap|volume_manager_total_volumes|container_memory_rss|kube_deployment_status_observed_generation|kube_deployment_metadata_generation|kube_pod_info|kube_node_status_condition|kubelet_pleg_relist_duration_seconds_bucket|kubelet_volume_stats_inodes_used|kube_horizontalpodautoscaler_status_desired_replicas|kube_statefulset_status_current_revision|namespace_memory:kube_pod_container_resource_requests:sum|kube_resourcequota|process_cpu_seconds_total|kubelet_node_config_error|kubelet_cgroup_manager_duration_seconds_bucket|kubelet_pod_worker_duration_seconds_count|kube_horizontalpodautoscaler_status_current_replicas|kube_node_spec_taint|process_resident_memory_bytes|kube_statefulset_status_replicas_ready|node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate|node_namespace_pod_container:container_memory_cache|kubelet_pleg_relist_interval_seconds_bucket|kubelet_certificate_manager_server_ttl_seconds|kubelet_server_expiration_renew_errors|container_network_transmit_bytes_total|container_network_transmit_packets_dropped_total|cluster:namespace:pod_memory:active:kube_pod_container_resource_requests|kubelet_running_containers|kube_statefulset_replicas|kube_job_status_succeeded|kube_deployment_status_replicas_updated|namespace_workload_pod:kube_pod_owner:relabel|container_fs_reads_total|container_memory_cache|kube_node_status_capacity|kubelet_certificate_manager_client_expiration_renew_errors|kube_job_failed|namespace_cpu:kube_pod_container_resource_limits:sum|node_namespace_pod_container:container_memory_working_set_bytes|kube_namespace_status_phase|kube_statefulset_status_replicas|kube_horizontalpodautoscaler_spec_max_replicas|kube_daemonset_status_desired_number_scheduled|kubelet_pleg_relist_duration_seconds_count|kube_daemonset_status_updated_number_scheduled|kube_daemonset_status_number_misscheduled|kube_deployment_status_replicas_available|node_namespace_pod_container:container_memory_rss|kube_pod_container_resource_requests|container_cpu_cfs_throttled_periods_total|kube_statefulset_status_update_revision|kube_pod_status_phase
      sourceLabels:
      - __name__
    - action: replace
      targetLabel: job
      replacement: integrations/kubernetes/cadvisor
    - action: replace
      targetLabel: cluster
      replacement: ${cluster_name}
    path: /metrics/cadvisor
    port: https-metrics
    relabelings:
    - sourceLabels:
      - __metrics_path__
      targetLabel: metrics_path
    - action: replace
      targetLabel: job
      replacement: integrations/kubernetes/cadvisor
    scheme: https
    tlsConfig:
      insecureSkipVerify: true
  namespaceSelector:
    matchNames:
    - default
  selector:
    matchLabels:
      app.kubernetes.io/name: kubelet
---
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
  labels:
    instance: primary
  name: kube-state-metrics-monitor
  namespace: kube-system
spec:
  jobLabel: integrations/kubernetes/kube-state-metrics
  selector:
    matchLabels:
      app.kubernetes.io/name: kube-state-metrics
  namespaceSelector:
    matchNames:
    - kube-system
  endpoints:
  - port: http
    interval: 60s
    honorLabels: true
    metricRelabelings:
    - action: keep
      regex: container_network_transmit_packets_total|kube_horizontalpodautoscaler_spec_min_replicas|kube_pod_container_resource_limits|kubelet_pod_worker_duration_seconds_bucket|storage_operation_errors_total|kube_statefulset_status_replicas_updated|container_memory_working_set_bytes|kubelet_pod_start_duration_seconds_count|kube_statefulset_status_observed_generation|container_cpu_usage_seconds_total|kube_pod_owner|cluster:namespace:pod_cpu:active:kube_pod_container_resource_limits|cluster:namespace:pod_memory:active:kube_pod_container_resource_limits|node_namespace_pod_container:container_memory_swap|container_fs_writes_total|kubelet_running_pods|kubelet_volume_stats_capacity_bytes|kube_replicaset_owner|kube_deployment_spec_replicas|kube_statefulset_metadata_generation|node_quantile:kubelet_pleg_relist_duration_seconds:histogram_quantile|kubelet_certificate_manager_client_ttl_seconds|container_fs_reads_bytes_total|kubelet_cgroup_manager_duration_seconds_count|kubelet_running_container_count|container_fs_writes_bytes_total|container_network_receive_packets_dropped_total|kube_node_status_allocatable|go_goroutines|kubelet_node_name|kubelet_volume_stats_inodes|namespace_memory:kube_pod_container_resource_limits:sum|container_cpu_cfs_periods_total|kubelet_runtime_operations_errors_total|kubelet_running_pod_count|rest_client_requests_total|kube_job_spec_completions|namespace_cpu:kube_pod_container_resource_requests:sum|kube_node_info|namespace_workload_pod|kubernetes_build_info|machine_memory_bytes|container_network_receive_packets_total|kubelet_runtime_operations_total|kubelet_volume_stats_available_bytes|kube_daemonset_status_number_available|kube_pod_container_status_waiting_reason|up|storage_operation_duration_seconds_count|kube_daemonset_status_current_number_scheduled|container_network_receive_bytes_total|cluster:namespace:pod_cpu:active:kube_pod_container_resource_requests|container_memory_swap|volume_manager_total_volumes|container_memory_rss|kube_deployment_status_observed_generation|kube_deployment_metadata_generation|kube_pod_info|kube_node_status_condition|kubelet_pleg_relist_duration_seconds_bucket|kubelet_volume_stats_inodes_used|kube_horizontalpodautoscaler_status_desired_replicas|kube_statefulset_status_current_revision|namespace_memory:kube_pod_container_resource_requests:sum|kube_resourcequota|process_cpu_seconds_total|kubelet_node_config_error|kubelet_cgroup_manager_duration_seconds_bucket|kubelet_pod_worker_duration_seconds_count|kube_horizontalpodautoscaler_status_current_replicas|kube_node_spec_taint|process_resident_memory_bytes|kube_statefulset_status_replicas_ready|node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate|node_namespace_pod_container:container_memory_cache|kubelet_pleg_relist_interval_seconds_bucket|kubelet_certificate_manager_server_ttl_seconds|kubelet_server_expiration_renew_errors|container_network_transmit_bytes_total|container_network_transmit_packets_dropped_total|cluster:namespace:pod_memory:active:kube_pod_container_resource_requests|kubelet_running_containers|kube_statefulset_replicas|kube_job_status_succeeded|kube_deployment_status_replicas_updated|namespace_workload_pod:kube_pod_owner:relabel|container_fs_reads_total|container_memory_cache|kube_node_status_capacity|kubelet_certificate_manager_client_expiration_renew_errors|kube_job_failed|namespace_cpu:kube_pod_container_resource_limits:sum|node_namespace_pod_container:container_memory_working_set_bytes|kube_namespace_status_phase|kube_statefulset_status_replicas|kube_horizontalpodautoscaler_spec_max_replicas|kube_daemonset_status_desired_number_scheduled|kubelet_pleg_relist_duration_seconds_count|kube_daemonset_status_updated_number_scheduled|kube_daemonset_status_number_misscheduled|kube_deployment_status_replicas_available|node_namespace_pod_container:container_memory_rss|kube_pod_container_resource_requests|container_cpu_cfs_throttled_periods_total|kube_statefulset_status_update_revision|kube_pod_status_phase
      sourceLabels:
      - __name__
    relabelings:
    - action: replace
      targetLabel: job
      replacement: integrations/kubernetes/kube-state-metrics
---
apiVersion: monitoring.grafana.com/v1alpha1
kind: LogsInstance
metadata:
  name: primary
  namespace: grafana-agent
  labels:
    agent: grafana-agent-logs
spec:
  clients:
  - url: ${logs_url}
    basicAuth:
      username:
        name: ${logs_secret}
        key: ${logs_username_key}
      password:
        name: ${logs_secret}
        key: ${logs_password_key}

  # Supply an empty namespace selector to look in all namespaces. Remove
  # this to only look in the same namespace as the LogsInstance CR
  podLogsNamespaceSelector: { }
  podLogsSelector:
    matchLabels:
      instance: primary
---
# Collect kubernetes API events.
apiVersion: monitoring.grafana.com/v1alpha1
kind: Integration
metadata:
  name: eventhandler
  namespace: grafana-agent
  labels:
    agent: grafana-agent-integrations
spec:
  name: eventhandler
  type:
    unique: true
  config:
    logs_instance: grafana-agent/primary
    cache_path: "/var/lib/grafana-agent/data/eventhandler.cache"
---
apiVersion: monitoring.grafana.com/v1alpha1
kind: PodLogs
metadata:
  labels:
    instance: primary
  name: kubernetes-pods
  namespace: grafana-agent
spec:
  pipelineStages:
  - cri: { }
  namespaceSelector:
    any: true
  selector:
    matchLabels: { }
  relabelings:
  - sourceLabels:
    - __meta_kubernetes_pod_node_name
    targetLabel: host
  # add label "app" from pod to each log row to enable tracing to logs
  - sourceLabels:
      - __meta_kubernetes_pod_label_app
    targetLabel: app