-
Notifications
You must be signed in to change notification settings - Fork 239
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add
database_observability.mysql component (#2053)
* [draft] Add dbo11y component WIP * basic impl of Run and Update * export targets from component * scaffolding for collectors * default args * refactor ConnectionInfo * docs * rename * use instanceKey * db engine in component name * import logic for schema_table collector * Add logic for QuerySample collector * Add test for QuerySample collector * add test for SchemaTable collector * add test for ConnectionInfo collector * Fix logging msgs Co-authored-by: MattNolf <86960799+matthewnolf@users.noreply.github.com> * add index.md * review feedback: add goleak verification to tests * add example to docs * Call Stop when exiting Run loop * Add more logging before exiting loop * rename Run into Start * add todo about possible refactoring * fix tests after rename * Update internal/component/database_observability/mysql/collector/connection_info.go Co-authored-by: William Dumont <william.dumont@grafana.com> * fixes * rename component * rename to collect_interval * no need to save tmp new args * Update docs/sources/reference/components/database_observability/database_observability.mysql.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * extend docs structure * Update docs/sources/reference/components/database_observability/database_observability.mysql.md Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com> * fix table * review docs again * review shutdown --------- Co-authored-by: MattNolf <86960799+matthewnolf@users.noreply.github.com> Co-authored-by: William Dumont <william.dumont@grafana.com> Co-authored-by: Clayton Cornell <131809008+clayton-cornell@users.noreply.github.com>
- Loading branch information
4 people
authored
Nov 26, 2024
1 parent
e19c783
commit c92ae07
Showing
13 changed files
with
1,081 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
12 changes: 12 additions & 0 deletions
12
docs/sources/reference/components/database_observability/_index.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| --- | ||
| canonical: https://grafana.com/docs/alloy/latest/reference/components/database_observability/ | ||
| description: Learn about the database_observability components in Grafana Alloy | ||
| title: database_observability | ||
| weight: 100 | ||
| --- | ||
|
|
||
| # database_observability | ||
|
|
||
| This section contains reference documentation for the `database_observability` components. | ||
|
|
||
| {{< section >}} |
88 changes: 88 additions & 0 deletions
88
...ces/reference/components/database_observability/database_observability.mysql.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,88 @@ | ||
| --- | ||
| canonical: https://grafana.com/docs/alloy/latest/reference/components/database_observability.mysql/ | ||
| description: Learn about database_observability.mysql | ||
| title: database_observability.mysql | ||
| --- | ||
|
|
||
| <span class="badge docs-labels__stage docs-labels__item">Experimental</span> | ||
|
|
||
| # database_observability.mysql | ||
|
|
||
| {{< docs/shared lookup="stability/experimental.md" source="alloy" version="<ALLOY_VERSION>" >}} | ||
|
|
||
| ## Usage | ||
|
|
||
| ```alloy | ||
| database_observability.mysql "LABEL" { | ||
| data_source_name = DATA_SOURCE_NAME | ||
| forward_to = [LOKI_RECEIVERS] | ||
| } | ||
| ``` | ||
|
|
||
| ## Arguments | ||
|
|
||
| The following arguments are supported: | ||
|
|
||
| | Name | Type | Description | Default | Required | | ||
| | -------------------- | -------------- | ------------------------------------------------------------------------------------------------------------------- | ------- | -------- | | ||
| | `data_source_name` | `secret` | [Data Source Name](https://github.com/go-sql-driver/mysql#dsn-data-source-name) for the MySQL server to connect to. | | yes | | ||
| | `forward_to` | `list(LogsReceiver)` | Where to forward log entries after processing. | | yes | | ||
| | `collect_interval` | `duration` | How frequently to collect query samples from database | `"10s"` | no | | ||
|
|
||
| ## Blocks | ||
|
|
||
| The `database_observability.mysql` component does not support any blocks, and is configured fully through arguments. | ||
|
|
||
| ## Example | ||
|
|
||
| ```alloy | ||
| database_observability.mysql "orders_db" { | ||
| data_source_name = "user:pass@mysql:3306/" | ||
| forward_to = [loki.write.logs_service.receiver] | ||
| } | ||
| prometheus.scrape "orders_db" { | ||
| targets = database_observability.mysql.orders_db.targets | ||
| honor_labels = true // required to keep job and instance labels | ||
| forward_to = [prometheus.remote_write.metrics_service.receiver] | ||
| } | ||
| prometheus.remote_write "metrics_service" { | ||
| endpoint { | ||
| url = sys.env("GCLOUD_HOSTED_METRICS_URL") | ||
| basic_auth { | ||
| username = sys.env("GCLOUD_HOSTED_METRICS_ID") | ||
| password = sys.env("GCLOUD_RW_API_KEY") | ||
| } | ||
| } | ||
| } | ||
| loki.write "logs_service" { | ||
| endpoint { | ||
| url = sys.env("GCLOUD_HOSTED_LOGS_URL") | ||
| basic_auth { | ||
| username = sys.env("GCLOUD_HOSTED_LOGS_ID") | ||
| password = sys.env("GCLOUD_RW_API_KEY") | ||
| } | ||
| } | ||
| } | ||
| ``` | ||
|
|
||
| <!-- START GENERATED COMPATIBLE COMPONENTS --> | ||
|
|
||
| ## Compatible components | ||
|
|
||
| `database_observability.mysql` can accept arguments from the following components: | ||
|
|
||
| - Components that export [Loki `LogsReceiver`](../../../compatibility/#loki-logsreceiver-exporters) | ||
|
|
||
| `database_observability.mysql` has exports that can be consumed by the following components: | ||
|
|
||
| - Components that consume [Targets](../../../compatibility/#targets-consumers) | ||
|
|
||
| {{< admonition type="note" >}} | ||
| Connecting some components may not be sensible or components may require further configuration to make the connection work correctly. | ||
| Refer to the linked documentation for more details. | ||
| {{< /admonition >}} | ||
|
|
||
| <!-- END GENERATED COMPATIBLE COMPONENTS --> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
71 changes: 71 additions & 0 deletions
71
internal/component/database_observability/mysql/collector/connection_info.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,71 @@ | ||
| package collector | ||
|
|
||
| import ( | ||
| "context" | ||
| "net" | ||
| "regexp" | ||
| "strings" | ||
|
|
||
| "github.com/go-sql-driver/mysql" | ||
| "github.com/prometheus/client_golang/prometheus" | ||
| ) | ||
|
|
||
| var rdsRegex = regexp.MustCompile(`(?P<identifier>[^\.]+)\.([^\.]+)\.(?P<region>[^\.]+)\.rds\.amazonaws\.com`) | ||
|
|
||
| type ConnectionInfoArguments struct { | ||
| DSN string | ||
| Registry *prometheus.Registry | ||
| } | ||
|
|
||
| type ConnectionInfo struct { | ||
| DSN string | ||
| Registry *prometheus.Registry | ||
| InfoMetric *prometheus.GaugeVec | ||
| } | ||
|
|
||
| func NewConnectionInfo(args ConnectionInfoArguments) (*ConnectionInfo, error) { | ||
| infoMetric := prometheus.NewGaugeVec(prometheus.GaugeOpts{ | ||
| Name: "connection_info", | ||
| Help: "Information about the connection", | ||
| }, []string{"provider_name", "region", "db_instance_identifier"}) | ||
|
|
||
| args.Registry.MustRegister(infoMetric) | ||
|
|
||
| return &ConnectionInfo{ | ||
| DSN: args.DSN, | ||
| Registry: args.Registry, | ||
| InfoMetric: infoMetric, | ||
| }, nil | ||
| } | ||
|
|
||
| func (c *ConnectionInfo) Start(ctx context.Context) error { | ||
| cfg, err := mysql.ParseDSN(c.DSN) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| var ( | ||
| providerName = "unknown" | ||
| providerRegion = "unknown" | ||
| dbInstanceIdentifier = "unknown" | ||
| ) | ||
|
|
||
| host, _, err := net.SplitHostPort(cfg.Addr) | ||
| if err == nil && host != "" { | ||
| if strings.HasSuffix(host, "rds.amazonaws.com") { | ||
| providerName = "aws" | ||
| matches := rdsRegex.FindStringSubmatch(host) | ||
| if len(matches) > 3 { | ||
| dbInstanceIdentifier = matches[1] | ||
| providerRegion = matches[3] | ||
| } | ||
| } | ||
| } | ||
|
|
||
| c.InfoMetric.WithLabelValues(providerName, providerRegion, dbInstanceIdentifier).Set(1) | ||
| return nil | ||
| } | ||
|
|
||
| func (c *ConnectionInfo) Stop() { | ||
| c.Registry.Unregister(c.InfoMetric) | ||
| } |
57 changes: 57 additions & 0 deletions
57
internal/component/database_observability/mysql/collector/connection_info_test.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| package collector | ||
|
|
||
| import ( | ||
| "context" | ||
| "fmt" | ||
| "strings" | ||
| "testing" | ||
|
|
||
| "github.com/prometheus/client_golang/prometheus" | ||
| "github.com/prometheus/client_golang/prometheus/testutil" | ||
| "github.com/stretchr/testify/require" | ||
| "go.uber.org/goleak" | ||
| ) | ||
|
|
||
| func TestConnectionInfo(t *testing.T) { | ||
| defer goleak.VerifyNone(t) | ||
|
|
||
| const baseExpectedMetrics = ` | ||
| # HELP connection_info Information about the connection | ||
| # TYPE connection_info gauge | ||
| connection_info{db_instance_identifier="%s",provider_name="%s",region="%s"} 1 | ||
| ` | ||
|
|
||
| testCases := []struct { | ||
| name string | ||
| dsn string | ||
| expectedMetrics string | ||
| }{ | ||
| { | ||
| name: "generic dsn", | ||
| dsn: "user:pass@tcp(localhost:3306)/db", | ||
| expectedMetrics: fmt.Sprintf(baseExpectedMetrics, "unknown", "unknown", "unknown"), | ||
| }, | ||
| { | ||
| name: "AWS/RDS dsn", | ||
| dsn: "user:pass@tcp(products-db.abc123xyz.us-east-1.rds.amazonaws.com:3306)/db", | ||
| expectedMetrics: fmt.Sprintf(baseExpectedMetrics, "products-db", "aws", "us-east-1"), | ||
| }, | ||
| } | ||
|
|
||
| for _, tc := range testCases { | ||
| reg := prometheus.NewRegistry() | ||
|
|
||
| collector, err := NewConnectionInfo(ConnectionInfoArguments{ | ||
| DSN: tc.dsn, | ||
| Registry: reg, | ||
| }) | ||
| require.NoError(t, err) | ||
| require.NotNil(t, collector) | ||
|
|
||
| err = collector.Start(context.Background()) | ||
| require.NoError(t, err) | ||
|
|
||
| err = testutil.GatherAndCompare(reg, strings.NewReader(tc.expectedMetrics)) | ||
| require.NoError(t, err) | ||
| } | ||
| } |
Oops, something went wrong.