diff --git a/go.mod b/go.mod index d6811740ab..faa4ef7570 100644 --- a/go.mod +++ b/go.mod @@ -92,7 +92,6 @@ require ( k8s.io/cli-runtime v0.28.0 k8s.io/client-go v0.28.0 k8s.io/kubectl v0.28.0 - modernc.org/sqlite v1.26.0 sigs.k8s.io/controller-runtime v0.14.6 ) @@ -114,11 +113,11 @@ require ( github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect + github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/hashicorp/go-uuid v1.0.2 // indirect github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/imdario/mergo v0.3.12 // indirect - github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/minio/minio-go v6.0.14+incompatible // indirect @@ -129,7 +128,6 @@ require ( github.com/percona/percona-backup-mongodb v1.8.1 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/posener/complete v1.2.3 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/riywo/loginshell v0.0.0-20200815045211-7d26008be1ab // indirect github.com/sergi/go-diff v1.2.0 // indirect github.com/spf13/pflag v1.0.5 // indirect @@ -143,15 +141,6 @@ require ( k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect - lukechampine.com/uint128 v1.2.0 // indirect - modernc.org/cc/v3 v3.40.0 // indirect - modernc.org/ccgo/v3 v3.16.13 // indirect - modernc.org/libc v1.24.1 // indirect - modernc.org/mathutil v1.5.0 // indirect - modernc.org/memory v1.6.0 // indirect - modernc.org/opt v0.1.3 // indirect - modernc.org/strutil v1.1.3 // indirect - modernc.org/token v1.0.1 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect diff --git a/go.sum b/go.sum index ffb9938c40..337ac4744f 100644 --- a/go.sum +++ b/go.sum @@ -486,8 +486,6 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -712,9 +710,6 @@ github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwa github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/ramr/go-reaper v0.2.1 h1:zww+wlQOvTjBZuk1920R/e0GFEb6O7+B0WQLV6dM924= github.com/ramr/go-reaper v0.2.1/go.mod h1:AVypdzrcCXjSc/JYnlXl8TsB+z84WyFzxWE8Jh0MOJc= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= @@ -1233,34 +1228,6 @@ k8s.io/kubectl v0.28.0 h1:qhfju0OaU+JGeBlToPeeIg2UJUWP++QwTkpio6nlPKg= k8s.io/kubectl v0.28.0/go.mod h1:1We+E5nSX3/TVoSQ6y5Bzld5OhTBHZHlKEYl7g/NaTk= k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= -lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= -modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= -modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= -modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= -modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= -modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= -modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= -modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= -modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= -modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM= -modernc.org/libc v1.24.1/go.mod h1:FmfO1RLrU3MHJfyi9eYYmZBfi/R+tqZ6+hQ3yQQUkak= -modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= -modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/memory v1.6.0 h1:i6mzavxrE9a30whzMfwf7XWVODx2r5OYXvU46cirX7o= -modernc.org/memory v1.6.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.26.0 h1:SocQdLRSYlA8W99V8YH0NES75thx19d9sB/aFc4R8Lw= -modernc.org/sqlite v1.26.0/go.mod h1:FL3pVXie73rg3Rii6V/u5BoHlSoyeZeIgKZEgHARyCU= -modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= -modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= -modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY= -modernc.org/tcl v1.15.2/go.mod h1:3+k/ZaEbKrC8ePv8zJWPtBSW0V7Gg9g8rkmhI1Kfs3c= -modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg= -modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY= -modernc.org/z v1.7.3/go.mod h1:Ipv4tsdxZRbQyLq9Q1M6gdbkxYzdlrciF2Hi/lS7nWE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/managed/cmd/pmm-managed/main.go b/managed/cmd/pmm-managed/main.go index 0441592fdb..7a3a10cd50 100644 --- a/managed/cmd/pmm-managed/main.go +++ b/managed/cmd/pmm-managed/main.go @@ -779,6 +779,12 @@ func main() { //nolint:cyclop,maintidx } pmmdb.DSN.Params = q.Encode() + grafanadb := ds.GrafanaDBSelect + grafanadb.DSN.Scheme = "postgres" + grafanadb.DSN.Host = *postgresAddrF + grafanadb.DSN.DB = "grafana" + grafanadb.DSN.Params = q.Encode() + clickhouseDSN := "tcp://" + *clickhouseAddrF + "/" + *clickHouseDatabaseF qanDB := ds.QanDBSelect diff --git a/managed/services/config/pmm-managed.yaml b/managed/services/config/pmm-managed.yaml index c9dee75d74..133d1bfc78 100644 --- a/managed/services/config/pmm-managed.yaml +++ b/managed/services/config/pmm-managed.yaml @@ -19,7 +19,10 @@ services: GRAFANADB_SELECT: enabled: true timeout: 5s - db_file: /srv/grafana/grafana.db + use_separate_credentials: true + separate_credentials: + username: grafana + password: grafana ENV_VARS: enabled: true reporting: diff --git a/managed/services/telemetry/config.default.yml b/managed/services/telemetry/config.default.yml index a7803f8109..3d978acae6 100644 --- a/managed/services/telemetry/config.default.yml +++ b/managed/services/telemetry/config.default.yml @@ -362,7 +362,7 @@ telemetry: - id: GrafanaStatDailyActiveUsers source: GRAFANADB_SELECT - query: count(*) AS count FROM user WHERE last_seen_at > datetime('now', '-1 day') + query: count(*) AS count FROM public.user WHERE last_seen_at > now() - interval '1 day' summary: "Daily active users" data: - metric_name: "pmm_server_grafana_stat_daily_active_users" @@ -761,7 +761,7 @@ telemetry: #Grafana - id: GrafanaUsersCount source: GRAFANADB_SELECT - query: count(*) AS count FROM user + query: count(*) AS count FROM public.user summary: "Grafana Users Count" data: - metric_name: "grafana_users_count" @@ -769,7 +769,7 @@ telemetry: - id: GrafanaDarkThemeUsersCount source: GRAFANADB_SELECT - query: count(u.id) as count from user u left join preferences p on p.user_id = u.id where p.theme = 'dark' + query: count(u.id) as count from public.user u left join preferences p on p.user_id = u.id where p.theme = 'dark' summary: "Grafana Dark Theme Users Count" data: - metric_name: "grafana_dark_theme_users_count" @@ -777,7 +777,7 @@ telemetry: - id: GrafanaLightThemeUsersCount source: GRAFANADB_SELECT - query: count(u.id) as count from user u left join preferences p on p.user_id = u.id where p.theme = 'light' + query: count(u.id) as count from public.user u left join preferences p on p.user_id = u.id where p.theme = 'light' summary: "Grafana Light Theme Users Count" data: - metric_name: "grafana_light_theme_users_count" @@ -785,7 +785,7 @@ telemetry: - id: GrafanaDefaultThemeUsersCount source: GRAFANADB_SELECT - query: count(u.id) as count from user u left join preferences p on p.user_id = u.id where p.theme = '' or p.theme is null + query: count(u.id) as count from public.user u left join preferences p on p.user_id = u.id where p.theme = '' or p.theme is null summary: "Grafana Default Theme Users Count" data: - metric_name: "grafana_default_theme_users_count" @@ -793,7 +793,7 @@ telemetry: - id: GrafanaCustomDashboardsCount source: GRAFANADB_SELECT - query: count(*) as count from dashboard where created_by != -1 and is_folder = 0 + query: count(*) as count from dashboard where created_by != -1 and is_folder = false summary: "Grafana Custom Dashboards Count" data: - metric_name: "grafana_custom_dashboards_count" @@ -801,7 +801,7 @@ telemetry: - id: GrafanaCustomDashboardsPerPillar source: GRAFANADB_SELECT - query: inn.title as pillar, count(d.title) as count from dashboard d left join dashboard inn ON d.folder_id = inn.id where d.created_by != -1 and d.is_folder = 0 group by inn.title + query: inn.title as pillar, count(d.title) as count from dashboard d left join dashboard inn ON d.folder_id = inn.id where d.created_by != -1 and d.is_folder = false group by inn.title summary: "Grafana Custom Dashboards Count By Pillar" transform: type: JSON @@ -810,6 +810,7 @@ telemetry: - metric_name: "pillar" column: "pillar" - metric_name: "count" + column: "count" # API usage - id: APIUsageMetrics diff --git a/managed/services/telemetry/config.go b/managed/services/telemetry/config.go index 97ea351a8c..361aa4a561 100644 --- a/managed/services/telemetry/config.go +++ b/managed/services/telemetry/config.go @@ -41,17 +41,17 @@ const ( dsVM = DataSourceName("VM") dsQANDBSelect = DataSourceName("QANDB_SELECT") dsPMMDBSelect = DataSourceName("PMMDB_SELECT") - dsGrafanaDBSelect = DataSourceName("GRAFANADB_SELECT") + dsGRAFANADBSelect = DataSourceName("GRAFANADB_SELECT") dsEnvVars = DataSourceName("ENV_VARS") ) // DataSources holds all possible data source types. type DataSources struct { - VM *DataSourceVictoriaMetrics `yaml:"VM"` - QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"` - PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"` - GrafanaDBSelect *DSGrafanaSqliteDB `yaml:"GRAFANADB_SELECT"` - EnvVars *DSConfigEnvVars `yaml:"ENV_VARS"` + VM *DSConfigVM `yaml:"VM"` + QanDBSelect *DSConfigQAN `yaml:"QANDB_SELECT"` + PmmDBSelect *DSConfigPMMDB `yaml:"PMMDB_SELECT"` + GrafanaDBSelect *DSConfigGrafanaDB `yaml:"GRAFANADB_SELECT"` + EnvVars *DSConfigEnvVars `yaml:"ENV_VARS"` } // ServiceConfig telemetry config. @@ -76,32 +76,25 @@ type DSConfigQAN struct { DSN string `yaml:"-"` } -// DataSourceVictoriaMetrics telemetry config. -type DataSourceVictoriaMetrics struct { +// DSConfigVM telemetry config. +type DSConfigVM struct { Enabled bool `yaml:"enabled"` Timeout time.Duration `yaml:"timeout"` Address string `yaml:"address"` } -// DSGrafanaSqliteDB telemetry config. -type DSGrafanaSqliteDB struct { - Enabled bool `yaml:"enabled"` - Timeout time.Duration `yaml:"timeout"` - DBFile string `yaml:"db_file"` -} - // DSConfigPMMDB telemetry config. type DSConfigPMMDB struct { //nolint:musttag Enabled bool `yaml:"enabled"` Timeout time.Duration `yaml:"timeout"` UseSeparateCredentials bool `yaml:"use_separate_credentials"` - // Credentials used by PMM - DSN struct { + DSN struct { Scheme string Host string DB string Params string } `yaml:"-"` + // Credentials used by PMM Credentials struct { Username string Password string @@ -112,11 +105,15 @@ type DSConfigPMMDB struct { //nolint:musttag } `yaml:"separate_credentials"` } +// DSConfigGrafanaDB is a Grafana telemetry config. +type DSConfigGrafanaDB DSConfigPMMDB + +// DSConfigEnvVars is an env variable telemetry config. type DSConfigEnvVars struct { Enabled bool `yaml:"enabled"` } -// Config is a telemetry config. +// Config telemetry config. type Config struct { ID string `yaml:"id"` Source string `yaml:"source"` @@ -139,7 +136,7 @@ type ConfigTransformType string const ( // JSONTransform converts multiple metrics in one formatted as JSON. JSONTransform = ConfigTransformType("JSON") - // StripValuesTransform strips values from metrics, replacing them with 0/1 depending on presence. + // StripValuesTransform strips values from metrics, replacing them with 1 to indicate presence. StripValuesTransform = ConfigTransformType("StripValues") ) diff --git a/managed/services/telemetry/config_test.go b/managed/services/telemetry/config_test.go index 7a7da4ab7e..445e716427 100644 --- a/managed/services/telemetry/config_test.go +++ b/managed/services/telemetry/config_test.go @@ -47,10 +47,12 @@ datasources: GRAFANADB_SELECT: enabled: true timeout: 2s - db_file: /srv/grafana/grafana.db + use_separate_credentials: true + separate_credentials: + username: grafana + password: grafana ENV_VARS: enabled: true - reporting: send: true send_on_start: true @@ -74,7 +76,7 @@ reporting: SendTimeout: time.Second * 10, }, DataSources: DataSources{ - VM: &DataSourceVictoriaMetrics{ + VM: &DSConfigVM{ Enabled: true, Timeout: time.Second * 2, Address: "http://localhost:80/victoriametrics/", @@ -95,10 +97,17 @@ reporting: Password: "pmm-managed", }, }, - GrafanaDBSelect: &DSGrafanaSqliteDB{ - Enabled: true, - Timeout: time.Second * 2, - DBFile: "/srv/grafana/grafana.db", + GrafanaDBSelect: &DSConfigGrafanaDB{ + Enabled: true, + Timeout: time.Second * 2, + UseSeparateCredentials: true, + SeparateCredentials: struct { + Username string `yaml:"username"` + Password string `yaml:"password"` + }{ + Username: "grafana", + Password: "grafana", + }, }, EnvVars: &DSConfigEnvVars{ Enabled: true, diff --git a/managed/services/telemetry/datasource_grafana_sqlitedb.go b/managed/services/telemetry/datasource_grafana_sqlitedb.go deleted file mode 100644 index 1c45141088..0000000000 --- a/managed/services/telemetry/datasource_grafana_sqlitedb.go +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright (C) 2023 Percona LLC -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -// Package telemetry provides telemetry functionality. -package telemetry - -import ( - "context" - "database/sql" - "io" - "os" - - // Events, errors and driver for grafana sqlite database. - pmmv1 "github.com/percona-platform/saas/gen/telemetry/events/pmm" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - _ "modernc.org/sqlite" -) - -type dsGrafanaSelect struct { - l *logrus.Entry - config DSGrafanaSqliteDB - db *sql.DB - tempFile string -} - -// check interfaces. -var ( - _ DataSource = (*dsGrafanaSelect)(nil) -) - -// Enabled flag that determines if data source is enabled. -func (d *dsGrafanaSelect) Enabled() bool { - return d.config.Enabled -} - -// NewDataSourceGrafanaSqliteDB makes new data source for grafana sqlite database metrics. -func NewDataSourceGrafanaSqliteDB(config DSGrafanaSqliteDB, l *logrus.Entry) DataSource { - return &dsGrafanaSelect{ - l: l, - config: config, - db: nil, - tempFile: "", - } -} - -func (d *dsGrafanaSelect) Init(ctx context.Context) error { - // validate source file db - sourceFileStat, err := os.Stat(d.config.DBFile) - if err != nil { - return err - } - - if sourceFileStat.Size() == 0 { - return errors.Errorf("Sourcefile %s is empty.", d.config.DBFile) - } - - if !sourceFileStat.Mode().IsRegular() { - return errors.Wrapf(err, "%s is not a regular file", d.config.DBFile) - } - - source, err := os.Open(d.config.DBFile) - if err != nil { - return err - } - - tempFile, err := os.CreateTemp(os.TempDir(), "grafana") - if err != nil { - return err - } - - defer func() { - if err := source.Close(); err != nil { - d.l.Errorf("Error closing file. %s", err) - } - }() - - nBytes, err := io.Copy(tempFile, source) - d.l.Debugf("grafana sqlitedb copied with total bytes: %d", nBytes) - if err != nil || nBytes == 0 { - return errors.Wrapf(err, "cannot create copy of database file %s", d.config.DBFile) - } - - db, err := sql.Open("sqlite", tempFile.Name()) - if err != nil { - return err - } - - d.tempFile = tempFile.Name() - d.db = db - - return nil -} - -func (d *dsGrafanaSelect) FetchMetrics(ctx context.Context, config Config) ([]*pmmv1.ServerMetric_Metric, error) { - if d.db == nil { - return nil, errors.Errorf("temporary grafana database is not initialized: %s", d.config.DBFile) - } - return fetchMetricsFromDB(ctx, d.l, d.config.Timeout, d.db, config) -} - -func (d *dsGrafanaSelect) Dispose(ctx context.Context) error { - err := d.db.Close() - if err != nil { - return err - } - - err = os.Remove(d.tempFile) - if err != nil { - return errors.Wrapf(err, "failed to remove sqlite database file") - } - - return nil -} diff --git a/managed/services/telemetry/datasource_grafana_sqlitedb_test.go b/managed/services/telemetry/datasource_grafana_sqlitedb_test.go deleted file mode 100644 index ec140ce550..0000000000 --- a/managed/services/telemetry/datasource_grafana_sqlitedb_test.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (C) 2023 Percona LLC -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -// Package telemetry provides telemetry functionality. -package telemetry - -import ( - "context" - "path/filepath" - "testing" - "time" - - "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestGrafanaSqliteDatasource(t *testing.T) { - t.Parallel() - logger := logrus.StandardLogger() - logger.SetLevel(logrus.DebugLevel) - logEntry := logrus.NewEntry(logger) - - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(func() { - cancel() - }) - - config := &Config{ - ID: "test", - Source: "GRAFANADB_SELECT", - Query: "count(*) AS total from user", - Summary: "Simple query", - Data: []ConfigData{ - { - MetricName: "total_users_in_database", - Column: "total", - }, - }, - } - - t.Run("get metrics from db", func(t *testing.T) { - t.Parallel() - databaseFile, err := filepath.Abs("../../testdata/telemetry/grafana_sqlite.db") - require.NoError(t, err) - - conf := &DSGrafanaSqliteDB{ - Enabled: true, - Timeout: time.Second * 10, - DBFile: databaseFile, - } - grafanaDB := NewDataSourceGrafanaSqliteDB(*conf, logEntry) - - err = grafanaDB.Init(ctx) - require.NoError(t, err) - - metrics, err := grafanaDB.FetchMetrics(ctx, *config) - require.NoError(t, err) - assert.Equal(t, len(metrics), 1) - - err = grafanaDB.Dispose(ctx) - require.NoError(t, err) - - serviceMetric := metrics[0] - assert.Equal(t, serviceMetric.Key, "total_users_in_database") - assert.Equal(t, serviceMetric.Value, "1") - }) - - t.Run("file not found", func(t *testing.T) { - t.Parallel() - conf := &DSGrafanaSqliteDB{ - Enabled: true, - Timeout: time.Second * 10, - DBFile: "/invalid/path/", - } - - grafanaDB := NewDataSourceGrafanaSqliteDB(*conf, logEntry) - - err := grafanaDB.Init(ctx) - assert.Error(t, err, "no such file or directory") - - metrics, err := grafanaDB.FetchMetrics(ctx, *config) - assert.Error(t, err, "temporary grafana database is not initialized") - assert.Nil(t, metrics) - }) -} diff --git a/managed/services/telemetry/datasource_grafanadb_select.go b/managed/services/telemetry/datasource_grafanadb_select.go new file mode 100644 index 0000000000..9361ff93c4 --- /dev/null +++ b/managed/services/telemetry/datasource_grafanadb_select.go @@ -0,0 +1,104 @@ +// Copyright (C) 2023 Percona LLC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package telemetry provides telemetry functionality. +package telemetry + +import ( + "context" + "database/sql" + "net/url" + "time" + + // Events, errors and driver for grafana database. + pmmv1 "github.com/percona-platform/saas/gen/telemetry/events/pmm" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +type dsGrafanaDBSelect struct { + l *logrus.Entry + config DSConfigGrafanaDB + db *sql.DB +} + +// check interfaces. +var ( + _ DataSource = (*dsGrafanaDBSelect)(nil) +) + +// Enabled flag that determines if the data source is enabled. +func (d *dsGrafanaDBSelect) Enabled() bool { + return d.config.Enabled +} + +// NewDsGrafanaDBSelect makes a new data source to collect grafana metrics. +func NewDsGrafanaDBSelect(config DSConfigGrafanaDB, l *logrus.Entry) DataSource { + return &dsGrafanaDBSelect{ + l: l, + config: config, + } +} + +func (d *dsGrafanaDBSelect) Init(ctx context.Context) error { + db, err := openGrafanaDBConnection(d.config, d.l) + if err != nil { + return err + } + + d.db = db + return nil +} + +func openGrafanaDBConnection(config DSConfigGrafanaDB, l *logrus.Entry) (*sql.DB, error) { + var user *url.Userinfo + if config.UseSeparateCredentials { + user = url.UserPassword(config.SeparateCredentials.Username, config.SeparateCredentials.Password) + } else { + user = url.UserPassword(config.Credentials.Username, config.Credentials.Password) + } + uri := url.URL{ + Scheme: config.DSN.Scheme, + User: user, + Host: config.DSN.Host, + Path: config.DSN.DB, + RawQuery: config.DSN.Params, + } + dsn := uri.String() + + db, err := sql.Open("postgres", dsn) + if err != nil { + return nil, errors.Wrap(err, "failed to create a connection pool to PostgreSQL") + } + + db.SetConnMaxIdleTime(time.Second * 30) + db.SetConnMaxLifetime(time.Second * 180) + db.SetMaxIdleConns(1) + db.SetMaxOpenConns(1) + + if err := db.Ping(); err != nil { + l.Warnf("Grafana DB is not reachable [%s]: %s", config.DSN.Host, err) + } + + return db, nil +} + +func (d *dsGrafanaDBSelect) FetchMetrics(ctx context.Context, config Config) ([]*pmmv1.ServerMetric_Metric, error) { + return fetchMetricsFromDB(ctx, d.l, d.config.Timeout, d.db, config) +} + +func (d *dsGrafanaDBSelect) Dispose(ctx context.Context) error { + return d.db.Close() +} diff --git a/managed/services/telemetry/datasource_pmmdb_select.go b/managed/services/telemetry/datasource_pmmdb_select.go index 8633784e7c..09c9249b8c 100644 --- a/managed/services/telemetry/datasource_pmmdb_select.go +++ b/managed/services/telemetry/datasource_pmmdb_select.go @@ -88,7 +88,7 @@ func openPMMDBConnection(config DSConfigPMMDB, l *logrus.Entry) (*sql.DB, error) db.SetMaxOpenConns(1) if err := db.Ping(); err != nil { - l.Warnf("DB is not reachable [%s]: %s", config.DSN.DB, err) + l.Warnf("PMM DB is not reachable at [%s]: %s", config.DSN.Host, err) } return db, nil diff --git a/managed/services/telemetry/datasource_qandb_select.go b/managed/services/telemetry/datasource_qandb_select.go index d0e9049415..b458a0872c 100644 --- a/managed/services/telemetry/datasource_qandb_select.go +++ b/managed/services/telemetry/datasource_qandb_select.go @@ -64,7 +64,7 @@ func openQANDBConnection(dsn string, enabled bool, l *logrus.Entry) (*sql.DB, er return nil, errors.Wrap(err, "Failed to open connection to QAN DB") } if err := db.Ping(); err != nil { - l.Warnf("DB is not reachable [%s]: %s", dsn, err) + l.Warnf("ClickHouse DB is not reachable [%s]: %s", dsn, err) } return db, nil } diff --git a/managed/services/telemetry/datasource_victoria_metrics.go b/managed/services/telemetry/datasource_victoria_metrics.go index 359032aa5c..f1b9e9d931 100644 --- a/managed/services/telemetry/datasource_victoria_metrics.go +++ b/managed/services/telemetry/datasource_victoria_metrics.go @@ -29,7 +29,7 @@ import ( type dataSourceVictoriaMetrics struct { l *logrus.Entry - config DataSourceVictoriaMetrics + config DSConfigVM vm v1.API } @@ -43,7 +43,7 @@ func (d *dataSourceVictoriaMetrics) Enabled() bool { } // NewDataSourceVictoriaMetrics makes new data source for victoria metrics. -func NewDataSourceVictoriaMetrics(config DataSourceVictoriaMetrics, l *logrus.Entry) (DataSource, error) { //nolint:ireturn +func NewDataSourceVictoriaMetrics(config DSConfigVM, l *logrus.Entry) (DataSource, error) { if !config.Enabled { return &dataSourceVictoriaMetrics{ l: l, diff --git a/managed/services/telemetry/datasources.go b/managed/services/telemetry/datasources.go index af9a0c36d7..39eaa3ad61 100644 --- a/managed/services/telemetry/datasources.go +++ b/managed/services/telemetry/datasources.go @@ -52,7 +52,7 @@ func NewDataSourceRegistry(config ServiceConfig, l *logrus.Entry) (DataSourceLoc return nil, err } - grafanaDB := NewDataSourceGrafanaSqliteDB(*config.DataSources.GrafanaDBSelect, l) + grafanaDB := NewDsGrafanaDBSelect(*config.DataSources.GrafanaDBSelect, l) envVars := NewDataSourceEnvVars(*config.DataSources.EnvVars, l) @@ -62,7 +62,7 @@ func NewDataSourceRegistry(config ServiceConfig, l *logrus.Entry) (DataSourceLoc dsVM: vmDB, dsPMMDBSelect: pmmDB, dsQANDBSelect: qanDB, - dsGrafanaDBSelect: grafanaDB, + dsGRAFANADBSelect: grafanaDB, dsEnvVars: envVars, }, }, nil diff --git a/managed/services/telemetry/telemetry.go b/managed/services/telemetry/telemetry.go index 94c17d8a99..d6951ae405 100644 --- a/managed/services/telemetry/telemetry.go +++ b/managed/services/telemetry/telemetry.go @@ -219,9 +219,13 @@ func (s *Service) prepareReport(ctx context.Context) *pmmv1.ServerMetric { // initialize datasources for sourceName, dataSource := range s.dataSourcesMap { + if !dataSource.Enabled() { + s.l.Warnf("Datasource %s is disabled, skipping initialization.", sourceName) + continue + } err := dataSource.Init(ctx) if err != nil { - s.l.Warnf("Telemetry datasource %s init failed: %v", sourceName, err) + s.l.Warnf("Telemetry datasource %s init failed: %s", sourceName, err) continue } initializedDataSources[sourceName] = dataSource @@ -248,11 +252,11 @@ func (s *Service) prepareReport(ctx context.Context) *pmmv1.ServerMetric { // locate DS in initialized state ds := initializedDataSources[DataSourceName(telemetry.Source)] if ds == nil { - s.l.Debugf("cannot find initialized telemetry datasource: %s", telemetry.Source) + s.l.Debugf("Cannot find initialized telemetry datasource: %s", telemetry.Source) continue } if !ds.Enabled() { - s.l.Debugf("datasource %s is disabled", telemetry.Source) + s.l.Debugf("Datasource %s is disabled", telemetry.Source) continue } @@ -263,7 +267,7 @@ func (s *Service) prepareReport(ctx context.Context) *pmmv1.ServerMetric { s.l.Debugf("fetching [%s] took [%s]", telemetry.ID, metricFetchTook) totalTime += metricFetchTook if err != nil { - s.l.Debugf("failed to extract metric from datasource for [%s]:[%s]: %v", telemetry.Source, telemetry.ID, err) + s.l.Debugf("Failed to extract metric from datasource for [%s]:[%s]: %s", telemetry.Source, telemetry.ID, err) continue } @@ -273,7 +277,7 @@ func (s *Service) prepareReport(ctx context.Context) *pmmv1.ServerMetric { telemetryCopy := telemetry // G601: Implicit memory aliasing in for loop. (gosec) metrics, err = transformToJSON(&telemetryCopy, metrics) if err != nil { - s.l.Debugf("failed to transform to JSON: %s", err) + s.l.Debugf("Failed to transform to JSON: %s", err) continue } case StripValuesTransform: @@ -295,14 +299,14 @@ func (s *Service) prepareReport(ctx context.Context) *pmmv1.ServerMetric { for sourceName, dataSource := range initializedDataSources { err := dataSource.Dispose(ctx) if err != nil { - s.l.Debugf("Disposing of %s datasource failed: %v", sourceName, err) + s.l.Debugf("Disposing of %s datasource failed: %s", sourceName, err) continue } } telemetryMetric.Metrics = removeEmpty(telemetryMetric.Metrics) - s.l.Debugf("fetching all metrics took [%s]", totalTime) + s.l.Debugf("Fetching all metrics took [%s]", totalTime) return telemetryMetric } @@ -312,7 +316,7 @@ func (s *Service) locateDataSources(telemetryConfig []Config) map[DataSourceName for _, telemetry := range telemetryConfig { ds, err := s.LocateTelemetryDataSource(telemetry.Source) if err != nil { - s.l.Debugf("failed to lookup telemetry datasource for [%s]:[%s]", telemetry.Source, telemetry.ID) + s.l.Debugf("Failed to lookup telemetry datasource for [%s]:[%s]", telemetry.Source, telemetry.ID) continue } dataSources[DataSourceName(telemetry.Source)] = ds @@ -354,7 +358,7 @@ func (s *Service) makeMetric(ctx context.Context) (*pmmv1.ServerMetric, error) { serverID, err := hex.DecodeString(serverIDToUse) if err != nil { - return nil, errors.Wrapf(err, "failed to decode UUID %q", serverIDToUse) + return nil, errors.Wrapf(err, "failed to decode UUID %s", serverIDToUse) } _, distMethod, _ := s.dus.getDistributionMethodAndOS() @@ -387,7 +391,7 @@ func (s *Service) send(ctx context.Context, report *reporter.ReportRequest) erro s.l.Debugf("Using %s as telemetry host.", s.config.SaasHostname) err = s.portalClient.SendTelemetry(ctx, report) attempt++ - s.l.Debugf("sendV2Request (attempt %d/%d) result: %v", attempt, s.config.Reporting.RetryCount, err) + s.l.Debugf("SendV2Request (attempt %d/%d) result: %s", attempt, s.config.Reporting.RetryCount, err) if err == nil { return nil } diff --git a/managed/services/telemetry/telemetry_test.go b/managed/services/telemetry/telemetry_test.go index 501147eb66..62bbcec44e 100644 --- a/managed/services/telemetry/telemetry_test.go +++ b/managed/services/telemetry/telemetry_test.go @@ -194,7 +194,7 @@ func getServiceConfig(pgPortHost string, qanDSN string, vmDSN string) ServiceCon SendTimeout: time.Second * 10, }, DataSources: DataSources{ - VM: &DataSourceVictoriaMetrics{ + VM: &DSConfigVM{ Enabled: true, Timeout: time.Second * 2, Address: vmDSN, @@ -227,10 +227,28 @@ func getServiceConfig(pgPortHost string, qanDSN string, vmDSN string) ServiceCon Params: "sslmode=disable", }, }, - GrafanaDBSelect: &DSGrafanaSqliteDB{ - Enabled: true, - Timeout: time.Second * 2, - DBFile: "/srv/grafana/grafana.db", + GrafanaDBSelect: &DSConfigGrafanaDB{ + Enabled: true, + Timeout: time.Second * 2, + UseSeparateCredentials: true, + SeparateCredentials: struct { + Username string `yaml:"username"` + Password string `yaml:"password"` + }{ + Username: "grafana", + Password: "grafana", + }, + DSN: struct { + Scheme string + Host string + DB string + Params string + }{ + Scheme: "postgres", + Host: pgPortHost, + DB: "grafana", + Params: "sslmode=disable", + }, }, EnvVars: &DSConfigEnvVars{ Enabled: true, diff --git a/managed/testdata/telemetry/grafana_sqlite.db b/managed/testdata/telemetry/grafana_sqlite.db deleted file mode 100644 index f2a5c296c0..0000000000 Binary files a/managed/testdata/telemetry/grafana_sqlite.db and /dev/null differ