PMM-12185 migrate telemetry GrafanaDS to using PostgreSQL (#2542)

* PMM-12185 migrate telemetry GrafanaDS to using PostgreSQL

* PMM-12185 configure GrafanaDB connection

* PMM-12185 fix grafanaDB SQL stmnts

* PMM-12185 don't init when ds is disabled

* PMM-12185 cleanup the code

* PMM-12185 update go modules

* PMM-12185 Apply suggestions from code review

Co-authored-by: Nurlan Moldomurov <nurlan.moldomurov@percona.com>

* PMM-12185 remove the debug stmnt

* PMM-12185 remove unused lint suppressions

* PMM-12185 fix inconsistent log output

* PMM-12185 trim trailing spaces

* PMM-12185 rename VM config type for consistency sake

---------

Co-authored-by: Nurlan Moldomurov <nurlan.moldomurov@percona.com>

go.mod

@@ -92,7 +92,6 @@ require (
 	k8s.io/cli-runtime v0.28.0
 	k8s.io/client-go v0.28.0
 	k8s.io/kubectl v0.28.0
-	modernc.org/sqlite v1.26.0
 	sigs.k8s.io/controller-runtime v0.14.6
 )
 
@@ -114,11 +113,11 @@ require (
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/hashicorp/go-uuid v1.0.2 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
-	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/mattn/go-ieproxy v0.0.1 // indirect
 	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/minio/minio-go v6.0.14+incompatible // indirect
@@ -129,7 +128,6 @@ require (
 	github.com/percona/percona-backup-mongodb v1.8.1 // indirect
 	github.com/pierrec/lz4 v2.6.1+incompatible // indirect
 	github.com/posener/complete v1.2.3 // indirect
-	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/riywo/loginshell v0.0.0-20200815045211-7d26008be1ab // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
@@ -143,15 +141,6 @@ require (
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
 	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
-	lukechampine.com/uint128 v1.2.0 // indirect
-	modernc.org/cc/v3 v3.40.0 // indirect
-	modernc.org/ccgo/v3 v3.16.13 // indirect
-	modernc.org/libc v1.24.1 // indirect
-	modernc.org/mathutil v1.5.0 // indirect
-	modernc.org/memory v1.6.0 // indirect
-	modernc.org/opt v0.1.3 // indirect
-	modernc.org/strutil v1.1.3 // indirect
-	modernc.org/token v1.0.1 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect

go.sum

@@ -486,8 +486,6 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
 github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
-github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
-github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
@@ -712,9 +710,6 @@ github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwa
 github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
 github.com/ramr/go-reaper v0.2.1 h1:zww+wlQOvTjBZuk1920R/e0GFEb6O7+B0WQLV6dM924=
 github.com/ramr/go-reaper v0.2.1/go.mod h1:AVypdzrcCXjSc/JYnlXl8TsB+z84WyFzxWE8Jh0MOJc=
-github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
-github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
-github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -1233,34 +1228,6 @@ k8s.io/kubectl v0.28.0 h1:qhfju0OaU+JGeBlToPeeIg2UJUWP++QwTkpio6nlPKg=
 k8s.io/kubectl v0.28.0/go.mod h1:1We+E5nSX3/TVoSQ6y5Bzld5OhTBHZHlKEYl7g/NaTk=
 k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk=
 k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI=
-lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk=
-modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw=
-modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0=
-modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw=
-modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY=
-modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk=
-modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
-modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
-modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
-modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM=
-modernc.org/libc v1.24.1/go.mod h1:FmfO1RLrU3MHJfyi9eYYmZBfi/R+tqZ6+hQ3yQQUkak=
-modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
-modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
-modernc.org/memory v1.6.0 h1:i6mzavxrE9a30whzMfwf7XWVODx2r5OYXvU46cirX7o=
-modernc.org/memory v1.6.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
-modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
-modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
-modernc.org/sqlite v1.26.0 h1:SocQdLRSYlA8W99V8YH0NES75thx19d9sB/aFc4R8Lw=
-modernc.org/sqlite v1.26.0/go.mod h1:FL3pVXie73rg3Rii6V/u5BoHlSoyeZeIgKZEgHARyCU=
-modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY=
-modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw=
-modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY=
-modernc.org/tcl v1.15.2/go.mod h1:3+k/ZaEbKrC8ePv8zJWPtBSW0V7Gg9g8rkmhI1Kfs3c=
-modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg=
-modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
-modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY=
-modernc.org/z v1.7.3/go.mod h1:Ipv4tsdxZRbQyLq9Q1M6gdbkxYzdlrciF2Hi/lS7nWE=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=

managed/cmd/pmm-managed/main.go

@@ -779,6 +779,12 @@ func main() { //nolint:cyclop,maintidx
 	}
 	pmmdb.DSN.Params = q.Encode()
 
+	grafanadb := ds.GrafanaDBSelect
+	grafanadb.DSN.Scheme = "postgres"
+	grafanadb.DSN.Host = *postgresAddrF
+	grafanadb.DSN.DB = "grafana"
+	grafanadb.DSN.Params = q.Encode()
+
 	clickhouseDSN := "tcp://" + *clickhouseAddrF + "/" + *clickHouseDatabaseF
 
 	qanDB := ds.QanDBSelect

managed/services/config/pmm-managed.yaml

@@ -19,7 +19,10 @@ services:
       GRAFANADB_SELECT:
         enabled: true
         timeout: 5s
-        db_file: /srv/grafana/grafana.db
+        use_separate_credentials: true
+        separate_credentials:
+          username: grafana
+          password: grafana
       ENV_VARS:
         enabled: true
     reporting:

managed/services/telemetry/config.default.yml

@@ -362,7 +362,7 @@ telemetry:
 
   - id: GrafanaStatDailyActiveUsers
     source: GRAFANADB_SELECT
-    query: count(*) AS count FROM user WHERE last_seen_at > datetime('now', '-1 day')
+    query: count(*) AS count FROM public.user WHERE last_seen_at > now() - interval '1 day'
     summary: "Daily active users"
     data:
       - metric_name: "pmm_server_grafana_stat_daily_active_users"
@@ -761,47 +761,47 @@ telemetry:
   #Grafana
   - id: GrafanaUsersCount
     source: GRAFANADB_SELECT
-    query: count(*) AS count FROM user
+    query: count(*) AS count FROM public.user
     summary: "Grafana Users Count"
     data:
       - metric_name: "grafana_users_count"
         column: "count"
 
   - id: GrafanaDarkThemeUsersCount
     source: GRAFANADB_SELECT
-    query: count(u.id) as count from user u left join preferences p on p.user_id = u.id where p.theme = 'dark'
+    query: count(u.id) as count from public.user u left join preferences p on p.user_id = u.id where p.theme = 'dark'
     summary: "Grafana Dark Theme Users Count"
     data:
       - metric_name: "grafana_dark_theme_users_count"
         column: "count"
 
   - id: GrafanaLightThemeUsersCount
     source: GRAFANADB_SELECT
-    query: count(u.id) as count from user u left join preferences p on p.user_id = u.id where p.theme = 'light'
+    query: count(u.id) as count from public.user u left join preferences p on p.user_id = u.id where p.theme = 'light'
     summary: "Grafana Light Theme Users Count"
     data:
       - metric_name: "grafana_light_theme_users_count"
         column: "count"
 
   - id: GrafanaDefaultThemeUsersCount
     source: GRAFANADB_SELECT
-    query: count(u.id) as count from user u left join preferences p on p.user_id = u.id where p.theme = '' or p.theme is null
+    query: count(u.id) as count from public.user u left join preferences p on p.user_id = u.id where p.theme = '' or p.theme is null
     summary: "Grafana Default Theme Users Count"
     data:
       - metric_name: "grafana_default_theme_users_count"
         column: "count"
 
   - id: GrafanaCustomDashboardsCount
     source: GRAFANADB_SELECT
-    query: count(*) as count from dashboard where created_by != -1 and is_folder = 0
+    query: count(*) as count from dashboard where created_by != -1 and is_folder = false
     summary: "Grafana Custom Dashboards Count"
     data:
       - metric_name: "grafana_custom_dashboards_count"
         column: "count"
 
   - id: GrafanaCustomDashboardsPerPillar
     source: GRAFANADB_SELECT
-    query: inn.title as pillar, count(d.title) as count from dashboard d left join dashboard inn ON d.folder_id =  inn.id where d.created_by != -1 and d.is_folder = 0 group by inn.title
+    query: inn.title as pillar, count(d.title) as count from dashboard d left join dashboard inn ON d.folder_id = inn.id where d.created_by != -1 and d.is_folder = false group by inn.title
     summary: "Grafana Custom Dashboards Count By Pillar"
     transform:
       type: JSON
@@ -810,6 +810,7 @@ telemetry:
       - metric_name: "pillar"
         column: "pillar"
       - metric_name: "count"
+        column: "count"
 
   # API usage
   - id: APIUsageMetrics

managed/services/telemetry/config.go

@@ -41,17 +41,17 @@ const (
 	dsVM              = DataSourceName("VM")
 	dsQANDBSelect     = DataSourceName("QANDB_SELECT")
 	dsPMMDBSelect     = DataSourceName("PMMDB_SELECT")
-	dsGrafanaDBSelect = DataSourceName("GRAFANADB_SELECT")
+	dsGRAFANADBSelect = DataSourceName("GRAFANADB_SELECT")
 	dsEnvVars         = DataSourceName("ENV_VARS")
 )
 
 // DataSources holds all possible data source types.
 type DataSources struct {
-	VM              *DataSourceVictoriaMetrics `yaml:"VM"`
-	QanDBSelect     *DSConfigQAN               `yaml:"QANDB_SELECT"`
-	PmmDBSelect     *DSConfigPMMDB             `yaml:"PMMDB_SELECT"`
-	GrafanaDBSelect *DSGrafanaSqliteDB         `yaml:"GRAFANADB_SELECT"`
-	EnvVars         *DSConfigEnvVars           `yaml:"ENV_VARS"`
+	VM              *DSConfigVM        `yaml:"VM"`
+	QanDBSelect     *DSConfigQAN       `yaml:"QANDB_SELECT"`
+	PmmDBSelect     *DSConfigPMMDB     `yaml:"PMMDB_SELECT"`
+	GrafanaDBSelect *DSConfigGrafanaDB `yaml:"GRAFANADB_SELECT"`
+	EnvVars         *DSConfigEnvVars   `yaml:"ENV_VARS"`
 }
 
 // ServiceConfig telemetry config.
@@ -76,32 +76,25 @@ type DSConfigQAN struct {
 	DSN     string        `yaml:"-"`
 }
 
-// DataSourceVictoriaMetrics telemetry config.
-type DataSourceVictoriaMetrics struct {
+// DSConfigVM telemetry config.
+type DSConfigVM struct {
 	Enabled bool          `yaml:"enabled"`
 	Timeout time.Duration `yaml:"timeout"`
 	Address string        `yaml:"address"`
 }
 
-// DSGrafanaSqliteDB telemetry config.
-type DSGrafanaSqliteDB struct {
-	Enabled bool          `yaml:"enabled"`
-	Timeout time.Duration `yaml:"timeout"`
-	DBFile  string        `yaml:"db_file"`
-}
-
 // DSConfigPMMDB telemetry config.
 type DSConfigPMMDB struct { //nolint:musttag
 	Enabled                bool          `yaml:"enabled"`
 	Timeout                time.Duration `yaml:"timeout"`
 	UseSeparateCredentials bool          `yaml:"use_separate_credentials"`
-	// Credentials used by PMM
-	DSN struct {
+	DSN                    struct {
 		Scheme string
 		Host   string
 		DB     string
 		Params string
 	} `yaml:"-"`
+	// Credentials used by PMM
 	Credentials struct {
 		Username string
 		Password string
@@ -112,11 +105,15 @@ type DSConfigPMMDB struct { //nolint:musttag
 	} `yaml:"separate_credentials"`
 }
 
+// DSConfigGrafanaDB is a Grafana telemetry config.
+type DSConfigGrafanaDB DSConfigPMMDB
+
+// DSConfigEnvVars is an env variable telemetry config.
 type DSConfigEnvVars struct {
 	Enabled bool `yaml:"enabled"`
 }
 
-// Config is a telemetry config.
+// Config telemetry config.
 type Config struct {
 	ID        string           `yaml:"id"`
 	Source    string           `yaml:"source"`
@@ -139,7 +136,7 @@ type ConfigTransformType string
 const (
 	// JSONTransform converts multiple metrics in one formatted as JSON.
 	JSONTransform = ConfigTransformType("JSON")
-	// StripValuesTransform strips values from metrics, replacing them with 0/1 depending on presence.
+	// StripValuesTransform strips values from metrics, replacing them with 1 to indicate presence.
 	StripValuesTransform = ConfigTransformType("StripValues")
 )
 

managed/services/telemetry/config_test.go

@@ -47,10 +47,12 @@ datasources:
   GRAFANADB_SELECT:
     enabled: true
     timeout: 2s
-    db_file: /srv/grafana/grafana.db
+    use_separate_credentials: true
+    separate_credentials:
+      username: grafana
+      password: grafana
   ENV_VARS:
     enabled: true
-
 reporting:
   send: true
   send_on_start: true
@@ -74,7 +76,7 @@ reporting:
 			SendTimeout:  time.Second * 10,
 		},
 		DataSources: DataSources{
-			VM: &DataSourceVictoriaMetrics{
+			VM: &DSConfigVM{
 				Enabled: true,
 				Timeout: time.Second * 2,
 				Address: "http://localhost:80/victoriametrics/",
@@ -95,10 +97,17 @@ reporting:
 					Password: "pmm-managed",
 				},
 			},
-			GrafanaDBSelect: &DSGrafanaSqliteDB{
-				Enabled: true,
-				Timeout: time.Second * 2,
-				DBFile:  "/srv/grafana/grafana.db",
+			GrafanaDBSelect: &DSConfigGrafanaDB{
+				Enabled:                true,
+				Timeout:                time.Second * 2,
+				UseSeparateCredentials: true,
+				SeparateCredentials: struct {
+					Username string `yaml:"username"`
+					Password string `yaml:"password"`
+				}{
+					Username: "grafana",
+					Password: "grafana",
+				},
 			},
 			EnvVars: &DSConfigEnvVars{
 				Enabled: true,