Remove datasource option from SQL module and add tests

metricbeat/docs/modules/sql.asciidoc

@@ -8,7 +8,7 @@ This file is generated! See scripts/mage/docs_collector.go
 
 beta[]
 
-This is the sql module that fetches metrics from a SQL database. You can define driver, datasource and SQL query.
+This is the sql module that fetches metrics from a SQL database. You can define driver and SQL query.
 
 
 
@@ -26,10 +26,9 @@ metricbeat.modules:
   metricsets:
     - query
   period: 10s
-  hosts: ["localhost"]
+  hosts: ["user=myuser password=mypassword dbname=mydb sslmode=disable"]
 
   driver: "postgres"
-  datasource: "user=myuser password=mypassword dbname=mydb sslmode=disable"
   sql_query: "select now()"
 
 ----

x-pack/metricbeat/metricbeat.reference.yml

@@ -965,10 +965,9 @@ metricbeat.modules:
   metricsets:
     - query
   period: 10s
-  hosts: ["localhost"]
+  hosts: ["user=myuser password=mypassword dbname=mydb sslmode=disable"]
 
   driver: "postgres"
-  datasource: "user=myuser password=mypassword dbname=mydb sslmode=disable"
   sql_query: "select now()"
 
 

x-pack/metricbeat/module/sql/_meta/config.yml

@@ -2,9 +2,8 @@
   metricsets:
     - query
   period: 10s
-  hosts: ["localhost"]
+  hosts: ["user=myuser password=mypassword dbname=mydb sslmode=disable"]
 
   driver: "postgres"
-  datasource: "user=myuser password=mypassword dbname=mydb sslmode=disable"
   sql_query: "select now()"
 

x-pack/metricbeat/module/sql/_meta/docs.asciidoc

@@ -1,3 +1,3 @@
-This is the sql module that fetches metrics from a SQL database. You can define driver, datasource and SQL query.
+This is the sql module that fetches metrics from a SQL database. You can define driver and SQL query.
 
 

x-pack/metricbeat/module/sql/docker-compose.yml

@@ -0,0 +1,12 @@
+version: '2.3'
+
+services:
+  mysql:
+    extends:
+      file: ../../../../metricbeat/module/mysql/docker-compose.yml
+      service: mysql
+
+  postgresql:
+    extends:
+      file: ../../../../metricbeat/module/postgresql/docker-compose.yml
+      service: postgresql

x-pack/metricbeat/module/sql/query/_meta/data.json

@@ -1,26 +1,30 @@
 {
-    "@timestamp":"2016-05-23T08:05:34.853Z",
-    "beat":{
-        "hostname":"beathost",
-        "name":"beathost"
+    "@timestamp": "2017-10-12T08:05:34.853Z",
+    "event": {
+        "dataset": "sql.query",
+        "duration": 115000,
+        "module": "sql"
     },
-    "metricset":{
-        "host":"localhost",
-        "module":"sql",
-        "name":"query",
-        "rtt":44269
+    "metricset": {
+        "name": "query",
+        "period": 10000
     },
-    "sql":{
-        "metrics":{
-           "numeric":{
-              "mynumericfield":1
-           },
-           "string":{
-              "mystringfield":"abc"
-           }
+    "service": {
+        "address": "root:test@tcp(172.22.0.3:3306)/",
+        "type": "sql"
+    },
+    "sql": {
+        "driver": "mysql",
+        "metrics": {
+            "numeric": {
+                "table_rows": 6
+            },
+            "string": {
+                "engine": "InnoDB",
+                "table_name": "sys_config",
+                "table_schema": "sys"
+            }
         },
-        "driver":"postgres",
-        "query":"select * from mytable"
-     },
-    "type":"metricsets"
+        "query": "select table_schema, table_name, engine, table_rows from information_schema.tables where table_rows \u003e 0;"
+    }
 }

x-pack/metricbeat/module/sql/query/_meta/data_postgres.json

@@ -0,0 +1,45 @@
+{
+    "@timestamp": "2017-10-12T08:05:34.853Z",
+    "event": {
+        "dataset": "sql.query",
+        "duration": 115000,
+        "module": "sql"
+    },
+    "metricset": {
+        "name": "query",
+        "period": 10000
+    },
+    "service": {
+        "address": "user=postgres password=postgres sslmode=disable host=172.22.0.2 port=5432",
+        "type": "sql"
+    },
+    "sql": {
+        "driver": "postgres",
+        "metrics": {
+            "numeric": {
+                "blk_read_time": 0,
+                "blk_write_time": 0,
+                "blks_hit": 4251,
+                "blks_read": 103,
+                "conflicts": 0,
+                "datid": 12379,
+                "deadlocks": 0,
+                "numbackends": 1,
+                "temp_bytes": 0,
+                "temp_files": 0,
+                "tup_deleted": 0,
+                "tup_fetched": 2847,
+                "tup_inserted": 0,
+                "tup_returned": 4877,
+                "tup_updated": 0,
+                "xact_commit": 49,
+                "xact_rollback": 0
+            },
+            "string": {
+                "datname": "postgres",
+                "stats_reset": "2020-01-20 19:48:28.217"
+            }
+        },
+        "query": "select * from pg_stat_database"
+    }
+}

x-pack/metricbeat/module/sql/query/query.go

@@ -15,6 +15,7 @@ import (
 	"github.com/elastic/beats/libbeat/common"
 	"github.com/elastic/beats/libbeat/common/cfgwarn"
 	"github.com/elastic/beats/metricbeat/mb"
+	"github.com/elastic/beats/metricbeat/mb/parse"
 
 	"github.com/jmoiron/sqlx"
 )
@@ -24,7 +25,9 @@ import (
 // the MetricSet for each host defined in the module's configuration. After the
 // MetricSet has been created then Fetch will begin to be called periodically.
 func init() {
-	mb.Registry.MustAddMetricSet("sql", "query", New)
+	mb.Registry.MustAddMetricSet("sql", "query", New,
+		mb.WithHostParser(parse.PassThruHostParser),
+	)
 }
 
 // MetricSet holds any configuration or state information. It must implement
@@ -33,9 +36,10 @@ func init() {
 // interface methods except for Fetch.
 type MetricSet struct {
 	mb.BaseMetricSet
-	Driver     string
-	Datasource string
-	Query      string
+	Driver string
+	Query  string
+
+	db *sqlx.DB
 }
 
 // New creates a new instance of the MetricSet. New is responsible for unpacking
@@ -44,9 +48,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	cfgwarn.Beta("The sql query metricset is beta.")
 
 	config := struct {
-		Driver     string `config:"driver"`
-		Datasource string `config:"datasource"`
-		Query      string `config:"sql_query"`
+		Driver string `config:"driver"`
+		Query  string `config:"sql_query"`
 	}{}
 
 	if err := base.Module().UnpackConfig(&config); err != nil {
@@ -56,7 +59,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 	return &MetricSet{
 		BaseMetricSet: base,
 		Driver:        config.Driver,
-		Datasource:    config.Datasource,
 		Query:         config.Query,
 	}, nil
 }
@@ -65,7 +67,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 // format. It publishes the event which is then forwarded to the output. In case
 // of an error set the Error field of mb.Event or simply call report.Error().
 func (m *MetricSet) Fetch(report mb.ReporterV2) error {
-	db, err := sqlx.Open(m.Driver, m.Datasource)
+	db, err := sqlx.Open(m.Driver, m.HostData().URI)
 	if err != nil {
 		return errors.Wrap(err, "error opening connection")
 	}

x-pack/metricbeat/module/sql/query/query_integration_test.go

@@ -0,0 +1,89 @@
+// +build integration
+
+package query
+
+import (
+	"fmt"
+	"net"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	// Drivers
+	_ "github.com/go-sql-driver/mysql"
+	_ "github.com/lib/pq"
+
+	"github.com/elastic/beats/libbeat/tests/compose"
+	mbtest "github.com/elastic/beats/metricbeat/mb/testing"
+	"github.com/elastic/beats/metricbeat/module/mysql"
+	"github.com/elastic/beats/metricbeat/module/postgresql"
+)
+
+type testFetchConfig struct {
+	Driver string
+	Query  string
+	Host   string
+}
+
+func TestMySQL(t *testing.T) {
+	service := compose.EnsureUp(t, "mysql")
+	config := testFetchConfig{
+		Driver: "mysql",
+		Query:  "select table_schema, table_name, engine, table_rows from information_schema.tables where table_rows > 0;",
+		Host:   mysql.GetMySQLEnvDSN(service.Host()),
+	}
+
+	t.Run("fetch", func(t *testing.T) {
+		testFetch(t, config)
+	})
+
+	t.Run("data", func(t *testing.T) {
+		testData(t, config, "")
+	})
+}
+
+func TestPostgreSQL(t *testing.T) {
+	service := compose.EnsureUp(t, "postgresql")
+	host, port, err := net.SplitHostPort(service.Host())
+	require.NoError(t, err)
+
+	user := postgresql.GetEnvUsername()
+	password := postgresql.GetEnvPassword()
+
+	config := testFetchConfig{
+		Driver: "postgres",
+		Query:  "select * from pg_stat_database",
+		Host:   fmt.Sprintf("user=%s password=%s sslmode=disable host=%s port=%s", user, password, host, port),
+	}
+
+	t.Run("fetch", func(t *testing.T) {
+		testFetch(t, config)
+	})
+
+	t.Run("data", func(t *testing.T) {
+		testData(t, config, "./_meta/data_postgres.json")
+	})
+}
+
+func testFetch(t *testing.T, cfg testFetchConfig) {
+	m := mbtest.NewFetcher(t, getConfig(cfg))
+	events, errs := m.FetchEvents()
+	require.Empty(t, errs)
+	require.NotEmpty(t, events)
+	t.Logf("%s/%s event: %+v", m.Module().Name(), m.Name(), events[0])
+}
+
+func testData(t *testing.T, cfg testFetchConfig, postfix string) {
+	m := mbtest.NewFetcher(t, getConfig(cfg))
+	m.WriteEvents(t, postfix)
+}
+
+func getConfig(cfg testFetchConfig) map[string]interface{} {
+	return map[string]interface{}{
+		"module":     "sql",
+		"metricsets": []string{"query"},
+		"hosts":      []string{cfg.Host},
+		"driver":     cfg.Driver,
+		"sql_query":  cfg.Query,
+	}
+}

x-pack/metricbeat/modules.d/sql.yml.disabled

@@ -5,9 +5,8 @@
   metricsets:
     - query
   period: 10s
-  hosts: ["localhost"]
+  hosts: ["user=myuser password=mypassword dbname=mydb sslmode=disable"]
 
   driver: "postgres"
-  datasource: "user=myuser password=mypassword dbname=mydb sslmode=disable"
   sql_query: "select now()"