Add username/pass options for PostgreSQL (elastic#2890)

Similar to elastic#2889 but for PostgreSQL. Also adds docs to the Postgres module,
which were missing, and adjusted the integration tests to use the username
option instead of the full URL.
(cherry picked from commit f0b52e1)

CHANGELOG.asciidoc

@@ -48,6 +48,13 @@ https://github.com/elastic/beats/compare/v5.0.0...5.0[Check the HEAD diff]
 
 *Metricbeat*
 
+- Add experimental filebeat metricset in the beats module. {pull}2297[2297]
+- Add experimental libbeat metricset in the beats module. {pull}2339[2339]
+- Add experimental docker module. Provided by Ingensi and @douaejeouit based on dockbeat.
+- Add username and password config options to the MongoDB module. {pull}2889[2889]
+- Add username and password config options to the PostgreSQL module. {pull}2889[2890]
+- Add system core metricset for Windows. {pull}2883}[2883]
+
 *Packetbeat*
 
 *Topbeat*

metricbeat/docker-compose.yml

@@ -21,9 +21,10 @@ beat:
     - MYSQL_DSN=root:test@tcp(mysql:3306)/
     - MYSQL_HOST=mysql
     - MYSQL_PORT=3306
-    - POSTGRESQL_DSN=postgres://postgres@postgresql:5432?sslmode=disable
+    - POSTGRESQL_DSN=postgres://postgresql:5432?sslmode=disable
     - POSTGRESQL_HOST=postgresql
     - POSTGRESQL_PORT=5432
+    - POSTGRESQL_USERNAME=postgres
     - ZOOKEEPER_HOST=zookeeper
     - ZOOKEEPER_PORT=2181
     - HAPROXY_HOST=haproxy

metricbeat/docs/modules/postgresql.asciidoc

@@ -3,10 +3,56 @@ This file is generated! See scripts/docs_collector.py
 ////
 
 [[metricbeat-module-postgresql]]
-== postgresql Module
+== PostgreSQL Module
 
-This is the postgresql Module.
+This module periodically fetches metrics from
+https://www.postgresql.org/[PostgreSQL] servers.
 
+[float]
+=== Module-Specific Configuration Notes
+
+When configuring the `hosts` option, you must use Postgres URLs of the following
+format:
+
+-----------------------------------
+[postgres://][user:pass@]host[:port][?options]
+-----------------------------------
+
+The URL can be as simple as:
+
+[source,yaml]
+----------------------------------------------------------------------
+- module: postgresql
+  hosts: ["postgres://localhost"]
+----------------------------------------------------------------------
+
+Or more complex like:
+
+[source,yaml]
+----------------------------------------------------------------------
+- module: postgresql
+  hosts: ["postgres://localhost:40001?sslmode=disable", "postgres://otherhost:40001"]
+----------------------------------------------------------------------
+
+WARNING: In case you use username and password in the hosts array, this
+information will be sent with each event as part of the `metricset.host` field.
+To prevent sending username and password the config options `username` and
+`password` can be used.
+
+[source,yaml]
+----
+- module: postgresql
+  metricsets: ["status"]
+  hosts: ["postgres://localhost:5432"]
+  username: root
+  password: test
+----
+
+[float]
+=== Compatibility
+
+This module was tested with PostgreSQL 9.5.3 and is expected to work with all
+versions >= 9.
 
 
 [float]
@@ -33,10 +79,20 @@ metricbeat.modules:
   #period: 10s
 
   # The host must be passed as PostgreSQL DSN. Example:
-  # postgres://pqgotest:password@localhost:5432?sslmode=disable
+  # postgres://localhost:5432?sslmode=disable
   # The available parameters are documented here:
   # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters
-  #hosts: ["postgres://postgres@localhost:5432"]
+  #
+  # Warning: specifying the user/password in the hosts array is possible
+  # but it will result in the password being present in the output documents.
+  # We recommend using the username and password options instead.
+  #hosts: ["postgres://localhost:5432"]
+
+  # Username to use when connecting to PostgreSQL. Empty by default.
+  #username: user
+
+  # Password to use when connecting to PostgreSQL. Empty by default.
+  #password: pass
 
 ----
 

metricbeat/etc/beat.full.yml

@@ -129,10 +129,20 @@ metricbeat.modules:
   #period: 10s
 
   # The host must be passed as PostgreSQL DSN. Example:
-  # postgres://pqgotest:password@localhost:5432?sslmode=disable
+  # postgres://localhost:5432?sslmode=disable
   # The available parameters are documented here:
   # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters
-  #hosts: ["postgres://postgres@localhost:5432"]
+  #
+  # Warning: specifying the user/password in the hosts array is possible
+  # but it will result in the password being present in the output documents.
+  # We recommend using the username and password options instead.
+  #hosts: ["postgres://localhost:5432"]
+
+  # Username to use when connecting to PostgreSQL. Empty by default.
+  #username: user
+
+  # Password to use when connecting to PostgreSQL. Empty by default.
+  #password: pass
 
 
 #-------------------------------- Redis Module -------------------------------

metricbeat/metricbeat.full.yml

@@ -129,10 +129,20 @@ metricbeat.modules:
   #period: 10s
 
   # The host must be passed as PostgreSQL DSN. Example:
-  # postgres://pqgotest:password@localhost:5432?sslmode=disable
+  # postgres://localhost:5432?sslmode=disable
   # The available parameters are documented here:
   # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters
-  #hosts: ["postgres://postgres@localhost:5432"]
+  #
+  # Warning: specifying the user/password in the hosts array is possible
+  # but it will result in the password being present in the output documents.
+  # We recommend using the username and password options instead.
+  #hosts: ["postgres://localhost:5432"]
+
+  # Username to use when connecting to PostgreSQL. Empty by default.
+  #username: user
+
+  # Password to use when connecting to PostgreSQL. Empty by default.
+  #password: pass
 
 
 #-------------------------------- Redis Module -------------------------------

metricbeat/module/postgresql/_meta/config.yml

@@ -13,8 +13,18 @@
   #period: 10s
 
   # The host must be passed as PostgreSQL DSN. Example:
-  # postgres://pqgotest:password@localhost:5432?sslmode=disable
+  # postgres://localhost:5432?sslmode=disable
   # The available parameters are documented here:
   # https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters
-  #hosts: ["postgres://postgres@localhost:5432"]
+  #
+  # Warning: specifying the user/password in the hosts array is possible
+  # but it will result in the password being present in the output documents.
+  # We recommend using the username and password options instead.
+  #hosts: ["postgres://localhost:5432"]
+
+  # Username to use when connecting to PostgreSQL. Empty by default.
+  #username: user
+
+  # Password to use when connecting to PostgreSQL. Empty by default.
+  #password: pass
 

metricbeat/module/postgresql/_meta/docs.asciidoc

@@ -1,4 +1,50 @@
-== postgresql Module
+== PostgreSQL Module
 
-This is the postgresql Module.
+This module periodically fetches metrics from
+https://www.postgresql.org/[PostgreSQL] servers.
 
+[float]
+=== Module-Specific Configuration Notes
+
+When configuring the `hosts` option, you must use Postgres URLs of the following
+format:
+
+-----------------------------------
+[postgres://][user:pass@]host[:port][?options]
+-----------------------------------
+
+The URL can be as simple as:
+
+[source,yaml]
+----------------------------------------------------------------------
+- module: postgresql
+  hosts: ["postgres://localhost"]
+----------------------------------------------------------------------
+
+Or more complex like:
+
+[source,yaml]
+----------------------------------------------------------------------
+- module: postgresql
+  hosts: ["postgres://localhost:40001?sslmode=disable", "postgres://otherhost:40001"]
+----------------------------------------------------------------------
+
+WARNING: In case you use username and password in the hosts array, this
+information will be sent with each event as part of the `metricset.host` field.
+To prevent sending username and password the config options `username` and
+`password` can be used.
+
+[source,yaml]
+----
+- module: postgresql
+  metricsets: ["status"]
+  hosts: ["postgres://localhost:5432"]
+  username: root
+  password: test
+----
+
+[float]
+=== Compatibility
+
+This module was tested with PostgreSQL 9.5.3 and is expected to work with all
+versions >= 9.

metricbeat/module/postgresql/activity/activity.go

@@ -22,29 +22,43 @@ func init() {
 // MetricSet type defines all fields of the Postgresql MetricSet
 type MetricSet struct {
 	mb.BaseMetricSet
+	connectionString string
 }
 
 // New create a new instance of the MetricSet
 // Part of new is also setting up the configuration by processing additional
 // configuration entries if needed.
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
-	config := struct{}{}
+	config := struct {
+		Hosts    []string `config:"hosts"    validate:"nonzero,required"`
+		Username string   `config:"username"`
+		Password string   `config:"password"`
+	}{
+		Username: "",
+		Password: "",
+	}
 
 	if err := base.Module().UnpackConfig(&config); err != nil {
 		return nil, err
 	}
 
+	url, err := postgresql.ParseURL(base.Host(), config.Username, config.Password,
+		base.Module().Config().Timeout)
+	if err != nil {
+		return nil, err
+	}
+
 	return &MetricSet{
-		BaseMetricSet: base,
+		BaseMetricSet:    base,
+		connectionString: url,
 	}, nil
 }
 
 // Fetch implements the data gathering and data conversion to the right format.
 func (m *MetricSet) Fetch() ([]common.MapStr, error) {
 
-	// TODO: Find a way to pass the timeout
-	db, err := sql.Open("postgres", m.Host())
+	db, err := sql.Open("postgres", m.connectionString)
 	if err != nil {
 		return nil, err
 	}

metricbeat/module/postgresql/activity/activity_integration_test.go

@@ -52,5 +52,7 @@ func getConfig() map[string]interface{} {
 		"module":     "postgresql",
 		"metricsets": []string{"activity"},
 		"hosts":      []string{postgresql.GetEnvDSN()},
+		"username":   postgresql.GetEnvUsername(),
+		"password":   postgresql.GetEnvPassword(),
 	}
 }

metricbeat/module/postgresql/bgwriter/bgwriter.go

@@ -23,26 +23,40 @@ func init() {
 // MetricSet type defines all fields of the MetricSet
 type MetricSet struct {
 	mb.BaseMetricSet
+	connectionString string
 }
 
 // New create a new instance of the MetricSet
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
-
-	config := struct{}{}
+	config := struct {
+		Hosts    []string `config:"hosts"    validate:"nonzero,required"`
+		Username string   `config:"username"`
+		Password string   `config:"password"`
+	}{
+		Username: "",
+		Password: "",
+	}
 
 	if err := base.Module().UnpackConfig(&config); err != nil {
 		return nil, err
 	}
 
+	url, err := postgresql.ParseURL(base.Host(), config.Username, config.Password,
+		base.Module().Config().Timeout)
+	if err != nil {
+		return nil, err
+	}
+
 	return &MetricSet{
-		BaseMetricSet: base,
+		BaseMetricSet:    base,
+		connectionString: url,
 	}, nil
 }
 
 // Fetch methods implements the data gathering and data conversion to the right format
 func (m *MetricSet) Fetch() (common.MapStr, error) {
 
-	db, err := sql.Open("postgres", m.Host())
+	db, err := sql.Open("postgres", m.connectionString)
 	if err != nil {
 		return nil, err
 	}

metricbeat/module/postgresql/bgwriter/bgwriter_integration_test.go

@@ -54,5 +54,7 @@ func getConfig() map[string]interface{} {
 		"module":     "postgresql",
 		"metricsets": []string{"bgwriter"},
 		"hosts":      []string{postgresql.GetEnvDSN()},
+		"username":   postgresql.GetEnvUsername(),
+		"password":   postgresql.GetEnvPassword(),
 	}
 }

metricbeat/module/postgresql/database/database.go

@@ -22,26 +22,41 @@ func init() {
 // MetricSet type defines all fields of the MetricSet
 type MetricSet struct {
 	mb.BaseMetricSet
+	connectionString string
 }
 
 // New create a new instance of the MetricSet
 func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
-	config := struct{}{}
+	config := struct {
+		Hosts    []string `config:"hosts"    validate:"nonzero,required"`
+		Username string   `config:"username"`
+		Password string   `config:"password"`
+	}{
+		Username: "",
+		Password: "",
+	}
 
 	if err := base.Module().UnpackConfig(&config); err != nil {
 		return nil, err
 	}
 
+	url, err := postgresql.ParseURL(base.Host(), config.Username, config.Password,
+		base.Module().Config().Timeout)
+	if err != nil {
+		return nil, err
+	}
+
 	return &MetricSet{
-		BaseMetricSet: base,
+		BaseMetricSet:    base,
+		connectionString: url,
 	}, nil
 }
 
 // Fetch methods implements the data gathering and data conversion to the right format
 func (m *MetricSet) Fetch() ([]common.MapStr, error) {
 
-	db, err := sql.Open("postgres", m.Host())
+	db, err := sql.Open("postgres", m.connectionString)
 	if err != nil {
 		return nil, err
 	}

metricbeat/module/postgresql/database/database_integration_test.go

@@ -54,5 +54,7 @@ func getConfig() map[string]interface{} {
 		"module":     "postgresql",
 		"metricsets": []string{"database"},
 		"hosts":      []string{postgresql.GetEnvDSN()},
+		"username":   postgresql.GetEnvUsername(),
+		"password":   postgresql.GetEnvPassword(),
 	}
 }