Fix bug when multiple instances were connecting to the same pg instance

Fix #1211

checks.d/postgres.py

@@ -203,6 +203,8 @@ def __init__(self, name, init_config, agentConfig, instances=None):
         self.versions = {}
         self.instance_metrics = {}
         self.bgw_metrics = {}
+        self.db_instance_metrics = []
+        self.db_bgw_metrics = []
 
     def _get_version(self, key, db):
         if key not in self.versions:
@@ -237,7 +239,22 @@ def _get_instance_metrics(self, key, db):
         """
         # Extended 9.2+ metrics if needed
         metrics = self.instance_metrics.get(key)
+
         if metrics is None:
+
+            # Hack to make sure that if we have multiple instances that connect to
+            # the same host, port, we don't collect metrics twice
+            # as it will result in https://github.com/DataDog/dd-agent/issues/1211
+            sub_key = key[:2]
+            if sub_key in self.db_instance_metrics:
+                self.instance_metrics[key] = {}
+                self.log.debug("Not collecting instance metrics for key: {0} as"\
+                    " they are already collected by another instance".format(key))
+                return {}
+
+            self.db_instance_metrics.append(sub_key)
+
+
             if self._is_9_2_or_above(key, db):
                 self.instance_metrics[key] = dict(self.COMMON_METRICS, **self.NEWER_92_METRICS)
             else:
@@ -252,7 +269,21 @@ def _get_bgw_metrics(self, key, db):
         """
         # Extended 9.2+ metrics if needed
         metrics = self.bgw_metrics.get(key)
+
         if metrics is None:
+
+            # Hack to make sure that if we have multiple instances that connect to
+            # the same host, port, we don't collect metrics twice
+            # as it will result in https://github.com/DataDog/dd-agent/issues/1211
+            sub_key = key[:2]
+            if sub_key in self.db_bgw_metrics:
+                self.bgw_metrics[key] = {}
+                self.log.debug("Not collecting bgw metrics for key: {0} as"\
+                    " they are already collected by another instance".format(key))
+                return {}
+
+            self.db_bgw_metrics.append(sub_key)
+
             self.bgw_metrics[key] = dict(self.COMMON_BGW_METRICS)
             if self._is_9_1_or_above(key, db):
                 self.bgw_metrics[key].update(self.NEWER_91_BGW_METRICS)
@@ -452,7 +483,7 @@ def check(self, instance):
         if dbname is None:
             dbname = 'postgres'
 
-        key = '%s:%s:%s' % (host, port, dbname)
+        key = (host, port, dbname)
 
         # Clean up tags in case there was a None entry in the instance
         # e.g. if the yaml contains tags: but no actual tags

tests/test_postgresql.py

@@ -35,10 +35,16 @@ def test_checks(self):
                         },
                         "query": "SELECT datname, %s FROM pg_stat_database WHERE datname = 'datadog_test' LIMIT(1)", 
                         "relation": False,
-                    }
-                ]
+                    }]
+                },
+                {
+                'host': host,
+                'port': port,
+                'username': 'datadog',
+                'password': 'datadog',
+                'dbname': 'postgres'
                 }
-            ]
+                ]
         }
         agentConfig = {
             'version': '0.1',
@@ -51,7 +57,7 @@ def test_checks(self):
         self.check.run()
 
         # FIXME: Not great, should have a function like that available
-        key = '%s:%s:%s' % (host, port, dbname)
+        key = (host, port, dbname)
         db = self.check.dbs[key]
 
         metrics = self.check.get_metrics()
@@ -97,11 +103,11 @@ def test_checks(self):
         service_checks_count = len(service_checks)
         self.assertTrue(type(service_checks) == type([]))
         self.assertTrue(service_checks_count > 0)
-        self.assertEquals(len([sc for sc in service_checks if sc['check'] == "postgres.can_connect"]), 2, service_checks)
+        self.assertEquals(len([sc for sc in service_checks if sc['check'] == "postgres.can_connect"]), 4, service_checks)
         # Assert that all service checks have the proper tags: host, port and db
         self.assertEquals(len([sc for sc in service_checks if "host:localhost" in sc['tags']]), service_checks_count, service_checks)
         self.assertEquals(len([sc for sc in service_checks if "port:%s" % config['instances'][0]['port'] in sc['tags']]), service_checks_count, service_checks)
-        self.assertEquals(len([sc for sc in service_checks if "db:%s" % config['instances'][0]['dbname'] in sc['tags']]), service_checks_count, service_checks)
+        self.assertEquals(len([sc for sc in service_checks if "db:%s" % config['instances'][0]['dbname'] in sc['tags']]), service_checks_count/2, service_checks)
 
         time.sleep(1)
         self.check.run()
@@ -112,7 +118,7 @@ def test_checks(self):
         self.assertEquals(len([m for m in metrics if 'table:persons' in str(m[3].get('tags', [])) ]), 11, metrics)
 
         self.metrics = metrics
-        self.assertMetric("custom.numbackendss")
+        self.assertMetric("custom.numbackends")
 
 if __name__ == '__main__':
     unittest.main()