Serve metrics under a dedicated port (#48)

etcd v3.3 introduced a new flag to allow serving `/metrics` and `/health` under a different port than e.g. `/v2/keys`. This allows us to protect etcd's data via firewall rules but still let monitoring tools to access the monitoring information.

See feature request in etcd repo: etcd-io/etcd#8060.
The implementation landed in v3.3: etcd-io/etcd#8242

This PR instructs etcd to serve metrics and health under the additonal port `2381` *unconditionally* **when the used etcd binary is** `>=v3.3.x`. However, if not explicitely set in the `senza.yaml` this port won't be mapped to the outside and therefore isn't accessible. It doesn't expose more information than anything under `2379` already does.

etcd.py

@@ -37,6 +37,7 @@ class EtcdMember:
     API_VERSION = '/v2/'
     DEFAULT_CLIENT_PORT = 2379
     DEFAULT_PEER_PORT = 2380
+    DEFAULT_METRICS_PORT = 2381
     AG_TAG = 'aws:autoscaling:groupName'
     CF_TAG = 'aws:cloudformation:stack-name'
 
@@ -56,6 +57,7 @@ def __init__(self, arg, region=None):
 
         self.client_port = self.DEFAULT_CLIENT_PORT
         self.peer_port = self.DEFAULT_PEER_PORT
+        self.metrics_port = self.DEFAULT_METRICS_PORT
 
         self.client_urls = []  # these values could be assigned only from the running etcd
         self.peer_urls = []  # cluster by performing http://addr:client_port/v2/members api call
@@ -241,8 +243,9 @@ def delete_member(self, member):
         self.adjust_security_groups('revoke_ingress', member)
         return result
 
-    def etcd_arguments(self, data_dir, initial_cluster, cluster_state):
-        return [
+    def etcd_arguments(self, data_dir, initial_cluster, cluster_state, run_old):
+        # common flags that always have to be set
+        arguments = [
             '-name',
             self.instance_id,
             '--data-dir',
@@ -263,6 +266,20 @@ def etcd_arguments(self, data_dir, initial_cluster, cluster_state):
             cluster_state
         ]
 
+        # this section handles etcd version specific flags
+        etcdversion = os.environ.get('ETCDVERSION_PREV' if run_old else 'ETCDVERSION')
+        if etcdversion:
+            etcdversion = tuple(int(x) for x in etcdversion.split('.'))
+            # etcd >= v3.3: serve metrics on an additonal port
+            if etcdversion >= (3, 3):
+                arguments += [
+                    '-listen-metrics-urls',
+                    'http://0.0.0.0:{}'.format(self.metrics_port),
+                ]
+
+        # return final list of arguments
+        return arguments
+
 
 class EtcdCluster:
     REGIONS = []  # more then one (1) Region if this a Multi-Region-Cluster
@@ -454,7 +471,7 @@ def register_me(self, cluster):
         peers = ','.join(['{}={}'.format(m.instance_id or m.name, m.peer_url) for m in cluster.members
                          if (include_ec2_instances and m.instance_id) or m.peer_urls])
 
-        return self.me.etcd_arguments(self.DATA_DIR, peers, cluster_state)
+        return self.me.etcd_arguments(self.DATA_DIR, peers, cluster_state, self.run_old)
 
     def run(self):
         cluster = EtcdCluster(self)