Vendorized prometheus_client 0.20.0.post1

Signed-off-by: Andreas Maier <maiera@de.ibm.com>

Author: andy-maier

Makefile

@@ -100,6 +100,11 @@ package_dir := $(package_name)
 package_py_files := \
     $(wildcard $(package_dir)/*.py) \
     $(wildcard $(package_dir)/*/*.py) \
+    $(wildcard $(package_dir)/*/*/*.py) \
+    $(wildcard $(package_dir)/*/*/*/*.py) \
+
+src_py_files := \
+    $(wildcard $(package_dir)/*.py) \
 
 test_dir := tests
 test_py_files := \
@@ -237,14 +242,14 @@ develop: $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done
 .PHONY: check
 check: $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done
 	@echo "Makefile: Performing flake8 checks with PACKAGE_LEVEL=$(PACKAGE_LEVEL)"
-	flake8 --config .flake8 $(package_py_files) $(test_py_files) setup.py $(doc_dir)/conf.py
+	flake8 --config .flake8 $(src_py_files) $(test_py_files) setup.py $(doc_dir)/conf.py
 	@echo "Makefile: Done performing flake8 checks"
 	@echo "Makefile: $@ done."
 
 .PHONY: pylint
 pylint: $(done_dir)/develop_$(pymn)_$(PACKAGE_LEVEL).done
 	@echo "Makefile: Performing pylint checks with PACKAGE_LEVEL=$(PACKAGE_LEVEL)"
-	pylint --rcfile=.pylintrc --disable=fixme $(package_py_files) $(test_py_files) setup.py $(doc_dir)/conf.py
+	pylint --rcfile=.pylintrc --disable=fixme $(src_py_files) $(test_py_files) setup.py $(doc_dir)/conf.py
 	@echo "Makefile: Done performing pylint checks"
 	@echo "Makefile: $@ done."
 

docs/changes.rst

@@ -42,8 +42,8 @@ Released: not yet
 
 * Fixed AttributeError when using 'storage_groups' on 'Client' object.
 
-* Used a forked version 0.20.0.post1 of the 'prometheus_client' package to pick
-  up the following fixes:
+* Vendorized a forked version 0.20.0.post1 of the 'prometheus_client' package
+  to pick up the following fixes:
 
   - Fixed HTTP verb tampering test failures. (issue #494)
   - Fixed vulnerabilities in Prometheus server detected by testssl.sh.

requirements.txt

@@ -14,8 +14,8 @@ zhmcclient>=1.14.0
 # prometheus-client 0.20.0.post1 (on forked repo) adds the following PRs:
 # - Removed CBC ciphers to address CVE-2013-0169 (LUCKY13) (PR https://github.com/prometheus/client_python/pull/1051)
 # - Reject invalid HTTP methods and resources (PR https://github.com/prometheus/client_python/pull/1019)
+# For now, 0.20.0.post1 has been vendorized.
 # TODO: Use official prometheus-client version (0.21.0 ?) with these PRs once released.
-prometheus-client @ git+https://github.com/andy-maier/client_python.git@release_0.20.0.post1
 # prometheus-client>=0.21.0
 
 urllib3>=1.26.19

setup.py

@@ -82,7 +82,12 @@ def read_file(a_file):
     name='zhmc_prometheus_exporter',
     version=package_version,
     packages=[
-        'zhmc_prometheus_exporter'
+        'zhmc_prometheus_exporter',
+        'zhmc_prometheus_exporter.vendor',
+        'zhmc_prometheus_exporter.vendor.prometheus_client',
+        'zhmc_prometheus_exporter.vendor.prometheus_client.bridge',
+        'zhmc_prometheus_exporter.vendor.prometheus_client.openmetrics',
+        'zhmc_prometheus_exporter.vendor.prometheus_client.twisted',
     ],
     package_data={
         'zhmc_prometheus_exporter': ['schemas/*.yaml'],

tests/test_all.py

@@ -28,8 +28,8 @@
 import pytest
 import zhmcclient
 import zhmcclient_mock
-import prometheus_client
 
+from zhmc_prometheus_exporter.vendor import prometheus_client
 from zhmc_prometheus_exporter import zhmc_prometheus_exporter
 
 

zhmc_prometheus_exporter/vendor/__init__.py

@@ -0,0 +1,4 @@
+# Versions of the vendored packages:
+
+# prometheus_client: https://github.com/andy-maier/client_python/releases/tag/0.20.0.post1
+prometheus_client_version = '0.20.0.post1'

zhmc_prometheus_exporter/vendor/prometheus_client/__init__.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+
+from . import (
+    exposition, gc_collector, metrics, metrics_core, platform_collector,
+    process_collector, registry,
+)
+from .exposition import (
+    CONTENT_TYPE_LATEST, delete_from_gateway, generate_latest,
+    instance_ip_grouping_key, make_asgi_app, make_wsgi_app, MetricsHandler,
+    push_to_gateway, pushadd_to_gateway, start_http_server, start_wsgi_server,
+    write_to_textfile,
+)
+from .gc_collector import GC_COLLECTOR, GCCollector
+from .metrics import (
+    Counter, disable_created_metrics, enable_created_metrics, Enum, Gauge,
+    Histogram, Info, Summary,
+)
+from .metrics_core import Metric
+from .platform_collector import PLATFORM_COLLECTOR, PlatformCollector
+from .process_collector import PROCESS_COLLECTOR, ProcessCollector
+from .registry import CollectorRegistry, REGISTRY
+
+__all__ = (
+    'CollectorRegistry',
+    'REGISTRY',
+    'Metric',
+    'Counter',
+    'Gauge',
+    'Summary',
+    'Histogram',
+    'Info',
+    'Enum',
+    'enable_created_metrics',
+    'disable_created_metrics',
+    'CONTENT_TYPE_LATEST',
+    'generate_latest',
+    'MetricsHandler',
+    'make_wsgi_app',
+    'make_asgi_app',
+    'start_http_server',
+    'start_wsgi_server',
+    'write_to_textfile',
+    'push_to_gateway',
+    'pushadd_to_gateway',
+    'delete_from_gateway',
+    'instance_ip_grouping_key',
+    'ProcessCollector',
+    'PROCESS_COLLECTOR',
+    'PlatformCollector',
+    'PLATFORM_COLLECTOR',
+    'GCCollector',
+    'GC_COLLECTOR',
+)
+
+if __name__ == '__main__':
+    c = Counter('cc', 'A counter')
+    c.inc()
+
+    g = Gauge('gg', 'A gauge')
+    g.set(17)
+
+    s = Summary('ss', 'A summary', ['a', 'b'])
+    s.labels('c', 'd').observe(17)
+
+    h = Histogram('hh', 'A histogram')
+    h.observe(.6)
+
+    start_http_server(8000)
+    import time
+
+    while True:
+        time.sleep(1)

zhmc_prometheus_exporter/vendor/prometheus_client/asgi.py

@@ -0,0 +1,40 @@
+from typing import Callable
+from urllib.parse import parse_qs
+
+from .exposition import _bake_output
+from .registry import CollectorRegistry, REGISTRY
+
+
+def make_asgi_app(registry: CollectorRegistry = REGISTRY, disable_compression: bool = False) -> Callable:
+    """Create a ASGI app which serves the metrics from a registry."""
+
+    async def prometheus_app(scope, receive, send):
+        assert scope.get("type") == "http"
+        # Prepare parameters
+        params = parse_qs(scope.get('query_string', b''))
+        accept_header = ",".join([
+            value.decode("utf8") for (name, value) in scope.get('headers')
+            if name.decode("utf8").lower() == 'accept'
+        ])
+        accept_encoding_header = ",".join([
+            value.decode("utf8") for (name, value) in scope.get('headers')
+            if name.decode("utf8").lower() == 'accept-encoding'
+        ])
+        # Bake output
+        status, headers, output = _bake_output(registry, accept_header, accept_encoding_header, params, disable_compression)
+        formatted_headers = []
+        for header in headers:
+            formatted_headers.append(tuple(x.encode('utf8') for x in header))
+        # Return output
+        payload = await receive()
+        if payload.get("type") == "http.request":
+            await send(
+                {
+                    "type": "http.response.start",
+                    "status": int(status.split(' ')[0]),
+                    "headers": formatted_headers,
+                }
+            )
+            await send({"type": "http.response.body", "body": output})
+
+    return prometheus_app

zhmc_prometheus_exporter/vendor/prometheus_client/bridge/__init__.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+
+import logging
+import re
+import socket
+import threading
+import time
+from timeit import default_timer
+from typing import Callable, Tuple
+
+from ..registry import CollectorRegistry, REGISTRY
+
+# Roughly, have to keep to what works as a file name.
+# We also remove periods, so labels can be distinguished.
+
+_INVALID_GRAPHITE_CHARS = re.compile(r"[^a-zA-Z0-9_-]")
+
+
+def _sanitize(s):
+    return _INVALID_GRAPHITE_CHARS.sub('_', s)
+
+
+class _RegularPush(threading.Thread):
+    def __init__(self, pusher, interval, prefix):
+        super().__init__()
+        self._pusher = pusher
+        self._interval = interval
+        self._prefix = prefix
+
+    def run(self):
+        wait_until = default_timer()
+        while True:
+            while True:
+                now = default_timer()
+                if now >= wait_until:
+                    # May need to skip some pushes.
+                    while wait_until < now:
+                        wait_until += self._interval
+                    break
+                # time.sleep can return early.
+                time.sleep(wait_until - now)
+            try:
+                self._pusher.push(prefix=self._prefix)
+            except OSError:
+                logging.exception("Push failed")
+
+
+class GraphiteBridge:
+    def __init__(self,
+                 address: Tuple[str, int],
+                 registry: CollectorRegistry = REGISTRY,
+                 timeout_seconds: float = 30,
+                 _timer: Callable[[], float] = time.time,
+                 tags: bool = False,
+                 ):
+        self._address = address
+        self._registry = registry
+        self._tags = tags
+        self._timeout = timeout_seconds
+        self._timer = _timer
+
+    def push(self, prefix: str = '') -> None:
+        now = int(self._timer())
+        output = []
+
+        prefixstr = ''
+        if prefix:
+            prefixstr = prefix + '.'
+
+        for metric in self._registry.collect():
+            for s in metric.samples:
+                if s.labels:
+                    if self._tags:
+                        sep = ';'
+                        fmt = '{0}={1}'
+                    else:
+                        sep = '.'
+                        fmt = '{0}.{1}'
+                    labelstr = sep + sep.join(
+                        [fmt.format(
+                            _sanitize(k), _sanitize(v))
+                            for k, v in sorted(s.labels.items())])
+                else:
+                    labelstr = ''
+                output.append(f'{prefixstr}{_sanitize(s.name)}{labelstr} {float(s.value)} {now}\n')
+
+        conn = socket.create_connection(self._address, self._timeout)
+        conn.sendall(''.join(output).encode('ascii'))
+        conn.close()
+
+    def start(self, interval: float = 60.0, prefix: str = '') -> None:
+        t = _RegularPush(self, interval, prefix)
+        t.daemon = True
+        t.start()

zhmc_prometheus_exporter/vendor/prometheus_client/bridge/graphite.py

@@ -0,0 +1,82 @@
+from timeit import default_timer
+from types import TracebackType
+from typing import (
+    Any, Callable, Literal, Optional, Tuple, Type, TYPE_CHECKING, TypeVar,
+    Union,
+)
+
+from .decorator import decorate
+
+if TYPE_CHECKING:
+    from . import Counter
+    F = TypeVar("F", bound=Callable[..., Any])
+
+
+class ExceptionCounter:
+    def __init__(self, counter: "Counter", exception: Union[Type[BaseException], Tuple[Type[BaseException], ...]]) -> None:
+        self._counter = counter
+        self._exception = exception
+
+    def __enter__(self) -> None:
+        pass
+
+    def __exit__(self, typ: Optional[Type[BaseException]], value: Optional[BaseException], traceback: Optional[TracebackType]) -> Literal[False]:
+        if isinstance(value, self._exception):
+            self._counter.inc()
+        return False
+
+    def __call__(self, f: "F") -> "F":
+        def wrapped(func, *args, **kwargs):
+            with self:
+                return func(*args, **kwargs)
+
+        return decorate(f, wrapped)
+
+
+class InprogressTracker:
+    def __init__(self, gauge):
+        self._gauge = gauge
+
+    def __enter__(self):
+        self._gauge.inc()
+
+    def __exit__(self, typ, value, traceback):
+        self._gauge.dec()
+
+    def __call__(self, f: "F") -> "F":
+        def wrapped(func, *args, **kwargs):
+            with self:
+                return func(*args, **kwargs)
+
+        return decorate(f, wrapped)
+
+
+class Timer:
+    def __init__(self, metric, callback_name):
+        self._metric = metric
+        self._callback_name = callback_name
+
+    def _new_timer(self):
+        return self.__class__(self._metric, self._callback_name)
+
+    def __enter__(self):
+        self._start = default_timer()
+        return self
+
+    def __exit__(self, typ, value, traceback):
+        # Time can go backwards.
+        duration = max(default_timer() - self._start, 0)
+        callback = getattr(self._metric, self._callback_name)
+        callback(duration)
+
+    def labels(self, *args, **kw):
+        self._metric = self._metric.labels(*args, **kw)
+
+    def __call__(self, f: "F") -> "F":
+        def wrapped(func, *args, **kwargs):
+            # Obtaining new instance of timer every time
+            # ensures thread safety and reentrancy.
+            with self._new_timer():
+                return func(*args, **kwargs)
+
+        return decorate(f, wrapped)