From 68455393c1069a34e563f7c2d8f11acf546213c1 Mon Sep 17 00:00:00 2001 From: isra17 Date: Sun, 23 Oct 2022 13:27:52 -0400 Subject: [PATCH] Fix exception in Urllib3 when dealing with filelike body. --- CHANGELOG.md | 5 +- .../instrumentation/urllib3/__init__.py | 22 +- .../tests/test_urllib3_ip_support.py | 137 --------- .../tests/test_urllib3_metrics.py | 270 ++++++++++++++++++ 4 files changed, 292 insertions(+), 142 deletions(-) create mode 100644 instrumentation/opentelemetry-instrumentation-urllib3/tests/test_urllib3_metrics.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 4331bb599c..852fb467fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased +- Fix exception in Urllib3 when dealing with filelike body. + ([#1399](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1399)) + ### Added - `opentelemetry-instrumentation-redis` Add `sanitize_query` config option to allow query sanitization. ([#1572](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1572)) -- `opentelemetry-instrumentation-elasticsearch` Add optional db.statement query sanitization. +- `opentelemetry-instrumentation-elasticsearch` Add optional db.statement query sanitization. ([#1598](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1598)) - `opentelemetry-instrumentation-celery` Record exceptions as events on the span. ([#1573](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/1573)) diff --git a/instrumentation/opentelemetry-instrumentation-urllib3/src/opentelemetry/instrumentation/urllib3/__init__.py b/instrumentation/opentelemetry-instrumentation-urllib3/src/opentelemetry/instrumentation/urllib3/__init__.py index 02f701068b..6e0da3be36 100644 --- a/instrumentation/opentelemetry-instrumentation-urllib3/src/opentelemetry/instrumentation/urllib3/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-urllib3/src/opentelemetry/instrumentation/urllib3/__init__.py @@ -64,7 +64,9 @@ def response_hook(span, request, response): --- """ +import collections.abc import contextlib +import io import typing from timeit import default_timer from typing import Collection @@ -213,8 +215,9 @@ def instrumented_urlopen(wrapped, instance, args, kwargs): if callable(response_hook): response_hook(span, instance, response) - request_size = 0 if body is None else len(body) + request_size = _get_body_size(body) response_size = int(response.headers.get("Content-Length", 0)) + metric_attributes = _create_metric_attributes( instance, response, method ) @@ -222,9 +225,10 @@ def instrumented_urlopen(wrapped, instance, args, kwargs): duration_histogram.record( elapsed_time, attributes=metric_attributes ) - request_size_histogram.record( - request_size, attributes=metric_attributes - ) + if request_size is not None: + request_size_histogram.record( + request_size, attributes=metric_attributes + ) response_size_histogram.record( response_size, attributes=metric_attributes ) @@ -268,6 +272,16 @@ def _get_url( return url +def _get_body_size(body: object) -> typing.Optional[int]: + if body is None: + return 0 + if isinstance(body, collections.abc.Sized): + return len(body) + if isinstance(body, io.BytesIO): + return body.getbuffer().nbytes + return None + + def _should_append_port(scheme: str, port: typing.Optional[int]) -> bool: if not port: return False diff --git a/instrumentation/opentelemetry-instrumentation-urllib3/tests/test_urllib3_ip_support.py b/instrumentation/opentelemetry-instrumentation-urllib3/tests/test_urllib3_ip_support.py index d47cd8f1ea..b7820a0d72 100644 --- a/instrumentation/opentelemetry-instrumentation-urllib3/tests/test_urllib3_ip_support.py +++ b/instrumentation/opentelemetry-instrumentation-urllib3/tests/test_urllib3_ip_support.py @@ -12,11 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from timeit import default_timer - import urllib3 -import urllib3.exceptions -from urllib3.request import encode_multipart_formdata from opentelemetry import trace from opentelemetry.instrumentation.urllib3 import URLLib3Instrumentor @@ -87,136 +83,3 @@ def assert_success_span( "net.peer.ip": self.assert_ip, } self.assertGreaterEqual(span.attributes.items(), attributes.items()) - - -class TestURLLib3InstrumentorMetric(HttpTestBase, TestBase): - def setUp(self): - super().setUp() - self.assert_ip = self.server.server_address[0] - self.assert_port = self.server.server_address[1] - self.http_host = ":".join(map(str, self.server.server_address[:2])) - self.http_url_base = "http://" + self.http_host - self.http_url = self.http_url_base + "/status/200" - URLLib3Instrumentor().instrument(meter_provider=self.meter_provider) - - def tearDown(self): - super().tearDown() - URLLib3Instrumentor().uninstrument() - - def test_metric_uninstrument(self): - with urllib3.PoolManager() as pool: - pool.request("GET", self.http_url) - URLLib3Instrumentor().uninstrument() - pool.request("GET", self.http_url) - - metrics_list = self.memory_metrics_reader.get_metrics_data() - for resource_metric in metrics_list.resource_metrics: - for scope_metric in resource_metric.scope_metrics: - for metric in scope_metric.metrics: - for point in list(metric.data.data_points): - self.assertEqual(point.count, 1) - - def test_basic_metric_check_client_size_get(self): - with urllib3.PoolManager() as pool: - start_time = default_timer() - response = pool.request("GET", self.http_url) - client_duration_estimated = (default_timer() - start_time) * 1000 - - expected_attributes = { - "http.status_code": 200, - "http.host": self.assert_ip, - "http.method": "GET", - "http.flavor": "1.1", - "http.scheme": "http", - "net.peer.name": self.assert_ip, - "net.peer.port": self.assert_port, - } - expected_data = { - "http.client.request.size": 0, - "http.client.response.size": len(response.data), - } - expected_metrics = [ - "http.client.duration", - "http.client.request.size", - "http.client.response.size", - ] - - resource_metrics = ( - self.memory_metrics_reader.get_metrics_data().resource_metrics - ) - for metrics in resource_metrics: - for scope_metrics in metrics.scope_metrics: - self.assertEqual(len(scope_metrics.metrics), 3) - for metric in scope_metrics.metrics: - for data_point in metric.data.data_points: - if metric.name in expected_data: - self.assertEqual( - data_point.sum, expected_data[metric.name] - ) - if metric.name == "http.client.duration": - self.assertAlmostEqual( - data_point.sum, - client_duration_estimated, - delta=1000, - ) - self.assertIn(metric.name, expected_metrics) - self.assertDictEqual( - expected_attributes, - dict(data_point.attributes), - ) - self.assertEqual(data_point.count, 1) - - def test_basic_metric_check_client_size_post(self): - with urllib3.PoolManager() as pool: - start_time = default_timer() - data_fields = {"data": "test"} - response = pool.request("POST", self.http_url, fields=data_fields) - client_duration_estimated = (default_timer() - start_time) * 1000 - - expected_attributes = { - "http.status_code": 501, - "http.host": self.assert_ip, - "http.method": "POST", - "http.flavor": "1.1", - "http.scheme": "http", - "net.peer.name": self.assert_ip, - "net.peer.port": self.assert_port, - } - - body = encode_multipart_formdata(data_fields)[0] - - expected_data = { - "http.client.request.size": len(body), - "http.client.response.size": len(response.data), - } - expected_metrics = [ - "http.client.duration", - "http.client.request.size", - "http.client.response.size", - ] - - resource_metrics = ( - self.memory_metrics_reader.get_metrics_data().resource_metrics - ) - for metrics in resource_metrics: - for scope_metrics in metrics.scope_metrics: - self.assertEqual(len(scope_metrics.metrics), 3) - for metric in scope_metrics.metrics: - for data_point in metric.data.data_points: - if metric.name in expected_data: - self.assertEqual( - data_point.sum, expected_data[metric.name] - ) - if metric.name == "http.client.duration": - self.assertAlmostEqual( - data_point.sum, - client_duration_estimated, - delta=1000, - ) - self.assertIn(metric.name, expected_metrics) - - self.assertDictEqual( - expected_attributes, - dict(data_point.attributes), - ) - self.assertEqual(data_point.count, 1) diff --git a/instrumentation/opentelemetry-instrumentation-urllib3/tests/test_urllib3_metrics.py b/instrumentation/opentelemetry-instrumentation-urllib3/tests/test_urllib3_metrics.py new file mode 100644 index 0000000000..ca4b505e5b --- /dev/null +++ b/instrumentation/opentelemetry-instrumentation-urllib3/tests/test_urllib3_metrics.py @@ -0,0 +1,270 @@ +# Copyright The OpenTelemetry Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +from timeit import default_timer + +import httpretty +import urllib3 +import urllib3.exceptions +from urllib3.request import encode_multipart_formdata + +from opentelemetry.instrumentation.urllib3 import URLLib3Instrumentor +from opentelemetry.sdk.metrics.export import ( + HistogramDataPoint, + NumberDataPoint, +) +from opentelemetry.test.httptest import HttpTestBase +from opentelemetry.test.test_base import TestBase + + +class TestURLLib3InstrumentorMetric(HttpTestBase, TestBase): + HTTP_URL = "http://httpbin.org/status/200" + + def setUp(self): + super().setUp() + URLLib3Instrumentor().instrument() + httpretty.enable(allow_net_connect=False) + httpretty.register_uri(httpretty.GET, self.HTTP_URL, body="Hello!") + httpretty.register_uri(httpretty.POST, self.HTTP_URL, body="Hello!") + self.pool = urllib3.PoolManager() + + def tearDown(self): + super().tearDown() + self.pool.clear() + URLLib3Instrumentor().uninstrument() + + httpretty.disable() + httpretty.reset() + + def get_sorted_metrics(self): + resource_metrics = ( + self.memory_metrics_reader.get_metrics_data().resource_metrics + ) + for metrics in resource_metrics: + for scope_metrics in metrics.scope_metrics: + all_metrics = list(scope_metrics.metrics) + return self.sorted_metrics(all_metrics) + + @staticmethod + def sorted_metrics(metrics): + """ + Sorts metrics by metric name. + """ + return sorted( + metrics, + key=lambda m: m.name, + ) + + def assert_metric_expected( + self, metric, expected_value, expected_attributes + ): + data_point = next(metric.data.data_points) + + if isinstance(data_point, HistogramDataPoint): + self.assertEqual( + data_point.sum, + expected_value, + ) + elif isinstance(data_point, NumberDataPoint): + self.assertEqual( + data_point.value, + expected_value, + ) + + self.assertDictEqual( + expected_attributes, + dict(data_point.attributes), + ) + + def assert_duration_metric_expected( + self, metric, duration_estimated, expected_attributes + ): + data_point = next(metric.data.data_points) + + self.assertAlmostEqual( + data_point.sum, + duration_estimated, + delta=200, + ) + + self.assertDictEqual( + expected_attributes, + dict(data_point.attributes), + ) + + def test_basic_metrics(self): + start_time = default_timer() + response = self.pool.request("GET", self.HTTP_URL) + client_duration_estimated = (default_timer() - start_time) * 1000 + + metrics = self.get_sorted_metrics() + + ( + client_duration, + client_request_size, + client_response_size, + ) = metrics + + self.assertEqual(client_duration.name, "http.client.duration") + self.assert_duration_metric_expected( + client_duration, + client_duration_estimated, + { + "http.flavor": "1.1", + "http.host": "httpbin.org", + "http.method": "GET", + "http.scheme": "http", + "http.status_code": 200, + "net.peer.name": "httpbin.org", + "net.peer.port": 80, + }, + ) + + self.assertEqual(client_request_size.name, "http.client.request.size") + self.assert_metric_expected( + client_request_size, + 0, + { + "http.flavor": "1.1", + "http.host": "httpbin.org", + "http.method": "GET", + "http.scheme": "http", + "http.status_code": 200, + "net.peer.name": "httpbin.org", + "net.peer.port": 80, + }, + ) + + self.assertEqual( + client_response_size.name, "http.client.response.size" + ) + self.assert_metric_expected( + client_response_size, + len(response.data), + { + "http.flavor": "1.1", + "http.host": "httpbin.org", + "http.method": "GET", + "http.scheme": "http", + "http.status_code": 200, + "net.peer.name": "httpbin.org", + "net.peer.port": 80, + }, + ) + + def test_str_request_body_size_metrics(self): + self.pool.request("POST", self.HTTP_URL, body="foobar") + + metrics = self.get_sorted_metrics() + (_, client_request_size, _) = metrics + + self.assertEqual(client_request_size.name, "http.client.request.size") + self.assert_metric_expected( + client_request_size, + 6, + { + "http.flavor": "1.1", + "http.host": "httpbin.org", + "http.method": "POST", + "http.scheme": "http", + "http.status_code": 200, + "net.peer.name": "httpbin.org", + "net.peer.port": 80, + }, + ) + + def test_bytes_request_body_size_metrics(self): + self.pool.request("POST", self.HTTP_URL, body=b"foobar") + + metrics = self.get_sorted_metrics() + (_, client_request_size, _) = metrics + + self.assertEqual(client_request_size.name, "http.client.request.size") + self.assert_metric_expected( + client_request_size, + 6, + { + "http.flavor": "1.1", + "http.host": "httpbin.org", + "http.method": "POST", + "http.scheme": "http", + "http.status_code": 200, + "net.peer.name": "httpbin.org", + "net.peer.port": 80, + }, + ) + + def test_fields_request_body_size_metrics(self): + self.pool.request("POST", self.HTTP_URL, fields={"foo": "bar"}) + + metrics = self.get_sorted_metrics() + (_, client_request_size, _) = metrics + + self.assertEqual(client_request_size.name, "http.client.request.size") + self.assert_metric_expected( + client_request_size, + len(encode_multipart_formdata({"foo": "bar"})[0]), + { + "http.flavor": "1.1", + "http.host": "httpbin.org", + "http.method": "POST", + "http.scheme": "http", + "http.status_code": 200, + "net.peer.name": "httpbin.org", + "net.peer.port": 80, + }, + ) + + def test_bytesio_request_body_size_metrics(self): + self.pool.request("POST", self.HTTP_URL, body=io.BytesIO(b"foobar")) + + metrics = self.get_sorted_metrics() + (_, client_request_size, _) = metrics + + self.assertEqual(client_request_size.name, "http.client.request.size") + self.assert_metric_expected( + client_request_size, + 6, + { + "http.flavor": "1.1", + "http.host": "httpbin.org", + "http.method": "POST", + "http.scheme": "http", + "http.status_code": 200, + "net.peer.name": "httpbin.org", + "net.peer.port": 80, + }, + ) + + def test_generator_request_body_size_metrics(self): + self.pool.request( + "POST", self.HTTP_URL, body=(b for b in (b"foo", b"bar")) + ) + + metrics = self.get_sorted_metrics() + self.assertEqual(len(metrics), 2) + self.assertNotIn("http.client.request.size", [m.name for m in metrics]) + + def test_metric_uninstrument(self): + self.pool.request("GET", self.HTTP_URL) + URLLib3Instrumentor().uninstrument() + self.pool.request("GET", self.HTTP_URL) + + metrics_list = self.memory_metrics_reader.get_metrics_data() + for resource_metric in metrics_list.resource_metrics: + for scope_metric in resource_metric.scope_metrics: + for metric in scope_metric.metrics: + for point in list(metric.data.data_points): + self.assertEqual(point.count, 1)