From 4892f1b416091df96aaa5c39b439adee992ba02f Mon Sep 17 00:00:00 2001 From: Leo10Gama Date: Fri, 17 Jun 2022 10:02:58 -0700 Subject: [PATCH 1/4] Send Event metrics via Telemetry --- samcli/lib/telemetry/metric.py | 4 ++++ tests/integration/telemetry/test_experimental_metric.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/samcli/lib/telemetry/metric.py b/samcli/lib/telemetry/metric.py index 8fc803fba1..006bb75323 100644 --- a/samcli/lib/telemetry/metric.py +++ b/samcli/lib/telemetry/metric.py @@ -17,6 +17,7 @@ from samcli.lib.warnings.sam_cli_warning import TemplateWarningsChecker from samcli.commands.exceptions import UserException from samcli.lib.telemetry.cicd import CICDDetector, CICDPlatform +from samcli.lib.telemetry.event import EventTracker from samcli.lib.telemetry.project_metadata import get_git_remote_origin_url, get_project_name, get_initial_commit_hash from samcli.commands._utils.experimental import get_all_experimental_statues from .telemetry import Telemetry @@ -158,6 +159,9 @@ def wrapped(*args, **kwargs): metric_specific_attributes["gitOrigin"] = get_git_remote_origin_url() metric_specific_attributes["projectName"] = get_project_name() metric_specific_attributes["initialCommit"] = get_initial_commit_hash() + # Event metrics + metric_specific_attributes["events"] = [e.to_json() for e in EventTracker.get_tracked_events()] + EventTracker.clear_trackers() metric.add_data("metricSpecificAttributes", metric_specific_attributes) # Metric about command's execution characteristics metric.add_data("duration", duration_fn()) diff --git a/tests/integration/telemetry/test_experimental_metric.py b/tests/integration/telemetry/test_experimental_metric.py index 2bcd93e340..98bd45649a 100644 --- a/tests/integration/telemetry/test_experimental_metric.py +++ b/tests/integration/telemetry/test_experimental_metric.py @@ -59,6 +59,7 @@ def test_must_send_experimental_metrics_if_experimental_command(self): "gitOrigin": ANY, "projectName": ANY, "initialCommit": ANY, + "events": ANY, }, "duration": ANY, "exitReason": ANY, @@ -113,6 +114,7 @@ def test_must_send_experimental_metrics_if_experimental_option(self): "gitOrigin": ANY, "projectName": ANY, "initialCommit": ANY, + "events": ANY, }, "duration": ANY, "exitReason": ANY, @@ -175,6 +177,7 @@ def test_must_send_cdk_project_type_metrics(self): "gitOrigin": ANY, "projectName": ANY, "initialCommit": ANY, + "events": ANY, }, "duration": ANY, "exitReason": ANY, From 1b5efd5cf4be6d9544f3bf48a225c7df845ae1f2 Mon Sep 17 00:00:00 2001 From: Leo10Gama Date: Fri, 17 Jun 2022 16:54:25 -0700 Subject: [PATCH 2/4] Add unit tests for event telemetry --- tests/unit/lib/telemetry/test_metric.py | 59 +++++++++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/tests/unit/lib/telemetry/test_metric.py b/tests/unit/lib/telemetry/test_metric.py index e4b59fd6ec..83980647c2 100644 --- a/tests/unit/lib/telemetry/test_metric.py +++ b/tests/unit/lib/telemetry/test_metric.py @@ -9,6 +9,7 @@ from unittest import TestCase from unittest.mock import patch, Mock, ANY, call +from samcli.lib.telemetry.event import Event, EventTracker import samcli.lib.telemetry.metric from samcli.lib.telemetry.cicd import CICDPlatform @@ -339,6 +340,64 @@ def real_fn(a, b=None): "The command metrics be emitted when used as a decorator", ) + @patch("samcli.lib.telemetry.metric.Context") + def test_must_return_empty_list_of_events(self, ContextMock): + ContextMock.get_current_context.return_value = self.context_mock + + def func_with_no_events(): + pass + + track_command(func_with_no_events)() + + args, _ = self.telemetry_instance.emit.call_args_list[0] + metric = args[0] + assert metric.get_metric_name() == "commandRun" + self.assertEqual(metric.get_data()["metricSpecificAttributes"]["events"], []) + + @patch("samcli.lib.telemetry.metric.Context") + def test_must_return_list_of_events(self, ContextMock): + ContextMock.get_current_context.return_value = self.context_mock + dummy_event = Mock(event_name=Mock(value="TestEvent"), event_value="TestValue") + dummy_event.to_json.return_value = Event.to_json(dummy_event) + + def func_with_event(): + EventTracker._events.append(dummy_event) + + track_command(func_with_event)() + + args, _ = self.telemetry_instance.emit.call_args_list[0] + metric = args[0] + assert metric.get_metric_name() == "commandRun" + self.assertEqual( + metric.get_data()["metricSpecificAttributes"]["events"], + [{"event_name": dummy_event.event_name.value, "event_value": dummy_event.event_value}], + ) + + @patch("samcli.lib.telemetry.metric.Context") + def test_must_return_list_of_multiple_events(self, ContextMock): + ContextMock.get_current_context.return_value = self.context_mock + dummy_event1 = Mock(event_name=Mock(value="Test1"), event_value="TestValue1") + dummy_event1.to_json.return_value = Event.to_json(dummy_event1) + dummy_event2 = Mock(event_name=Mock(value="TestEvent2"), event_value="TestValue2") + dummy_event2.to_json.return_value = Event.to_json(dummy_event2) + + def func_with_event(): + EventTracker._events.append(dummy_event1) + EventTracker._events.append(dummy_event2) + + track_command(func_with_event)() + + args, _ = self.telemetry_instance.emit.call_args_list[0] + metric = args[0] + assert metric.get_metric_name() == "commandRun" + self.assertEqual( + metric.get_data()["metricSpecificAttributes"]["events"], + [ + {"event_name": dummy_event1.event_name.value, "event_value": dummy_event1.event_value}, + {"event_name": dummy_event2.event_name.value, "event_value": dummy_event2.event_value}, + ], + ) + class TestParameterCapture(TestCase): def setUp(self): From 3f546f12b8c51abdb8f2abd04c250798d6898e12 Mon Sep 17 00:00:00 2001 From: Leo10Gama Date: Fri, 17 Jun 2022 19:11:25 -0700 Subject: [PATCH 3/4] Simplify event telemetry unit tests --- tests/unit/lib/telemetry/test_metric.py | 74 ++++++++++--------------- 1 file changed, 28 insertions(+), 46 deletions(-) diff --git a/tests/unit/lib/telemetry/test_metric.py b/tests/unit/lib/telemetry/test_metric.py index 83980647c2..2cd5c75e2f 100644 --- a/tests/unit/lib/telemetry/test_metric.py +++ b/tests/unit/lib/telemetry/test_metric.py @@ -340,63 +340,45 @@ def real_fn(a, b=None): "The command metrics be emitted when used as a decorator", ) + @parameterized.expand( + [ + ([],), + ([Mock(event_name=Mock(value="TestEvent"), event_value="TestValue")],), + ( + [ + Mock(event_name=Mock(value="Test1"), event_value="TestValue1"), + Mock(event_name=Mock(value="Test2"), event_value="TestValue2"), + ], + ), + ( + [ + Mock(event_name=Mock(value="T1"), event_value="1"), + Mock(event_name=Mock(value="T2"), event_value="2"), + Mock(event_name=Mock(value="T3"), event_value="3"), + ], + ), + ([],), + ([Mock(event_name=Mock(value="BackToOneToEnsure"), event_value="ValuesAreTrulyReset")],), + ] + ) @patch("samcli.lib.telemetry.metric.Context") - def test_must_return_empty_list_of_events(self, ContextMock): + def test_must_return_list_of_events(self, events, ContextMock): ContextMock.get_current_context.return_value = self.context_mock - - def func_with_no_events(): - pass - - track_command(func_with_no_events)() - - args, _ = self.telemetry_instance.emit.call_args_list[0] - metric = args[0] - assert metric.get_metric_name() == "commandRun" - self.assertEqual(metric.get_data()["metricSpecificAttributes"]["events"], []) - - @patch("samcli.lib.telemetry.metric.Context") - def test_must_return_list_of_events(self, ContextMock): - ContextMock.get_current_context.return_value = self.context_mock - dummy_event = Mock(event_name=Mock(value="TestEvent"), event_value="TestValue") - dummy_event.to_json.return_value = Event.to_json(dummy_event) + for e in events: + e.to_json.return_value = Event.to_json(e) def func_with_event(): - EventTracker._events.append(dummy_event) - - track_command(func_with_event)() + for e in events: + EventTracker._events.append(e) - args, _ = self.telemetry_instance.emit.call_args_list[0] - metric = args[0] - assert metric.get_metric_name() == "commandRun" - self.assertEqual( - metric.get_data()["metricSpecificAttributes"]["events"], - [{"event_name": dummy_event.event_name.value, "event_value": dummy_event.event_value}], - ) - - @patch("samcli.lib.telemetry.metric.Context") - def test_must_return_list_of_multiple_events(self, ContextMock): - ContextMock.get_current_context.return_value = self.context_mock - dummy_event1 = Mock(event_name=Mock(value="Test1"), event_value="TestValue1") - dummy_event1.to_json.return_value = Event.to_json(dummy_event1) - dummy_event2 = Mock(event_name=Mock(value="TestEvent2"), event_value="TestValue2") - dummy_event2.to_json.return_value = Event.to_json(dummy_event2) - - def func_with_event(): - EventTracker._events.append(dummy_event1) - EventTracker._events.append(dummy_event2) + expected = [e.to_json() for e in events] track_command(func_with_event)() args, _ = self.telemetry_instance.emit.call_args_list[0] metric = args[0] assert metric.get_metric_name() == "commandRun" - self.assertEqual( - metric.get_data()["metricSpecificAttributes"]["events"], - [ - {"event_name": dummy_event1.event_name.value, "event_value": dummy_event1.event_value}, - {"event_name": dummy_event2.event_name.value, "event_value": dummy_event2.event_value}, - ], - ) + self.assertEqual(metric.get_data()["metricSpecificAttributes"]["events"], expected) class TestParameterCapture(TestCase): From f2bbc2d2770a46b0816e942a3fdcaf9c14b9a9bf Mon Sep 17 00:00:00 2001 From: Leo10Gama Date: Mon, 20 Jun 2022 21:34:26 -0700 Subject: [PATCH 4/4] Clean up unit tests --- tests/unit/lib/telemetry/test_metric.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/unit/lib/telemetry/test_metric.py b/tests/unit/lib/telemetry/test_metric.py index 2cd5c75e2f..7d14233059 100644 --- a/tests/unit/lib/telemetry/test_metric.py +++ b/tests/unit/lib/telemetry/test_metric.py @@ -357,13 +357,13 @@ def real_fn(a, b=None): Mock(event_name=Mock(value="T3"), event_value="3"), ], ), - ([],), - ([Mock(event_name=Mock(value="BackToOneToEnsure"), event_value="ValuesAreTrulyReset")],), ] ) @patch("samcli.lib.telemetry.metric.Context") - def test_must_return_list_of_events(self, events, ContextMock): + @patch("samcli.lib.telemetry.event.EventTracker.clear_trackers", side_effect=EventTracker.clear_trackers) + def test_must_return_list_of_events(self, events, clr_mock, ContextMock): ContextMock.get_current_context.return_value = self.context_mock + clr_mock.clear_trackers.return_value = EventTracker.clear_trackers() for e in events: e.to_json.return_value = Event.to_json(e) @@ -378,7 +378,10 @@ def func_with_event(): args, _ = self.telemetry_instance.emit.call_args_list[0] metric = args[0] assert metric.get_metric_name() == "commandRun" - self.assertEqual(metric.get_data()["metricSpecificAttributes"]["events"], expected) + metric_events = metric.get_data()["metricSpecificAttributes"]["events"] + clr_mock.assert_called() + self.assertEqual(len(events), len(metric_events)) + self.assertEqual(metric_events, expected) class TestParameterCapture(TestCase):