diff --git a/gcloud/logging/_gax.py b/gcloud/logging/_gax.py index 52ec001e088f..3f34645f3588 100644 --- a/gcloud/logging/_gax.py +++ b/gcloud/logging/_gax.py @@ -154,7 +154,8 @@ def list_sinks(self, project, page_size=0, page_token=None): with another call (pass that value as ``page_token``). """ options = _build_paging_options(page_token) - page_iter = self._gax_api.list_sinks(project, page_size, options) + path = 'projects/%s' % (project,) + page_iter = self._gax_api.list_sinks(path, page_size, options) sinks = [_log_sink_pb_to_mapping(log_sink_pb) for log_sink_pb in page_iter.next()] token = page_iter.page_token or None @@ -289,7 +290,8 @@ def list_metrics(self, project, page_size=0, page_token=None): with another call (pass that value as ``page_token``). """ options = _build_paging_options(page_token) - page_iter = self._gax_api.list_log_metrics(project, page_size, options) + path = 'projects/%s' % (project,) + page_iter = self._gax_api.list_log_metrics(path, page_size, options) metrics = [_log_metric_pb_to_mapping(log_metric_pb) for log_metric_pb in page_iter.next()] token = page_iter.page_token or None @@ -416,6 +418,41 @@ def _pb_timestamp_to_rfc3339(timestamp_pb): return _datetime_to_rfc3339(timestamp) +def _value_pb_to_value(value_pb): + """Helper for :func:`_log_entry_pb_to_mapping`.""" + kind = value_pb.WhichOneof('kind') + + if kind is None: + result = None + + elif kind == 'string_value': + result = value_pb.string_value + + elif kind == 'bool_value': + result = value_pb.bool_value + + elif kind == 'number_value': + result = value_pb.number_value + + elif kind == 'list_value': + result = [_value_pb_to_value(element) + for element in value_pb.list_value.values] + + elif kind == 'struct_value': + result = _struct_pb_to_mapping(value_pb.struct_value) + + else: + raise ValueError('Value protobuf had unknown kind: %s' % (kind,)) + + return result + + +def _struct_pb_to_mapping(struct_pb): + """Helper for :func:`_log_entry_pb_to_mapping`.""" + return dict([(key, _value_pb_to_value(struct_pb.fields[key])) + for key in struct_pb.fields]) + + def _log_entry_pb_to_mapping(entry_pb): """Helper for :meth:`list_entries`, et aliae @@ -426,27 +463,32 @@ def _log_entry_pb_to_mapping(entry_pb): mapping = { 'logName': entry_pb.log_name, 'resource': _mon_resource_pb_to_mapping(entry_pb.resource), - 'severity': entry_pb.severity, + 'severity': LogSeverity.Name(entry_pb.severity), 'insertId': entry_pb.insert_id, 'timestamp': _pb_timestamp_to_rfc3339(entry_pb.timestamp), 'labels': entry_pb.labels, - 'textPayload': entry_pb.text_payload, - 'jsonPayload': entry_pb.json_payload, - 'protoPayload': entry_pb.proto_payload, } + if entry_pb.HasField('text_payload'): + mapping['textPayload'] = entry_pb.text_payload + + if entry_pb.HasField('json_payload'): + mapping['jsonPayload'] = _struct_pb_to_mapping(entry_pb.json_payload) + + if entry_pb.HasField('proto_payload'): + mapping['protoPayload'] = entry_pb.proto_payload if entry_pb.http_request: request = entry_pb.http_request mapping['httpRequest'] = { - 'request_method': request.request_method, - 'request_url': request.request_url, + 'requestMethod': request.request_method, + 'requestUrl': request.request_url, 'status': request.status, 'referer': request.referer, - 'user_agent': request.user_agent, - 'cache_hit': request.cache_hit, - 'request_size': request.request_size, - 'response_size': request.response_size, - 'remote_ip': request.remote_ip, + 'userAgent': request.user_agent, + 'cacheHit': request.cache_hit, + 'requestSize': request.request_size, + 'responseSize': request.response_size, + 'remoteIp': request.remote_ip, } if entry_pb.operation: diff --git a/gcloud/logging/test__gax.py b/gcloud/logging/test__gax.py index 3174771d5989..d68082109cde 100644 --- a/gcloud/logging/test__gax.py +++ b/gcloud/logging/test__gax.py @@ -78,13 +78,16 @@ def test_list_entries_no_paging(self): self.assertTrue(options.page_token is INITIAL_PAGE) def test_list_entries_with_paging(self): + from google.protobuf.struct_pb2 import Value from gcloud._testing import _GAXPageIterator SIZE = 23 TOKEN = 'TOKEN' NEW_TOKEN = 'NEW_TOKEN' PAYLOAD = {'message': 'MESSAGE', 'weather': 'sunny'} + struct_pb = _StructPB(dict([(key, Value(string_value=value)) + for key, value in PAYLOAD.items()])) response = _GAXPageIterator( - [_LogEntryPB(self.LOG_NAME, json_payload=PAYLOAD)], NEW_TOKEN) + [_LogEntryPB(self.LOG_NAME, json_payload=struct_pb)], NEW_TOKEN) gax_api = _GAXLoggingAPI(_list_log_entries_response=response) api = self._makeOne(gax_api) @@ -109,6 +112,7 @@ def test_list_entries_with_paging(self): def test_list_entries_with_extra_properties(self): from datetime import datetime + from google.logging.type.log_severity_pb2 import WARNING from gcloud._testing import _GAXPageIterator from gcloud._helpers import UTC from gcloud._helpers import _datetime_to_rfc3339 @@ -126,7 +130,7 @@ def test_list_entries_with_extra_properties(self): request = _HTTPRequestPB() operation = _LogEntryOperationPB() EXTRAS = { - 'severity': SEVERITY, + 'severity': WARNING, 'labels': LABELS, 'insert_id': IID, 'http_request': request, @@ -154,15 +158,15 @@ def test_list_entries_with_extra_properties(self): self.assertEqual(entry['insertId'], IID) self.assertEqual(entry['timestamp'], _datetime_to_rfc3339(NOW)) EXPECTED_REQUEST = { - 'request_method': request.request_method, - 'request_url': request.request_url, + 'requestMethod': request.request_method, + 'requestUrl': request.request_url, 'status': request.status, - 'request_size': request.request_size, - 'response_size': request.response_size, + 'requestSize': request.request_size, + 'responseSize': request.response_size, 'referer': request.referer, - 'user_agent': request.user_agent, - 'remote_ip': request.remote_ip, - 'cache_hit': request.cache_hit, + 'userAgent': request.user_agent, + 'remoteIp': request.remote_ip, + 'cacheHit': request.cache_hit, } self.assertEqual(entry['httpRequest'], EXPECTED_REQUEST) EXPECTED_OPERATION = { @@ -439,7 +443,7 @@ def test_list_sinks_no_paging(self): self.assertEqual(token, TOKEN) project, page_size, options = gax_api._list_sinks_called_with - self.assertEqual(project, self.PROJECT) + self.assertEqual(project, self.PROJECT_PATH) self.assertEqual(page_size, 0) self.assertEqual(options.page_token, INITIAL_PAGE) @@ -465,7 +469,7 @@ def test_list_sinks_w_paging(self): self.assertEqual(token, None) project, page_size, options = gax_api._list_sinks_called_with - self.assertEqual(project, self.PROJECT) + self.assertEqual(project, self.PROJECT_PATH) self.assertEqual(page_size, PAGE_SIZE) self.assertEqual(options.page_token, TOKEN) @@ -643,7 +647,7 @@ def test_list_metrics_no_paging(self): self.assertEqual(token, TOKEN) project, page_size, options = gax_api._list_log_metrics_called_with - self.assertEqual(project, self.PROJECT) + self.assertEqual(project, self.PROJECT_PATH) self.assertEqual(page_size, 0) self.assertEqual(options.page_token, INITIAL_PAGE) @@ -669,7 +673,7 @@ def test_list_metrics_w_paging(self): self.assertEqual(token, None) project, page_size, options = gax_api._list_log_metrics_called_with - self.assertEqual(project, self.PROJECT) + self.assertEqual(project, self.PROJECT_PATH) self.assertEqual(page_size, PAGE_SIZE) self.assertEqual(options.page_token, TOKEN) @@ -811,6 +815,75 @@ def test_metric_delete_hit(self): self.assertEqual(options, None) +@unittest2.skipUnless(_HAVE_GAX, 'No gax-python') +class Test_value_pb_to_value(_Base, unittest2.TestCase): + + def _callFUT(self, value_pb): + from gcloud.logging._gax import _value_pb_to_value + return _value_pb_to_value(value_pb) + + def test_w_null_values(self): + from google.protobuf.struct_pb2 import Value + value_pb = Value() + self.assertEqual(self._callFUT(value_pb), None) + value_pb = Value(null_value=None) + self.assertEqual(self._callFUT(value_pb), None) + + def test_w_string_value(self): + from google.protobuf.struct_pb2 import Value + STRING = 'STRING' + value_pb = Value(string_value=STRING) + self.assertEqual(self._callFUT(value_pb), STRING) + + def test_w_bool_values(self): + from google.protobuf.struct_pb2 import Value + true_value_pb = Value(bool_value=True) + self.assertTrue(self._callFUT(true_value_pb) is True) + false_value_pb = Value(bool_value=False) + self.assertTrue(self._callFUT(false_value_pb) is False) + + def test_w_number_values(self): + from google.protobuf.struct_pb2 import Value + ANSWER = 42 + PI = 3.1415926 + int_value_pb = Value(number_value=ANSWER) + self.assertEqual(self._callFUT(int_value_pb), ANSWER) + float_value_pb = Value(number_value=PI) + self.assertEqual(self._callFUT(float_value_pb), PI) + + def test_w_list_value(self): + from google.protobuf.struct_pb2 import Value + STRING = 'STRING' + PI = 3.1415926 + value_pb = Value() + value_pb.list_value.values.add(string_value=STRING) + value_pb.list_value.values.add(bool_value=True) + value_pb.list_value.values.add(number_value=PI) + self.assertEqual(self._callFUT(value_pb), [STRING, True, PI]) + + def test_w_struct_value(self): + from google.protobuf.struct_pb2 import Value + STRING = 'STRING' + PI = 3.1415926 + value_pb = Value() + value_pb.struct_value.fields['string'].string_value = STRING + value_pb.struct_value.fields['bool'].bool_value = True + value_pb.struct_value.fields['number'].number_value = PI + self.assertEqual(self._callFUT(value_pb), + {'string': STRING, 'bool': True, 'number': PI}) + + def test_w_unknown_kind(self): + + class _Value(object): + + def WhichOneof(self, name): + assert name == 'kind' + return 'UNKNOWN' + + with self.assertRaises(ValueError): + self._callFUT(_Value()) + + class _GAXBaseAPI(object): _random_gax_error = False @@ -974,9 +1047,15 @@ def __init__(self, type_='global', **labels): self.labels = labels +class _StructPB(object): + + def __init__(self, fields): + self.fields = fields + + class _LogEntryPB(object): - severity = 'DEFAULT' + severity = 0 http_request = operation = insert_id = None text_payload = json_payload = proto_payload = None @@ -987,6 +1066,9 @@ def __init__(self, log_name, **kw): self.labels = kw.pop('labels', {}) self.__dict__.update(kw) + def HasField(self, field_name): + return getattr(self, field_name, None) is not None + @staticmethod def _make_timestamp(): from datetime import datetime diff --git a/system_tests/logging_.py b/system_tests/logging_.py index db29f854d4d0..5f9fb207974a 100644 --- a/system_tests/logging_.py +++ b/system_tests/logging_.py @@ -33,6 +33,33 @@ TOPIC_NAME = 'gcloud-python-system-testing%s' % (_RESOURCE_ID,) +def _retry_backoff(result_predicate, meth, *args, **kw): + from grpc.beta.interfaces import StatusCode + from grpc.framework.interfaces.face.face import AbortionError + backoff_intervals = [1, 2, 4, 8] + while True: + try: + result = meth(*args, **kw) + except AbortionError as error: + if error.code != StatusCode.UNAVAILABLE: + raise + if backoff_intervals: + time.sleep(backoff_intervals.pop(0)) + continue + else: + raise + if result_predicate(result): + return result + if backoff_intervals: + time.sleep(backoff_intervals.pop(0)) + else: + raise RuntimeError('%s: %s %s' % (meth, args, kw)) + + +def _has_entries(result): + return len(result[0]) > 0 + + class Config(object): """Run-time configuration to be modified at set-up. @@ -75,8 +102,7 @@ def test_log_text(self): logger = Config.CLIENT.logger(self._logger_name()) self.to_delete.append(logger) logger.log_text(TEXT_PAYLOAD) - time.sleep(2) - entries, _ = logger.list_entries() + entries, _ = _retry_backoff(_has_entries, logger.list_entries) self.assertEqual(len(entries), 1) self.assertEqual(entries[0].payload, TEXT_PAYLOAD) @@ -86,7 +112,7 @@ def test_log_text_w_metadata(self): SEVERITY = 'INFO' METHOD = 'POST' URI = 'https://api.example.com/endpoint' - STATUS = '500' + STATUS = 500 REQUEST = { 'requestMethod': METHOD, 'requestUrl': URI, @@ -94,18 +120,22 @@ def test_log_text_w_metadata(self): } logger = Config.CLIENT.logger(self._logger_name()) self.to_delete.append(logger) + logger.log_text(TEXT_PAYLOAD, insert_id=INSERT_ID, severity=SEVERITY, http_request=REQUEST) - time.sleep(2) - entries, _ = logger.list_entries() + entries, _ = _retry_backoff(_has_entries, logger.list_entries) + self.assertEqual(len(entries), 1) - self.assertEqual(entries[0].payload, TEXT_PAYLOAD) - self.assertEqual(entries[0].insert_id, INSERT_ID) - self.assertEqual(entries[0].severity, SEVERITY) - request = entries[0].http_request + + entry = entries[0] + self.assertEqual(entry.payload, TEXT_PAYLOAD) + self.assertEqual(entry.insert_id, INSERT_ID) + self.assertEqual(entry.severity, SEVERITY) + + request = entry.http_request self.assertEqual(request['requestMethod'], METHOD) self.assertEqual(request['requestUrl'], URI) - self.assertEqual(request['status'], int(STATUS)) + self.assertEqual(request['status'], STATUS) def test_log_struct(self): JSON_PAYLOAD = { @@ -114,9 +144,10 @@ def test_log_struct(self): } logger = Config.CLIENT.logger(self._logger_name()) self.to_delete.append(logger) + logger.log_struct(JSON_PAYLOAD) - time.sleep(2) - entries, _ = logger.list_entries() + entries, _ = _retry_backoff(_has_entries, logger.list_entries) + self.assertEqual(len(entries), 1) self.assertEqual(entries[0].payload, JSON_PAYLOAD) @@ -129,7 +160,7 @@ def test_log_struct_w_metadata(self): SEVERITY = 'INFO' METHOD = 'POST' URI = 'https://api.example.com/endpoint' - STATUS = '500' + STATUS = 500 REQUEST = { 'requestMethod': METHOD, 'requestUrl': URI, @@ -137,10 +168,11 @@ def test_log_struct_w_metadata(self): } logger = Config.CLIENT.logger(self._logger_name()) self.to_delete.append(logger) + logger.log_struct(JSON_PAYLOAD, insert_id=INSERT_ID, severity=SEVERITY, http_request=REQUEST) - time.sleep(2) - entries, _ = logger.list_entries() + entries, _ = _retry_backoff(_has_entries, logger.list_entries) + self.assertEqual(len(entries), 1) self.assertEqual(entries[0].payload, JSON_PAYLOAD) self.assertEqual(entries[0].insert_id, INSERT_ID) @@ -148,7 +180,7 @@ def test_log_struct_w_metadata(self): request = entries[0].http_request self.assertEqual(request['requestMethod'], METHOD) self.assertEqual(request['requestUrl'], URI) - self.assertEqual(request['status'], int(STATUS)) + self.assertEqual(request['status'], STATUS) def test_create_metric(self): metric = Config.CLIENT.metric(