Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix GAX logging system tests #1952

Merged
merged 6 commits into from
Jul 6, 2016
Merged

Fix GAX logging system tests #1952

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
dedbaf1
Pass project path to GAX 'list_sinks'/'list_log_metrics'.
tseaver Jul 1, 2016
5646bd5
Moar impedance mapping: JSON camelCase vs. proto with_underscore.
tseaver Jul 1, 2016
3a3613e
Add backoff to 'list_entries' calls for eventual consistency.
tseaver Jul 1, 2016
d662dd9
Properly unpack JSON payload in GAX 'list_entries'.
tseaver Jul 1, 2016
ac6bd5c
Impedance matching: JSON severity names vs. protobuf numbers.
tseaver Jul 1, 2016
6ffe88d
Skip remainder of backoff loop on failure.
tseaver Jul 6, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 55 additions & 13 deletions gcloud/logging/_gax.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,8 @@ def list_sinks(self, project, page_size=0, page_token=None):
with another call (pass that value as ``page_token``).
"""
options = _build_paging_options(page_token)
page_iter = self._gax_api.list_sinks(project, page_size, options)
path = 'projects/%s' % (project,)
page_iter = self._gax_api.list_sinks(path, page_size, options)
sinks = [_log_sink_pb_to_mapping(log_sink_pb)
for log_sink_pb in page_iter.next()]
token = page_iter.page_token or None
Expand Down Expand Up @@ -289,7 +290,8 @@ def list_metrics(self, project, page_size=0, page_token=None):
with another call (pass that value as ``page_token``).
"""
options = _build_paging_options(page_token)
page_iter = self._gax_api.list_log_metrics(project, page_size, options)
path = 'projects/%s' % (project,)
page_iter = self._gax_api.list_log_metrics(path, page_size, options)
metrics = [_log_metric_pb_to_mapping(log_metric_pb)
for log_metric_pb in page_iter.next()]
token = page_iter.page_token or None
Expand Down Expand Up @@ -416,6 +418,41 @@ def _pb_timestamp_to_rfc3339(timestamp_pb):
return _datetime_to_rfc3339(timestamp)


def _value_pb_to_value(value_pb):
"""Helper for :func:`_log_entry_pb_to_mapping`."""
kind = value_pb.WhichOneof('kind')

if kind is None:
result = None

elif kind == 'string_value':
result = value_pb.string_value

elif kind == 'bool_value':
result = value_pb.bool_value

elif kind == 'number_value':
result = value_pb.number_value

elif kind == 'list_value':
result = [_value_pb_to_value(element)
for element in value_pb.list_value.values]

elif kind == 'struct_value':
result = _struct_pb_to_mapping(value_pb.struct_value)

else:
raise ValueError('Value protobuf had unknown kind: %s' % (kind,))

return result


def _struct_pb_to_mapping(struct_pb):
"""Helper for :func:`_log_entry_pb_to_mapping`."""
return dict([(key, _value_pb_to_value(struct_pb.fields[key]))
for key in struct_pb.fields])


def _log_entry_pb_to_mapping(entry_pb):
"""Helper for :meth:`list_entries`, et aliae
Expand All @@ -426,27 +463,32 @@ def _log_entry_pb_to_mapping(entry_pb):
mapping = {
'logName': entry_pb.log_name,
'resource': _mon_resource_pb_to_mapping(entry_pb.resource),
'severity': entry_pb.severity,
'severity': LogSeverity.Name(entry_pb.severity),
'insertId': entry_pb.insert_id,
'timestamp': _pb_timestamp_to_rfc3339(entry_pb.timestamp),
'labels': entry_pb.labels,
'textPayload': entry_pb.text_payload,
'jsonPayload': entry_pb.json_payload,
'protoPayload': entry_pb.proto_payload,
}
if entry_pb.HasField('text_payload'):
mapping['textPayload'] = entry_pb.text_payload

if entry_pb.HasField('json_payload'):
mapping['jsonPayload'] = _struct_pb_to_mapping(entry_pb.json_payload)

if entry_pb.HasField('proto_payload'):
mapping['protoPayload'] = entry_pb.proto_payload

if entry_pb.http_request:
request = entry_pb.http_request
mapping['httpRequest'] = {
'request_method': request.request_method,
'request_url': request.request_url,
'requestMethod': request.request_method,
'requestUrl': request.request_url,
'status': request.status,
'referer': request.referer,
'user_agent': request.user_agent,
'cache_hit': request.cache_hit,
'request_size': request.request_size,
'response_size': request.response_size,
'remote_ip': request.remote_ip,
'userAgent': request.user_agent,
'cacheHit': request.cache_hit,
'requestSize': request.request_size,
'responseSize': request.response_size,
'remoteIp': request.remote_ip,
}

if entry_pb.operation:
Expand Down
110 changes: 96 additions & 14 deletions gcloud/logging/test__gax.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,13 +78,16 @@ def test_list_entries_no_paging(self):
self.assertTrue(options.page_token is INITIAL_PAGE)

def test_list_entries_with_paging(self):
from google.protobuf.struct_pb2 import Value
from gcloud._testing import _GAXPageIterator
SIZE = 23
TOKEN = 'TOKEN'
NEW_TOKEN = 'NEW_TOKEN'
PAYLOAD = {'message': 'MESSAGE', 'weather': 'sunny'}
struct_pb = _StructPB(dict([(key, Value(string_value=value))
for key, value in PAYLOAD.items()]))
response = _GAXPageIterator(
[_LogEntryPB(self.LOG_NAME, json_payload=PAYLOAD)], NEW_TOKEN)
[_LogEntryPB(self.LOG_NAME, json_payload=struct_pb)], NEW_TOKEN)
gax_api = _GAXLoggingAPI(_list_log_entries_response=response)
api = self._makeOne(gax_api)

Expand All @@ -109,6 +112,7 @@ def test_list_entries_with_paging(self):

def test_list_entries_with_extra_properties(self):
from datetime import datetime
from google.logging.type.log_severity_pb2 import WARNING
from gcloud._testing import _GAXPageIterator
from gcloud._helpers import UTC
from gcloud._helpers import _datetime_to_rfc3339
Expand All @@ -126,7 +130,7 @@ def test_list_entries_with_extra_properties(self):
request = _HTTPRequestPB()
operation = _LogEntryOperationPB()
EXTRAS = {
'severity': SEVERITY,
'severity': WARNING,
'labels': LABELS,
'insert_id': IID,
'http_request': request,
Expand Down Expand Up @@ -154,15 +158,15 @@ def test_list_entries_with_extra_properties(self):
self.assertEqual(entry['insertId'], IID)
self.assertEqual(entry['timestamp'], _datetime_to_rfc3339(NOW))
EXPECTED_REQUEST = {
'request_method': request.request_method,
'request_url': request.request_url,
'requestMethod': request.request_method,
'requestUrl': request.request_url,
'status': request.status,
'request_size': request.request_size,
'response_size': request.response_size,
'requestSize': request.request_size,
'responseSize': request.response_size,
'referer': request.referer,
'user_agent': request.user_agent,
'remote_ip': request.remote_ip,
'cache_hit': request.cache_hit,
'userAgent': request.user_agent,
'remoteIp': request.remote_ip,
'cacheHit': request.cache_hit,
}
self.assertEqual(entry['httpRequest'], EXPECTED_REQUEST)
EXPECTED_OPERATION = {
Expand Down Expand Up @@ -439,7 +443,7 @@ def test_list_sinks_no_paging(self):
self.assertEqual(token, TOKEN)

project, page_size, options = gax_api._list_sinks_called_with
self.assertEqual(project, self.PROJECT)
self.assertEqual(project, self.PROJECT_PATH)
self.assertEqual(page_size, 0)
self.assertEqual(options.page_token, INITIAL_PAGE)

Expand All @@ -465,7 +469,7 @@ def test_list_sinks_w_paging(self):
self.assertEqual(token, None)

project, page_size, options = gax_api._list_sinks_called_with
self.assertEqual(project, self.PROJECT)
self.assertEqual(project, self.PROJECT_PATH)
self.assertEqual(page_size, PAGE_SIZE)
self.assertEqual(options.page_token, TOKEN)

Expand Down Expand Up @@ -643,7 +647,7 @@ def test_list_metrics_no_paging(self):
self.assertEqual(token, TOKEN)

project, page_size, options = gax_api._list_log_metrics_called_with
self.assertEqual(project, self.PROJECT)
self.assertEqual(project, self.PROJECT_PATH)
self.assertEqual(page_size, 0)
self.assertEqual(options.page_token, INITIAL_PAGE)

Expand All @@ -669,7 +673,7 @@ def test_list_metrics_w_paging(self):
self.assertEqual(token, None)

project, page_size, options = gax_api._list_log_metrics_called_with
self.assertEqual(project, self.PROJECT)
self.assertEqual(project, self.PROJECT_PATH)
self.assertEqual(page_size, PAGE_SIZE)
self.assertEqual(options.page_token, TOKEN)

Expand Down Expand Up @@ -811,6 +815,75 @@ def test_metric_delete_hit(self):
self.assertEqual(options, None)


@unittest2.skipUnless(_HAVE_GAX, 'No gax-python')
class Test_value_pb_to_value(_Base, unittest2.TestCase):

def _callFUT(self, value_pb):
from gcloud.logging._gax import _value_pb_to_value
return _value_pb_to_value(value_pb)

def test_w_null_values(self):
from google.protobuf.struct_pb2 import Value
value_pb = Value()
self.assertEqual(self._callFUT(value_pb), None)
value_pb = Value(null_value=None)
self.assertEqual(self._callFUT(value_pb), None)

def test_w_string_value(self):
from google.protobuf.struct_pb2 import Value
STRING = 'STRING'
value_pb = Value(string_value=STRING)
self.assertEqual(self._callFUT(value_pb), STRING)

def test_w_bool_values(self):
from google.protobuf.struct_pb2 import Value
true_value_pb = Value(bool_value=True)
self.assertTrue(self._callFUT(true_value_pb) is True)
false_value_pb = Value(bool_value=False)
self.assertTrue(self._callFUT(false_value_pb) is False)

def test_w_number_values(self):
from google.protobuf.struct_pb2 import Value
ANSWER = 42
PI = 3.1415926
int_value_pb = Value(number_value=ANSWER)
self.assertEqual(self._callFUT(int_value_pb), ANSWER)
float_value_pb = Value(number_value=PI)
self.assertEqual(self._callFUT(float_value_pb), PI)

def test_w_list_value(self):
from google.protobuf.struct_pb2 import Value
STRING = 'STRING'
PI = 3.1415926
value_pb = Value()
value_pb.list_value.values.add(string_value=STRING)
value_pb.list_value.values.add(bool_value=True)
value_pb.list_value.values.add(number_value=PI)
self.assertEqual(self._callFUT(value_pb), [STRING, True, PI])

def test_w_struct_value(self):
from google.protobuf.struct_pb2 import Value
STRING = 'STRING'
PI = 3.1415926
value_pb = Value()
value_pb.struct_value.fields['string'].string_value = STRING
value_pb.struct_value.fields['bool'].bool_value = True
value_pb.struct_value.fields['number'].number_value = PI
self.assertEqual(self._callFUT(value_pb),
{'string': STRING, 'bool': True, 'number': PI})

def test_w_unknown_kind(self):

class _Value(object):

def WhichOneof(self, name):
assert name == 'kind'
return 'UNKNOWN'

with self.assertRaises(ValueError):
self._callFUT(_Value())


class _GAXBaseAPI(object):

_random_gax_error = False
Expand Down Expand Up @@ -974,9 +1047,15 @@ def __init__(self, type_='global', **labels):
self.labels = labels


class _StructPB(object):

def __init__(self, fields):
self.fields = fields


class _LogEntryPB(object):

severity = 'DEFAULT'
severity = 0
http_request = operation = insert_id = None
text_payload = json_payload = proto_payload = None

Expand All @@ -987,6 +1066,9 @@ def __init__(self, log_name, **kw):
self.labels = kw.pop('labels', {})
self.__dict__.update(kw)

def HasField(self, field_name):
return getattr(self, field_name, None) is not None

@staticmethod
def _make_timestamp():
from datetime import datetime
Expand Down
Loading