Add system test for 'Sink.create' using a Pubsub topic.

docs/bigquery-usage.rst

@@ -291,26 +291,88 @@ Run a query which can be expected to complete within bounded time:
 
  >>> from gcloud import bigquery
  >>> client = bigquery.Client()
- >>> query = """\
- SELECT count(*) AS age_count FROM dataset_name.person_ages
- """
- >>> query = client.run_sync_query(query)
+ >>> QUERY = """\
+ ... SELECT count(*) AS age_count FROM dataset_name.person_ages
+ ... """
+ >>> query = client.run_sync_query(QUERY)
+ >>> query.timeout_ms = 1000
+ >>> query.run() # API request
+ >>> query.complete
+ True
+ >>> len(query.schema)
+ 1
+ >>> field = query.schema[0]
+ >>> field.name
+ u'count'
+ >>> field.field_type
+ u'INTEGER'
+ >>> field.mode
+ u'NULLABLE'
+ >>> query.rows
+ [(15,)]
+ >>> query.total_rows
+ 1
+
+If the rows returned by the query do not fit into the inital response,
+then we need to fetch the remaining rows via ``fetch_data``:
+
+.. doctest::
+
+ >>> from gcloud import bigquery
+ >>> client = bigquery.Client()
+ >>> QUERY = """\
+ ... SELECT * FROM dataset_name.person_ages
+ ... """
+ >>> query = client.run_sync_query(QUERY)
+ >>> query.timeout_ms = 1000
+ >>> query.run() # API request
+ >>> query.complete
+ True
+ >>> query.total_rows
+ 1234
+ >>> query.page_token
+ '8d6e452459238eb0fe87d8eb191dd526ee70a35e'
+ >>> do_something_with(query.schema, query.rows)
+ >>> token = query.page_token # for initial request
+ >>> while True:
+ ... do_something_with(query.schema, rows)
+ ... if token is None:
+ ... break
+ ... rows, _, token = query.fetch_data(page_token=token)
+
+
+If the query takes longer than the timeout allowed, ``query.complete``
+will be ``False``. In that case, we need to poll the associated job until
+it is done, and then fetch the reuslts:
+
+.. doctest::
+
+ >>> from gcloud import bigquery
+ >>> client = bigquery.Client()
+ >>> QUERY = """\
+ ... SELECT * FROM dataset_name.person_ages
+ ... """
+ >>> query = client.run_sync_query(QUERY)
  >>> query.timeout_ms = 1000
  >>> query.run() # API request
+ >>> query.complete
+ False
+ >>> job = query.job
  >>> retry_count = 100
- >>> while retry_count > 0 and not job.complete:
+ >>> while retry_count > 0 and job.state == 'running':
  ... retry_count -= 1
  ... time.sleep(10)
- ... query.reload() # API request
- >>> query.schema
- [{'name': 'age_count', 'type': 'integer', 'mode': 'nullable'}]
- >>> query.rows
- [(15,)]
+ ... job.reload() # API call
+ >>> job.state
+ 'done'
+ >>> token = None # for initial request
+ >>> while True:
+ ... rows, _, token = query.fetch_data(page_token=token)
+ ... do_something_with(query.schema, rows)
+ ... if token is None:
+ ... break
 
-.. note::
 
- If the query takes longer than the timeout allowed, ``job.complete``
- will be ``False``: we therefore poll until it is completed.
 
 Querying data (asynchronous)
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~

docs/pubsub-usage.rst

@@ -99,12 +99,12 @@ Test permissions allowed by the current IAM policy on a topic:
 .. doctest::
 
  >>> from gcloud import pubsub
- >>> from gcloud.pubsub.iam import OWNER_ROLE, WRITER_ROLE, READER_ROLE
+ >>> from gcloud.pubsub.iam import OWNER_ROLE, EDITOR_ROLE, VIEWER_ROLE
  >>> client = pubsub.Client()
  >>> topic = client.topic('topic_name')
  >>> allowed = topic.check_iam_permissions(
- ... [READER_ROLE, WRITER_ROLE, OWNER_ROLE]) # API request
- >>> allowed == [READER_ROLE, WRITER_ROLE]
+ ... [VIEWER_ROLE, EDITOR_ROLE, OWNER_ROLE]) # API request
+ >>> allowed == [VIEWER_ROLE, EDITOR_ROLE]
  True
 
 
@@ -349,11 +349,11 @@ Test permissions allowed by the current IAM policy on a subscription:
 .. doctest::
 
  >>> from gcloud import pubsub
- >>> from gcloud.pubsub.iam import OWNER_ROLE, WRITER_ROLE, READER_ROLE
+ >>> from gcloud.pubsub.iam import OWNER_ROLE, EDITOR_ROLE, VIEWER_ROLE
  >>> client = pubsub.Client()
  >>> topic = client.topic('topic_name')
  >>> subscription = topic.subscription('subscription_name')
  >>> allowed = subscription.check_iam_permissions(
- ... [READER_ROLE, WRITER_ROLE, OWNER_ROLE]) # API request
- >>> allowed == [READER_ROLE, WRITER_ROLE]
+ ... [VIEWER_ROLE, EDITOR_ROLE, OWNER_ROLE]) # API request
+ >>> allowed == [VIEWER_ROLE, EDITOR_ROLE]
  True

gcloud/bigquery/query.py

@@ -50,6 +50,7 @@ def __init__(self, query, client):
  self._properties = {}
  self.query = query
  self._configuration = _SyncQueryConfiguration()
+ self._job = None
 
  @property
  def project(self):
@@ -134,9 +135,12 @@ def job(self):
  :returns: Job instance used to run the query (None until
  ``jobReference`` property is set by the server).
  """
- job_ref = self._properties.get('jobReference')
- if job_ref is not None:
- return QueryJob(job_ref['jobId'], self.query, self._client)
+ if self._job is None:
+ job_ref = self._properties.get('jobReference')
+ if job_ref is not None:
+ self._job = QueryJob(job_ref['jobId'], self.query,
+ self._client)
+ return self._job
 
  @property
  def page_token(self):

gcloud/bigquery/test_query.py

@@ -156,6 +156,8 @@ def test_job_w_jobid(self):
  self.assertEqual(job.query, self.QUERY)
  self.assertTrue(job._client is client)
  self.assertEqual(job.name, SERVER_GENERATED)
+ fetched_later = query.job
+ self.assertTrue(fetched_later is job)
 
  def test_schema(self):
  client = _Client(self.PROJECT)

gcloud/dns/client.py

@@ -55,8 +55,9 @@ def quotas(self):
  """
  path = '/projects/%s' % (self.project,)
  resp = self.connection.api_request(method='GET', path=path)
+
  return dict([(key, int(value))
- for key, value in resp['quota'].items()])
+ for key, value in resp['quota'].items() if key != 'kind'])
 
  def list_zones(self, max_results=None, page_token=None):
  """List zones for the project associated with this client.

gcloud/dns/test_client.py

@@ -68,6 +68,42 @@ def test_quotas_defaults(self):
  self.assertEqual(req['method'], 'GET')
  self.assertEqual(req['path'], '/%s' % PATH)
 
+ def test_quotas_w_kind_key(self):
+ PROJECT = 'PROJECT'
+ PATH = 'projects/%s' % PROJECT
+ MANAGED_ZONES = 1234
+ RRS_PER_RRSET = 23
+ RRSETS_PER_ZONE = 345
+ RRSET_ADDITIONS = 456
+ RRSET_DELETIONS = 567
+ TOTAL_SIZE = 67890
+ DATA = {
+ 'quota': {
+ 'managedZones': str(MANAGED_ZONES),
+ 'resourceRecordsPerRrset': str(RRS_PER_RRSET),
+ 'rrsetsPerManagedZone': str(RRSETS_PER_ZONE),
+ 'rrsetAdditionsPerChange': str(RRSET_ADDITIONS),
+ 'rrsetDeletionsPerChange': str(RRSET_DELETIONS),
+ 'totalRrdataSizePerChange': str(TOTAL_SIZE),
+ }
+ }
+ CONVERTED = dict([(key, int(value))
+ for key, value in DATA['quota'].items()])
+ WITH_KIND = {'quota': DATA['quota'].copy()}
+ WITH_KIND['quota']['kind'] = 'dns#quota'
+ creds = _Credentials()
+ client = self._makeOne(PROJECT, creds)
+ conn = client.connection = _Connection(WITH_KIND)
+
+ quotas = client.quotas()
+
+ self.assertEqual(quotas, CONVERTED)
+
+ self.assertEqual(len(conn._requested), 1)
+ req = conn._requested[0]
+ self.assertEqual(req['method'], 'GET')
+ self.assertEqual(req['path'], '/%s' % PATH)
+
  def test_list_zones_defaults(self):
  from gcloud.dns.zone import ManagedZone
  PROJECT = 'PROJECT'

gcloud/pubsub/iam.py

@@ -16,10 +16,10 @@
 OWNER_ROLE = 'roles/owner'
 """IAM permission implying all rights to an object."""
 
-WRITER_ROLE = 'roles/writer'
+EDITOR_ROLE = 'roles/editor'
 """IAM permission implying rights to modify an object."""
 
-READER_ROLE = 'roles/reader'
+VIEWER_ROLE = 'roles/viewer'
 """IAM permission implying rights to access an object without modifying it."""
 
 
@@ -40,8 +40,8 @@ def __init__(self, etag=None, version=None):
  self.etag = etag
  self.version = version
  self.owners = set()
- self.writers = set()
- self.readers = set()
+ self.editors = set()
+ self.viewers = set()
 
  @staticmethod
  def user(email):
@@ -127,10 +127,10 @@ def from_api_repr(cls, resource):
  members = set(binding['members'])
  if role == OWNER_ROLE:
  policy.owners = members
- elif role == WRITER_ROLE:
- policy.writers = members
- elif role == READER_ROLE:
- policy.readers = members
+ elif role == EDITOR_ROLE:
+ policy.editors = members
+ elif role == VIEWER_ROLE:
+ policy.viewers = members
  else:
  raise ValueError('Unknown role: %s' % (role,))
  return policy
@@ -155,13 +155,13 @@ def to_api_repr(self):
  bindings.append(
  {'role': OWNER_ROLE, 'members': sorted(self.owners)})
 
- if self.writers:
+ if self.editors:
  bindings.append(
- {'role': WRITER_ROLE, 'members': sorted(self.writers)})
+ {'role': EDITOR_ROLE, 'members': sorted(self.editors)})
 
- if self.readers:
+ if self.viewers:
  bindings.append(
- {'role': READER_ROLE, 'members': sorted(self.readers)})
+ {'role': VIEWER_ROLE, 'members': sorted(self.viewers)})
 
  if bindings:
  resource['bindings'] = bindings

gcloud/pubsub/subscription.py

@@ -305,8 +305,9 @@ def set_iam_policy(self, policy, client=None):
  client = self._require_client(client)
  path = '%s:setIamPolicy' % (self.path,)
  resource = policy.to_api_repr()
+ wrapped = {'policy': resource}
  resp = client.connection.api_request(
- method='POST', path=path, data=resource)
+ method='POST', path=path, data=wrapped)
  return Policy.from_api_repr(resp)
 
  def check_iam_permissions(self, permissions, client=None):

gcloud/pubsub/test_iam.py

@@ -29,8 +29,8 @@ def test_ctor_defaults(self):
  self.assertEqual(policy.etag, None)
  self.assertEqual(policy.version, None)
  self.assertEqual(list(policy.owners), [])
- self.assertEqual(list(policy.writers), [])
- self.assertEqual(list(policy.readers), [])
+ self.assertEqual(list(policy.editors), [])
+ self.assertEqual(list(policy.viewers), [])
 
  def test_ctor_explicit(self):
  VERSION = 17
@@ -39,8 +39,8 @@ def test_ctor_explicit(self):
  self.assertEqual(policy.etag, ETAG)
  self.assertEqual(policy.version, VERSION)
  self.assertEqual(list(policy.owners), [])
- self.assertEqual(list(policy.writers), [])
- self.assertEqual(list(policy.readers), [])
+ self.assertEqual(list(policy.editors), [])
+ self.assertEqual(list(policy.viewers), [])
 
  def test_user(self):
  EMAIL = 'phred@example.com'
@@ -83,33 +83,33 @@ def test_from_api_repr_only_etag(self):
  self.assertEqual(policy.etag, 'ACAB')
  self.assertEqual(policy.version, None)
  self.assertEqual(list(policy.owners), [])
- self.assertEqual(list(policy.writers), [])
- self.assertEqual(list(policy.readers), [])
+ self.assertEqual(list(policy.editors), [])
+ self.assertEqual(list(policy.viewers), [])
 
  def test_from_api_repr_complete(self):
- from gcloud.pubsub.iam import OWNER_ROLE, WRITER_ROLE, READER_ROLE
+ from gcloud.pubsub.iam import OWNER_ROLE, EDITOR_ROLE, VIEWER_ROLE
  OWNER1 = 'user:phred@example.com'
  OWNER2 = 'group:cloud-logs@google.com'
- WRITER1 = 'domain:google.com'
- WRITER2 = 'user:phred@example.com'
- READER1 = 'serviceAccount:1234-abcdef@service.example.com'
- READER2 = 'user:phred@example.com'
+ EDITOR1 = 'domain:google.com'
+ EDITOR2 = 'user:phred@example.com'
+ VIEWER1 = 'serviceAccount:1234-abcdef@service.example.com'
+ VIEWER2 = 'user:phred@example.com'
  RESOURCE = {
  'etag': 'DEADBEEF',
  'version': 17,
  'bindings': [
  {'role': OWNER_ROLE, 'members': [OWNER1, OWNER2]},
- {'role': WRITER_ROLE, 'members': [WRITER1, WRITER2]},
- {'role': READER_ROLE, 'members': [READER1, READER2]},
+ {'role': EDITOR_ROLE, 'members': [EDITOR1, EDITOR2]},
+ {'role': VIEWER_ROLE, 'members': [VIEWER1, VIEWER2]},
  ],
  }
  klass = self._getTargetClass()
  policy = klass.from_api_repr(RESOURCE)
  self.assertEqual(policy.etag, 'DEADBEEF')
  self.assertEqual(policy.version, 17)
  self.assertEqual(sorted(policy.owners), [OWNER2, OWNER1])
- self.assertEqual(sorted(policy.writers), [WRITER1, WRITER2])
- self.assertEqual(sorted(policy.readers), [READER1, READER2])
+ self.assertEqual(sorted(policy.editors), [EDITOR1, EDITOR2])
+ self.assertEqual(sorted(policy.viewers), [VIEWER1, VIEWER2])
 
  def test_from_api_repr_bad_role(self):
  BOGUS1 = 'user:phred@example.com'
@@ -134,27 +134,27 @@ def test_to_api_repr_only_etag(self):
  self.assertEqual(policy.to_api_repr(), {'etag': 'DEADBEEF'})
 
  def test_to_api_repr_full(self):
- from gcloud.pubsub.iam import OWNER_ROLE, WRITER_ROLE, READER_ROLE
+ from gcloud.pubsub.iam import OWNER_ROLE, EDITOR_ROLE, VIEWER_ROLE
  OWNER1 = 'group:cloud-logs@google.com'
  OWNER2 = 'user:phred@example.com'
- WRITER1 = 'domain:google.com'
- WRITER2 = 'user:phred@example.com'
- READER1 = 'serviceAccount:1234-abcdef@service.example.com'
- READER2 = 'user:phred@example.com'
+ EDITOR1 = 'domain:google.com'
+ EDITOR2 = 'user:phred@example.com'
+ VIEWER1 = 'serviceAccount:1234-abcdef@service.example.com'
+ VIEWER2 = 'user:phred@example.com'
  EXPECTED = {
  'etag': 'DEADBEEF',
  'version': 17,
  'bindings': [
  {'role': OWNER_ROLE, 'members': [OWNER1, OWNER2]},
- {'role': WRITER_ROLE, 'members': [WRITER1, WRITER2]},
- {'role': READER_ROLE, 'members': [READER1, READER2]},
+ {'role': EDITOR_ROLE, 'members': [EDITOR1, EDITOR2]},
+ {'role': VIEWER_ROLE, 'members': [VIEWER1, VIEWER2]},
  ],
  }
  policy = self._makeOne('DEADBEEF', 17)
  policy.owners.add(OWNER1)
  policy.owners.add(OWNER2)
- policy.writers.add(WRITER1)
- policy.writers.add(WRITER2)
- policy.readers.add(READER1)
- policy.readers.add(READER2)
+ policy.editors.add(EDITOR1)
+ policy.editors.add(EDITOR2)
+ policy.viewers.add(VIEWER1)
+ policy.viewers.add(VIEWER2)
  self.assertEqual(policy.to_api_repr(), EXPECTED)