Implemented automatic caching for the discovery documents.

googleapiclient/discovery.py

@@ -149,7 +149,9 @@ def build(serviceName,
  developerKey=None,
  model=None,
  requestBuilder=HttpRequest,
- credentials=None):
+ credentials=None,
+ cache_discovery=True,
+ cache=None):
  """Construct a Resource for interacting with an API.
 
  Construct a Resource object for interacting with an API. The serviceName and
@@ -171,6 +173,9 @@ def build(serviceName,
  request.
  credentials: oauth2client.Credentials, credentials to be used for
  authentication.
+ cache_discovery: Boolean, whether or not to cache the discovery doc.
+ cache: googleapiclient.discovery_cache.base.CacheBase, an optional
+ cache object for the discovery documents.
 
  Returns:
  A Resource object with methods for interacting with the service.
@@ -185,22 +190,60 @@ def build(serviceName,
 
  requested_url = uritemplate.expand(discoveryServiceUrl, params)
 
+ content = retrieve_discovery_doc(requested_url, http, cache_discovery, cache)
+
+ return build_from_document(content, base=discoveryServiceUrl, http=http,
+ developerKey=developerKey, model=model, requestBuilder=requestBuilder,
+ credentials=credentials)
+
+
+def retrieve_discovery_doc(url, http, cache_discovery, cache=None):
+ """Retrieves the discovery_doc from cache or the internet.
+
+ Args:
+ url: string, the URL of the discovery document.
+ http: httplib2.Http, An instance of httplib2.Http or something that acts
+ like it that HTTP requests will be made through.
+ cache_discovery: Boolean, whether or not to cache the discovery doc.
+ cache: googleapiclient.discovery_cache.base.CacheBase, an optional
+ cache object for the discovery documents.
+
+ Returns:
+ A unicode string representation of the discovery document.
+ """
+ if http is None:
+ http = httplib2.Http()
+
+ if cache_discovery:
+ from . import discovery_cache
+ from .discovery_cache import base
+ if cache is None:
+ cache = discovery_cache.autodetect()
+ if cache:
+ if isinstance(cache, base.CacheBase):
+ content = cache.get(url)
+ if content:
+ return content
+ else:
+ logging.warning('The given cache object is not an instance of '
+ 'googleapiclient.discovery_cache.base.CacheBase.')
+
+ actual_url = url
  # REMOTE_ADDR is defined by the CGI spec [RFC3875] as the environment
  # variable that contains the network address of the client sending the
  # request. If it exists then add that to the request for the discovery
  # document to avoid exceeding the quota on discovery requests.
  if 'REMOTE_ADDR' in os.environ:
- requested_url = _add_query_parameter(requested_url, 'userIp',
- os.environ['REMOTE_ADDR'])
- logger.info('URL being requested: GET %s' % requested_url)
+ actual_url = _add_query_parameter(url, 'userIp', os.environ['REMOTE_ADDR'])
+ logger.info('URL being requested: GET %s' % actual_url)
 
- resp, content = http.request(requested_url)
+ resp, content = http.request(actual_url)
 
  if resp.status == 404:
  raise UnknownApiNameOrVersion("name: %s version: %s" % (serviceName,
- version))
+  version))
  if resp.status >= 400:
- raise HttpError(resp, content, uri=requested_url)
+ raise HttpError(resp, content, uri=actual_url)
 
  try:
  content = content.decode('utf-8')
@@ -212,10 +255,9 @@ def build(serviceName,
  except ValueError as e:
  logger.error('Failed to parse as JSON: ' + content)
  raise InvalidJsonError()
-
- return build_from_document(content, base=discoveryServiceUrl, http=http,
- developerKey=developerKey, model=model, requestBuilder=requestBuilder,
- credentials=credentials)
+ if cache_discovery and cache and isinstance(cache, base.CacheBase):
+ cache.set(url, content)
+ return content
 
 
 @positional(1)

googleapiclient/discovery_cache/__init__.py

@@ -0,0 +1,42 @@
+# Copyright 2014 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Caching utility for the discovery document."""
+
+
+from __future__ import absolute_import
+
+import logging
+import datetime
+
+
+DISCOVERY_DOC_MAX_AGE = 60 * 60 * 24 # 1 day
+
+
+def autodetect():
+ """Detects an appropriate cache module and returns it.
+
+ Returns:
+ An object with the following methods; get(url), set(url, content).
+ """
+ try:
+ from google.appengine.api import memcache
+ from . import appengine_memcache
+ return appengine_memcache.Cache(max_age = DISCOVERY_DOC_MAX_AGE)
+ except:
+ try:
+ from . import file_cache
+ return file_cache.Cache(max_age = DISCOVERY_DOC_MAX_AGE)
+ except Exception as e:
+ logging.warning(e, exc_info=True)

googleapiclient/discovery_cache/appengine_memcache.py

@@ -0,0 +1,64 @@
+# Copyright 2014 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""App Engine memcache based cache for the discovery document."""
+
+
+import logging
+
+# This is only an optional dependency because we only import this
+# module when google.appengine.api.memcache is available.
+from google.appengine.api import memcache
+
+from . import base
+
+NAMESPACE = 'google-api-client'
+
+
+class Cache(base.CacheBase):
+ """A cache with app engine memcache API.
+
+ Attributes:
+ max_age: Cache expiration in seconds.
+ """
+ def __init__(self, max_age):
+ """Constructor for appengine_memcache.Cache.
+
+ Args:
+ max_age: Cache expiration in seconds.
+ """
+ self.max_age = max_age
+
+ def get(self, url):
+ """Gets the content from the memcache with a given key.
+
+ Args:
+ url: string, the key for the cache.
+ """
+ try:
+ return memcache.get(url, namespace=NAMESPACE)
+ except Exception as e:
+ logging.warning(e, exc_info=True)
+
+ def set(self, url, content):
+ """Sets the given key and content in the cache.
+
+ Args:
+ url: string, the key for the cache.
+ content: string, the discovery document.
+ """
+ try:
+ memcache.set(url, content, time=int(self.max_age), namespace=NAMESPACE)
+ except Exception as e:
+ logging.warning(e, exc_info=True)

googleapiclient/discovery_cache/base.py

@@ -0,0 +1,42 @@
+# Copyright 2014 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""An abstract class for caching the discovery document."""
+
+
+import abc
+
+
+class CacheBase(object):
+ """A base abstract cache class."""
+ __metaclass__ = abc.ABCMeta
+
+ @abc.abstractmethod 
+ def get(self, url):
+ """Gets the content from the memcache with a given key.
+
+ Args:
+ url: string, the key for the cache.
+ """
+ return
+
+ @abc.abstractmethod 
+ def set(self, url, content):
+ """Sets the given key and content in the cache.
+
+ Args:
+ url: string, the key for the cache.
+ content: string, the discovery document.
+ """
+ return

googleapiclient/discovery_cache/file_cache.py

@@ -0,0 +1,89 @@
+# Copyright 2014 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""File based cache for the discovery document."""
+
+
+import datetime
+import json
+import logging
+import os
+import tempfile
+import threading
+
+from . import base
+
+
+FILENAME = 'discovery-doc.cache'
+EPOCH = datetime.datetime.utcfromtimestamp(0)
+
+
+def to_timestamp(d):
+ return (d - EPOCH).total_seconds()
+
+
+class Cache(base.CacheBase):
+ """A file based cache for the discovery documents.
+
+ Attributes:
+ max_age: Cache expiration in seconds.
+ lock: threading.Lock, to make the operations thread safe.
+ cache_file: string, full path to the cache file.
+ """
+ def __init__(self, max_age):
+ """Constructor for appengine_memcache.Cache.
+
+ Args:
+ max_age: Cache expiration in seconds.
+ """
+ self.max_age = max_age
+ self.lock = threading.Lock()
+ self.cache_file = os.path.join(tempfile.gettempdir(), FILENAME)
+ if not os.path.isfile(self.cache_file):
+ with open(self.cache_file, 'w+') as f:
+ json.dump({}, f)
+
+ def get(self, url):
+ """Gets the content from the memcache with a given key.
+
+ Args:
+ url: string, the key for the cache.
+ """
+ try:
+ with self.lock:
+ with open(self.cache_file, 'r') as f:
+ cache = json.load(f)
+ if url in cache:
+ content, t = cache.get(url, (None, 0))
+ if to_timestamp(datetime.datetime.now()) < t + self.max_age):
+ return content
+ except Exception as e:
+ logging.warning(e, exc_info=True)
+
+ def set(self, url, content):
+ """Sets the given key and content in the cache.
+
+ Args:
+ url: string, the key for the cache.
+ content: string, the discovery document.
+ """
+ try:
+ with self.lock:
+ with open(self.cache_file, 'r+') as f:
+ cache = json.load(f)
+ cache[url] = (content, to_timestamp(datetime.datetime.now()))
+ f.seek(0)
+ json.dump(cache, f)
+ except Exception as e:
+ logging.warning(e, exc_info=True)

tests/test_discovery.py

@@ -338,7 +338,7 @@ def test_tests_should_be_run_with_strict_positional_enforcement(self):
  def test_failed_to_parse_discovery_json(self):
  self.http = HttpMock(datafile('malformed.json'), {'status': '200'})
  try:
- plus = build('plus', 'v1', http=self.http)
+ plus = build('plus', 'v1', http=self.http, cache_discovery=False)
  self.fail("should have raised an exception over malformed JSON.")
  except InvalidJsonError:
  pass
@@ -548,7 +548,7 @@ def test_tunnel_patch(self):
  ({'status': '200'}, 'echo_request_headers_as_json'),
  ])
  http = tunnel_patch(http)
- zoo = build('zoo', 'v1', http=http)
+ zoo = build('zoo', 'v1', http=http, cache_discovery=False)
  resp = zoo.animals().patch(
  name='lion', body='{"description": "foo"}').execute()