From fcc7a740892c99351b314e2f3247597f7a6bece8 Mon Sep 17 00:00:00 2001 From: Jon Wayne Parrott Date: Fri, 20 Nov 2015 13:52:58 -0800 Subject: [PATCH] Restoring missing requirements.txt files Originally, I had consolidated to one top-level requirements.txt because it was difficult to keep them all updated. I've remedied this with two new scripts in our repositories: * check_requirements.py which will return non-zero if any requirements are out * update_requirements.py which will automatically update all requirements in requirements.txt to the latest available version. Tox now runs check_requirements.py over all requirements.txt files in the repository during testing. There's also an additional, manually-run eviroment that does the same with update_requirements. Additionally, I updated readmes to reflect the new requirements.txt and also updated gcloud auth login -> gcloud init. Phew. --- appengine/bigquery/requirements.txt | 2 +- appengine/mailgun/requirements.txt | 2 +- appengine/ndb/transactions/requirements.txt | 8 +- appengine/storage/requirements.txt | 2 +- bigquery/README.md | 4 +- bigquery/api/requirements.txt | 1 + .../README.md | 4 +- .../requirements.txt | 1 + cloud_logging/README.md | 4 +- cloud_logging/api/requirements.txt | 1 + compute/README.md | 2 +- compute/api/requirements.txt | 1 + monitoring/README.md | 2 +- monitoring/api/requirements.txt | 1 + requirements-dev.txt | 16 ++-- requirements.txt | 16 +--- scripts/check_requirements.py | 53 +++++++++++++ scripts/update_requirements.py | 74 +++++++++++++++++++ storage/README.md | 4 +- storage/api/compose_objects.py | 2 +- storage/api/list_objects.py | 2 +- storage/api/requirements.txt | 1 + storage/transfer_service/README.md | 3 +- storage/transfer_service/requirements.txt | 1 + tox.ini | 16 +++- 25 files changed, 177 insertions(+), 46 deletions(-) create mode 100644 bigquery/api/requirements.txt create mode 100644 blog/introduction_to_data_models_in_cloud_datastore/requirements.txt create mode 100644 cloud_logging/api/requirements.txt create mode 100644 compute/api/requirements.txt create mode 100644 monitoring/api/requirements.txt create mode 100755 scripts/check_requirements.py create mode 100755 scripts/update_requirements.py create mode 100644 storage/api/requirements.txt create mode 100644 storage/transfer_service/requirements.txt diff --git a/appengine/bigquery/requirements.txt b/appengine/bigquery/requirements.txt index d8055e00cd9a..891022d5dac8 100644 --- a/appengine/bigquery/requirements.txt +++ b/appengine/bigquery/requirements.txt @@ -1 +1 @@ -google-api-python-client +google-api-python-client==1.4.2 diff --git a/appengine/mailgun/requirements.txt b/appengine/mailgun/requirements.txt index fb881ece05bf..9fd60e085e54 100644 --- a/appengine/mailgun/requirements.txt +++ b/appengine/mailgun/requirements.txt @@ -1 +1 @@ -httplib2 +httplib2==0.9.2 diff --git a/appengine/ndb/transactions/requirements.txt b/appengine/ndb/transactions/requirements.txt index a257e293a8b5..632a1efabce7 100644 --- a/appengine/ndb/transactions/requirements.txt +++ b/appengine/ndb/transactions/requirements.txt @@ -1,7 +1 @@ -# This requirements file lists all third-party dependencies for this project. -# -# Run 'pip install -r requirements.txt -t lib/' to install these dependencies -# in `lib/` subdirectory. -# -# Note: The `lib` directory is added to `sys.path` by `appengine_config.py`. -Flask==0.10 +Flask==0.10.1 diff --git a/appengine/storage/requirements.txt b/appengine/storage/requirements.txt index d8055e00cd9a..891022d5dac8 100644 --- a/appengine/storage/requirements.txt +++ b/appengine/storage/requirements.txt @@ -1 +1 @@ -google-api-python-client +google-api-python-client==1.4.2 diff --git a/bigquery/README.md b/bigquery/README.md index 0cc93f51e8f4..6b5977b69317 100644 --- a/bigquery/README.md +++ b/bigquery/README.md @@ -7,9 +7,9 @@ This section contains samples for [Google BigQuery](https://cloud.google.com/big 1. Your environment must be setup with [authentication information](https://developers.google.com/identity/protocols/application-default-credentials#howtheywork). If you're running in your local development environment and you have the [Google Cloud SDK](https://cloud.google.com/sdk/) installed, you can do this easily by running: - $ gcloud auth login + $ gcloud init -2. Install dependencies from the top-level [`requirements.txt`](../requirements.txt): +2. Install dependencies in `requirements.txt`: $ pip install -r requirements.txt diff --git a/bigquery/api/requirements.txt b/bigquery/api/requirements.txt new file mode 100644 index 000000000000..891022d5dac8 --- /dev/null +++ b/bigquery/api/requirements.txt @@ -0,0 +1 @@ +google-api-python-client==1.4.2 diff --git a/blog/introduction_to_data_models_in_cloud_datastore/README.md b/blog/introduction_to_data_models_in_cloud_datastore/README.md index 869bb57ede45..1a05cc43036c 100644 --- a/blog/introduction_to_data_models_in_cloud_datastore/README.md +++ b/blog/introduction_to_data_models_in_cloud_datastore/README.md @@ -9,12 +9,12 @@ using [Google Cloud Datastore](https://cloud.google.com/datastore). 2. [Enable the Datastore API](https://console.developers.google.com/project/_/apiui/apiview/datastore/overview). -3. Install the [Google Cloud SDK](https://cloud.google.com/sdk) and be sure to run ``gcloud auth``. +3. Install the [Google Cloud SDK](https://cloud.google.com/sdk) and be sure to run ``gcloud init``. ## Running the samples -Install dependencies from the top-level [`requirements.txt`](../../requirements.txt): +Install dependencies from `requirements.txt`: pip install -r requirements.txt diff --git a/blog/introduction_to_data_models_in_cloud_datastore/requirements.txt b/blog/introduction_to_data_models_in_cloud_datastore/requirements.txt new file mode 100644 index 000000000000..ab54eebc1149 --- /dev/null +++ b/blog/introduction_to_data_models_in_cloud_datastore/requirements.txt @@ -0,0 +1 @@ +gcloud==0.8.0 diff --git a/cloud_logging/README.md b/cloud_logging/README.md index aaf34d84c0ae..f023b77ed2cd 100644 --- a/cloud_logging/README.md +++ b/cloud_logging/README.md @@ -7,9 +7,9 @@ This section contains samples for [Google Cloud Logging](https://cloud.google.co 1. Your environment must be setup with [authentication information](https://developers.google.com/identity/protocols/application-default-credentials#howtheywork). If you're running in your local development environment and you have the [Google Cloud SDK](https://cloud.google.com/sdk/) installed, you can do this easily by running: - $ gcloud auth login + $ gcloud init -2. Install dependencies from the top-level [`requirements.txt`](../requirements.txt): +2. Install dependencies from `requirements.txt`: $ pip install -r requirements.txt diff --git a/cloud_logging/api/requirements.txt b/cloud_logging/api/requirements.txt new file mode 100644 index 000000000000..891022d5dac8 --- /dev/null +++ b/cloud_logging/api/requirements.txt @@ -0,0 +1 @@ +google-api-python-client==1.4.2 diff --git a/compute/README.md b/compute/README.md index fd002bcb9d64..f0bc60f8b0af 100644 --- a/compute/README.md +++ b/compute/README.md @@ -11,7 +11,7 @@ locally and set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable. $ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service_account.json -2. Install dependencies from the top-level [`requirements.txt`](../requirements.txt): +2. Install dependencies from `requirements.txt` $ pip install -r requirements.txt diff --git a/compute/api/requirements.txt b/compute/api/requirements.txt new file mode 100644 index 000000000000..891022d5dac8 --- /dev/null +++ b/compute/api/requirements.txt @@ -0,0 +1 @@ +google-api-python-client==1.4.2 diff --git a/monitoring/README.md b/monitoring/README.md index 98c2b42d9e31..ad73848bcc7c 100644 --- a/monitoring/README.md +++ b/monitoring/README.md @@ -11,7 +11,7 @@ locally and set the `GOOGLE_APPLICATION_CREDENTIALS` environment variable. $ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/service_account.json -2. Install dependencies from the top-level [`requirements.txt`](../requirements.txt): +2. Install dependencies from `requirements.txt`: $ pip install -r requirements.txt diff --git a/monitoring/api/requirements.txt b/monitoring/api/requirements.txt new file mode 100644 index 000000000000..891022d5dac8 --- /dev/null +++ b/monitoring/api/requirements.txt @@ -0,0 +1 @@ +google-api-python-client==1.4.2 diff --git a/requirements-dev.txt b/requirements-dev.txt index eae64a52b695..77a7629315bf 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ -beautifulsoup4==4.4.0 -coverage==3.7.1 +beautifulsoup4==4.4.1 +coverage==4.0.2 Flask==0.10.1 funcsigs==0.4 itsdangerous==0.24 @@ -8,11 +8,11 @@ MarkupSafe==0.23 mock==1.3.0 nose==1.3.7 nose-exclude==0.4.1 -nosegae==0.5.7 -pbr==1.8.0 +nosegae==0.5.8 +pbr==1.8.1 PyYAML==3.11 waitress==0.8.10 -WebOb==1.4.1 -WebTest==2.0.18 -Werkzeug==0.10.4 -nose-timer +WebOb==1.5.1 +WebTest==2.0.20 +Werkzeug==0.11.2 +nose-timer==0.5.0 diff --git a/requirements.txt b/requirements.txt index 0620888a4763..8e0901955035 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,14 +1,4 @@ -gcloud==0.7.1 +gcloud==0.8.0 google-api-python-client==1.4.2 -google-apitools==0.4.11 -httplib2==0.9.1 -oauth2client==1.5.1 -protobuf==3.0.0a1 -protorpc==0.11.1 -pyasn1==0.1.8 -pyasn1-modules==0.0.7 -pycrypto==2.6.1 -rsa==3.2 -simplejson==3.8.0 -six==1.9.0 -uritemplate==0.6 +oauth2client==1.5.2 +requests[security]==2.8.1 diff --git a/scripts/check_requirements.py b/scripts/check_requirements.py new file mode 100755 index 000000000000..ecc8295f0881 --- /dev/null +++ b/scripts/check_requirements.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python + +# Copyright (C) 2013 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Checks dependencies in requirements.txt to ensure they are the latest version. +""" + +import argparse +import sys + +from update_requirements import get_package_info, read_requirements + + +def check_req(req): + info = get_package_info(req.project_name) + newest_version = info['version'] + current_spec = req.specs[0] if req.specs else ('==', 'unspecified') + if current_spec[1] != newest_version: + return req, newest_version + + +def main(req_file): + reqs = read_requirements(req_file) + outdated_reqs = filter(None, [check_req(req) for req in reqs]) + + if outdated_reqs: + for req in outdated_reqs: + print("{} is out of date, latest version is {}".format(*req)) + sys.exit(1) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + 'requirements_file', + help='Path the the requirements.txt file to check.') + + args = parser.parse_args() + + main(args.requirements_file) diff --git a/scripts/update_requirements.py b/scripts/update_requirements.py new file mode 100755 index 000000000000..e4f76a3c5ebe --- /dev/null +++ b/scripts/update_requirements.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python + +# Copyright (C) 2013 Google Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Updates dependencies in requirements.txt to the latest version. +""" + +import argparse + +from pip.req.req_file import parse_requirements +from pkg_resources import Requirement +import requests + + +def get_package_info(package): + url = 'https://pypi.python.org/pypi/{}/json'.format(package) + r = requests.get(url) + r.raise_for_status() + return r.json()['info'] + + +def read_requirements(req_file): + return [x.req for x in parse_requirements(req_file, session={})] + + +def update_req(req): + info = get_package_info(req.project_name) + newest_version = info['version'] + current_spec = req.specs[0] if req.specs else ('==', 'unspecified') + new_spec = ('==', newest_version) + if current_spec != new_spec: + newreq = Requirement(req.unsafe_name, [new_spec], req.extras) + print('Updated {} from {} -> {}'.format( + req.project_name, + current_spec[1], + newest_version)) + return newreq + return req + + +def write_requirements(reqs, req_file): + with open(req_file, 'w') as f: + for req in reqs: + f.write('{}\n'.format(req)) + + +def main(req_file): + reqs = read_requirements(req_file) + reqs = [update_req(req) for req in reqs] + write_requirements(reqs, req_file) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + 'requirements_file', + help='Path the the requirements.txt file to update.') + + args = parser.parse_args() + + main(args.requirements_file) diff --git a/storage/README.md b/storage/README.md index f625f9f39d25..7476a656d5ad 100644 --- a/storage/README.md +++ b/storage/README.md @@ -7,9 +7,9 @@ This section contains samples for [Google Cloud Storage](https://cloud.google.co 1. Your environment must be setup with [authentication information](https://developers.google.com/identity/protocols/application-default-credentials#howtheywork). If you're running in your local development environment and you have the [Google Cloud SDK](https://cloud.google.com/sdk/) installed, you can do this easily by running: - $ gcloud auth login + $ gcloud init -2. Install dependencies from the top-level [`requirements.txt`](../requirements.txt): +2. Install dependencies from `requirements.txt`: $ pip install -r requirements.txt diff --git a/storage/api/compose_objects.py b/storage/api/compose_objects.py index d3718d893466..18fe5058fa33 100644 --- a/storage/api/compose_objects.py +++ b/storage/api/compose_objects.py @@ -42,7 +42,7 @@ def main(bucket, destination, sources): # Get the application default credentials. When running locally, these are - # available after running `gcloud auth login`. When running on compute + # available after running `gcloud init`. When running on compute # engine, these are available from the environment. credentials = GoogleCredentials.get_application_default() diff --git a/storage/api/list_objects.py b/storage/api/list_objects.py index a30eeab66644..705aab8c6371 100644 --- a/storage/api/list_objects.py +++ b/storage/api/list_objects.py @@ -35,7 +35,7 @@ def main(bucket): # [START list_bucket] # Get the application default credentials. When running locally, these are - # available after running `gcloud auth login`. When running on compute + # available after running `gcloud init`. When running on compute # engine, these are available from the environment. credentials = GoogleCredentials.get_application_default() diff --git a/storage/api/requirements.txt b/storage/api/requirements.txt new file mode 100644 index 000000000000..891022d5dac8 --- /dev/null +++ b/storage/api/requirements.txt @@ -0,0 +1 @@ +google-api-python-client==1.4.2 diff --git a/storage/transfer_service/README.md b/storage/transfer_service/README.md index 3833f62935bc..7d36a64c731e 100644 --- a/storage/transfer_service/README.md +++ b/storage/transfer_service/README.md @@ -23,8 +23,7 @@ These samples are used on the following documentation pages: storage-transfer-@partnercontent.gserviceaccount.com 1. Set up gcloud for application default credentials. 1. `gcloud components update` - 1. `gcloud auth login` - 1. `gcloud config set project PROJECT_ID` + 1. `gcloud init` 1. Install [Google API Client Library for Python](https://developers.google.com/api-client-library/python/start/installation). ## Transfer from Amazon S3 to Google Cloud Storage diff --git a/storage/transfer_service/requirements.txt b/storage/transfer_service/requirements.txt new file mode 100644 index 000000000000..891022d5dac8 --- /dev/null +++ b/storage/transfer_service/requirements.txt @@ -0,0 +1 @@ +google-api-python-client==1.4.2 diff --git a/tox.ini b/tox.ini index 2f109bdd67ea..b900e04e0c25 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] skipsdist = True -envlist = pep8, gae, py27, py34 +envlist = pep8, reqcheck, gae, py27, py34 [testenv] passenv = PYTHONPATH GOOGLE_* GCLOUD_* TEST_* TRAVIS* @@ -15,6 +15,20 @@ commonargs = --cover-branches --cover-inclusive +[testenv:reqcheck] +deps = + requests[security] +commands = + bash -c "find . -name requirements.txt |\ + xargs -n 1 ./scripts/check_requirements.py" + +[testenv:requpdate] +deps = + requests[security] +commands = + bash -c "find . -name requirements.txt |\ + xargs -n 1 ./scripts/update_requirements.py" + [testenv:gae] deps = {[testenv]deps}