Change the way nox handles installing requirements.

nox.py

@@ -57,6 +57,10 @@
 APPENGINE_BLACKLIST = set()
 
 
+# Libraries that only work on Python 2.7
+PY27_ONLY_LIBRARIES = ['mysql-python']
+
+
 def list_files(folder, pattern):
     """Lists all files below the given folder that match the pattern."""
     for root, folders, files in os.walk(folder):
@@ -145,7 +149,7 @@ def run_tests_in_sesssion(
 
     It:
     1. Install the common testing utilities.
-    2. Installs the test requirements for the current interpreter.
+    2. Installs the test requirements.
     3. Determines which pytest arguments to use. skip_flaky causes extra
        arguments to be passed that will skip tests marked flaky.
     4. If posargs are specified, it will use that as the list of samples to
@@ -159,7 +163,7 @@ def run_tests_in_sesssion(
     """
     session.interpreter = interpreter
     session.install(REPO_TOOLS_REQ)
-    session.install('-r', 'requirements-{}-dev.txt'.format(interpreter))
+    session.install('-r', 'requirements-dev.txt')
 
     if use_appengine:
         setup_appengine(session)
@@ -184,11 +188,6 @@ def run_tests_in_sesssion(
         print('\n'.join(sample_directories))
 
     for sample in sample_directories:
-        # Install additional dependencies if they exist
-        dirname = sample if os.path.isdir(sample) else os.path.dirname(sample)
-        for reqfile in list_files(dirname, 'requirements*.txt'):
-            session.install('-r', reqfile)
-
         # Ignore lib and env directories
         ignore_args = [
             '--ignore', os.path.join(sample, 'lib'),
@@ -267,3 +266,38 @@ def session_reqcheck(session):
 
     for reqfile in list_files('.', 'requirements*.txt'):
         session.run('gcprepotools', command, reqfile)
+
+
+def session_reqrollup(session):
+    """Rolls up all requirements files into requirements-dev.txt.
+
+    This does not test for uniqueness. pip itself will validate that.
+    """
+    requirements = set()
+    requirements_files = list(list_files('.', 'requirements*.txt'))
+    requirements_files.append('./requirements-dev.in')
+
+    for filename in requirements_files:
+        if filename == './requirements-dev.txt':
+            continue
+
+        with open(filename, 'r') as f:
+            lines = f.readlines()
+        requirements.update(lines)
+
+    def mark_if_necessary(requirement):
+        """Adds environment markers to Python 2.7-only libraries."""
+        for library in PY27_ONLY_LIBRARIES:
+            if requirement.startswith(library):
+                return '{}; python_version == \'2.7\'\n'.format(
+                    requirement.strip())
+        return requirement
+
+    requirements = [
+        mark_if_necessary(requirement) for requirement in requirements]
+
+    with open('requirements-dev.txt', 'w') as f:
+        f.write('# This file is generated by nox -s reqrollup. Do not edit.\n')
+        for requirement in sorted(requirements, key=lambda s: s.lower()):
+            if not requirement.startswith('#'):
+                f.write(requirement)

requirements-dev.in

@@ -0,0 +1,13 @@
+# These requirements are compiled by nox -s reqrollup to requirements-dev.txt.
+beautifulsoup4==4.5.1
+coverage==4.2
+flaky==3.3.0
+funcsigs==1.0.2
+mock==2.0.0
+mysql-python==1.2.5
+PyCrypto==2.6.1
+pytest-cov==2.3.0
+pytest==2.9.2
+PyYAML==3.11
+responses==0.5.1
+WebTest==2.0.23

requirements-dev.txt

@@ -0,0 +1,49 @@
+# This file is generated by nox -s reqrollup. Do not edit.
+beautifulsoup4==4.5.1
+coverage==4.2
+cryptography==1.4
+Django==1.10
+flaky==3.3.0
+flask-cors==2.1.2
+Flask-Sockets==0.2.1
+Flask-SQLAlchemy==2.1
+Flask==0.11.1
+fluent-logger==0.4.3
+funcsigs==1.0.2
+gcloud==0.17.0
+gcloud[grpc]==0.17.0
+google-api-python-client==1.5.1
+grpc-google-cloud-speech-v1beta1==1.0.1
+grpcio==1.0.0rc1
+gunicorn==19.6.0
+httplib2==0.9.2
+kinto==3.3.2
+mailjet-rest==v1.2.2
+mock==2.0.0
+mysql-python==1.2.5; python_version == '2.7'
+mysqlclient==1.3.7
+oauth2client==3.0.0
+Pillow==3.3.0
+pyasn1-modules==0.0.8
+pyasn1==0.1.9
+PyAudio==0.2.9
+PyCrypto==2.6.1
+pymemcache==1.3.6
+PyMySQL==0.7.6
+pytest-cov==2.3.0
+pytest==2.9.2
+pyyaml==3.11
+PyYAML==3.11
+redis==2.10.5
+requests-toolbelt==0.7.0
+requests==2.10.0
+requests[security]==2.10.0
+responses==0.5.1
+rsa==3.4.2
+sendgrid==3.1.10
+simplejson==3.8.2
+six==1.10.0
+twilio==6.3.dev0
+uritemplate==0.6
+WebTest==2.0.23
+wheel==0.29.0