From 387519a86207d3711cf5bae5afc88189d9608d5a Mon Sep 17 00:00:00 2001
From: Jon Wayne Parrott <jonwayne@google.com>
Date: Wed, 3 Aug 2016 13:14:27 -0700
Subject: [PATCH] Change the way nox handles installing requirements.

Instead of running pip like 100 times to install dependencies for all samples,
there's a new `nox -s reqrollup` that collates everything into one file. Pip
is now only called once. This should somewhat speed up builds.

Change-Id: Ie836803a2301484efb3275d5577967495ea59e49
---
 nox.py                         | 48 ++++++++++++++++++++++++++++-----
 requirements-dev.in            | 13 +++++++++
 requirements-dev.txt           | 49 ++++++++++++++++++++++++++++++++++
 requirements-python2.7-dev.txt | 31 ---------------------
 requirements-python3.4-dev.txt | 29 --------------------
 5 files changed, 103 insertions(+), 67 deletions(-)
 create mode 100644 requirements-dev.in
 create mode 100644 requirements-dev.txt
 delete mode 100644 requirements-python2.7-dev.txt
 delete mode 100644 requirements-python3.4-dev.txt

diff --git a/nox.py b/nox.py
index e183ddc75ca..d6458265b7b 100644
--- a/nox.py
+++ b/nox.py
@@ -57,6 +57,10 @@
 APPENGINE_BLACKLIST = set()
 
 
+# Libraries that only work on Python 2.7
+PY27_ONLY_LIBRARIES = ['mysql-python']
+
+
 def list_files(folder, pattern):
     """Lists all files below the given folder that match the pattern."""
     for root, folders, files in os.walk(folder):
@@ -145,7 +149,7 @@ def run_tests_in_sesssion(
 
     It:
     1. Install the common testing utilities.
-    2. Installs the test requirements for the current interpreter.
+    2. Installs the test requirements.
     3. Determines which pytest arguments to use. skip_flaky causes extra
        arguments to be passed that will skip tests marked flaky.
     4. If posargs are specified, it will use that as the list of samples to
@@ -159,7 +163,7 @@ def run_tests_in_sesssion(
     """
     session.interpreter = interpreter
     session.install(REPO_TOOLS_REQ)
-    session.install('-r', 'requirements-{}-dev.txt'.format(interpreter))
+    session.install('-r', 'requirements-dev.txt')
 
     if use_appengine:
         setup_appengine(session)
@@ -184,11 +188,6 @@ def run_tests_in_sesssion(
         print('\n'.join(sample_directories))
 
     for sample in sample_directories:
-        # Install additional dependencies if they exist
-        dirname = sample if os.path.isdir(sample) else os.path.dirname(sample)
-        for reqfile in list_files(dirname, 'requirements*.txt'):
-            session.install('-r', reqfile)
-
         # Ignore lib and env directories
         ignore_args = [
             '--ignore', os.path.join(sample, 'lib'),
@@ -267,3 +266,38 @@ def session_reqcheck(session):
 
     for reqfile in list_files('.', 'requirements*.txt'):
         session.run('gcprepotools', command, reqfile)
+
+
+def session_reqrollup(session):
+    """Rolls up all requirements files into requirements-dev.txt.
+
+    This does not test for uniqueness. pip itself will validate that.
+    """
+    requirements = set()
+    requirements_files = list(list_files('.', 'requirements*.txt'))
+    requirements_files.append('./requirements-dev.in')
+
+    for filename in requirements_files:
+        if filename == './requirements-dev.txt':
+            continue
+
+        with open(filename, 'r') as f:
+            lines = f.readlines()
+        requirements.update(lines)
+
+    def mark_if_necessary(requirement):
+        """Adds environment markers to Python 2.7-only libraries."""
+        for library in PY27_ONLY_LIBRARIES:
+            if requirement.startswith(library):
+                return '{}; python_version == \'2.7\'\n'.format(
+                    requirement.strip())
+        return requirement
+
+    requirements = [
+        mark_if_necessary(requirement) for requirement in requirements]
+
+    with open('requirements-dev.txt', 'w') as f:
+        f.write('# This file is generated by nox -s reqrollup. Do not edit.\n')
+        for requirement in sorted(requirements, key=lambda s: s.lower()):
+            if not requirement.startswith('#'):
+                f.write(requirement)
diff --git a/requirements-dev.in b/requirements-dev.in
new file mode 100644
index 00000000000..ac597efa206
--- /dev/null
+++ b/requirements-dev.in
@@ -0,0 +1,13 @@
+# These requirements are compiled by nox -s reqrollup to requirements-dev.txt.
+beautifulsoup4==4.5.1
+coverage==4.2
+flaky==3.3.0
+funcsigs==1.0.2
+mock==2.0.0
+mysql-python==1.2.5
+PyCrypto==2.6.1
+pytest-cov==2.3.0
+pytest==2.9.2
+PyYAML==3.11
+responses==0.5.1
+WebTest==2.0.23
diff --git a/requirements-dev.txt b/requirements-dev.txt
new file mode 100644
index 00000000000..6900a7804b1
--- /dev/null
+++ b/requirements-dev.txt
@@ -0,0 +1,49 @@
+# This file is generated by nox -s reqrollup. Do not edit.
+beautifulsoup4==4.5.1
+coverage==4.2
+cryptography==1.4
+Django==1.10
+flaky==3.3.0
+flask-cors==2.1.2
+Flask-Sockets==0.2.1
+Flask-SQLAlchemy==2.1
+Flask==0.11.1
+fluent-logger==0.4.3
+funcsigs==1.0.2
+gcloud==0.17.0
+gcloud[grpc]==0.17.0
+google-api-python-client==1.5.1
+grpc-google-cloud-speech-v1beta1==1.0.1
+grpcio==1.0.0rc1
+gunicorn==19.6.0
+httplib2==0.9.2
+kinto==3.3.2
+mailjet-rest==v1.2.2
+mock==2.0.0
+mysql-python==1.2.5; python_version == '2.7'
+mysqlclient==1.3.7
+oauth2client==3.0.0
+Pillow==3.3.0
+pyasn1-modules==0.0.8
+pyasn1==0.1.9
+PyAudio==0.2.9
+PyCrypto==2.6.1
+pymemcache==1.3.6
+PyMySQL==0.7.6
+pytest-cov==2.3.0
+pytest==2.9.2
+pyyaml==3.11
+PyYAML==3.11
+redis==2.10.5
+requests-toolbelt==0.7.0
+requests==2.10.0
+requests[security]==2.10.0
+responses==0.5.1
+rsa==3.4.2
+sendgrid==3.1.10
+simplejson==3.8.2
+six==1.10.0
+twilio==6.3.dev0
+uritemplate==0.6
+WebTest==2.0.23
+wheel==0.29.0
diff --git a/requirements-python2.7-dev.txt b/requirements-python2.7-dev.txt
deleted file mode 100644
index a0455778e9b..00000000000
--- a/requirements-python2.7-dev.txt
+++ /dev/null
@@ -1,31 +0,0 @@
-beautifulsoup4==4.5.1
-coverage==4.2
-Django==1.10
-flaky==3.3.0
-Flask-Sockets==0.2.1
-Flask-SQLAlchemy==2.1
-Flask==0.11.1
-funcsigs==1.0.2
-gcloud==0.17.0
-google-api-python-client==1.5.1
-itsdangerous==0.24
-Jinja2==2.8
-MarkupSafe==0.23
-mock==2.0.0
-mysql-python==1.2.5
-oauth2client==3.0.0
-pbr==1.10.0
-PyCrypto==2.6.1
-pymemcache==1.3.6
-PyMySQL==0.7.6
-pytest-cov==2.3.0
-pytest==2.9.2
-PyYAML==3.11
-requests[security]==2.10.0
-responses==0.5.1
-sendgrid==3.1.10
-twilio==6.3.dev0
-waitress==1.0a2
-WebOb==1.6.1
-WebTest==2.0.23
-Werkzeug==0.11.10
diff --git a/requirements-python3.4-dev.txt b/requirements-python3.4-dev.txt
deleted file mode 100644
index fa653f45250..00000000000
--- a/requirements-python3.4-dev.txt
+++ /dev/null
@@ -1,29 +0,0 @@
-beautifulsoup4==4.5.1
-coverage==4.2
-Django==1.10
-flaky==3.3.0
-Flask-SQLAlchemy==2.1
-Flask==0.11.1
-funcsigs==1.0.2
-gcloud==0.17.0
-google-api-python-client==1.5.1
-itsdangerous==0.24
-Jinja2==2.8
-MarkupSafe==0.23
-mock==2.0.0
-oauth2client==3.0.0
-pbr==1.10.0
-PyCrypto==2.6.1
-pymemcache==1.3.6
-PyMySQL==0.7.6
-pytest-cov==2.3.0
-pytest==2.9.2
-PyYAML==3.11
-requests[security]==2.10.0
-responses==0.5.1
-sendgrid==3.1.10
-twilio==6.3.dev0
-waitress==1.0a2
-WebOb==1.6.1
-WebTest==2.0.23
-Werkzeug==0.11.10