8000 Change the way nox handles installing requirements. (#438) · asndev/python-docs-samples@19299d3 · GitHub
[go: up one dir, main page]
Skip to content

Commit 19299d3

Browse files
author
Jon Wayne Parrott
authored
Change the way nox handles installing requirements. (GoogleCloudPlatform#438)
Instead of running pip like 100 times to install dependencies for all samples, there's a new `nox -s reqrollup` that collates everything into one file. Pip is now only called once. This should somewhat speed up builds.
1 parent 330ed3b commit 19299d3

File tree

5 files changed

+103
-67
lines changed

5 files changed

+103
-67
lines changed

‎nox.py

Lines changed: 41 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@
5757
APPENGINE_BLACKLIST = set()
5858

5959

60+
# Libraries that only work on Python 2.7
61+
PY27_ONLY_LIBRARIES = ['mysql-python']
62+
63+
6064
def list_files(folder, pattern):
6165
"""Lists all files below the given folder that match the pattern."""
6266
for root, folders, files in os.walk(folder):
@@ -145,7 +149,7 @@ def run_tests_in_sesssion(
145149
146150
It:
147151
1. Install the common testing utilities.
148-
2. Installs the test requirements for the current interpreter.
152+
2. Installs the test requirements.
149153
3. Determines which pytest arguments to use. skip_flaky causes extra
150154
arguments to be passed that will skip tests marked flaky.
151155
4. If posargs are specified, it will use that as the list of samples to
@@ -159,7 +163,7 @@ def run_tests_in_sesssion(
159163
"""
160164
session.interpreter = interpreter
161165
session.install(REPO_TOOLS_REQ)
162-
session.install('-r', 'requirements-{}-dev.txt'.format(interpreter))
166+
session.install('-r', 'requirements-dev.txt')
163167

164168
if use_appengine:
165169
setup_appengine(session)
@@ -184,11 +188,6 @@ def run_tests_in_sesssion(
184188
print('\n'.join(sample_directories))
185189

186190
for sample in sample_directories:
187-
# Install additional dependencies if they exist
188-
dirname = sample if os.path.isdir(sample) else os.path.dirname(sample)
189-
for reqfile in list_files(dirname, 'requirements*.txt'):
190-
session.install('-r', reqfile)
191-
192191
# Ignore lib and env directories
193192
ignore_args = [
194193
'--ignore', os.path.join(sample, 'lib'),
@@ -267,3 +266,38 @@ def session_reqcheck(session):
267266

268267
for reqfile in list_files('.', 'requirements*.txt'):
269268
session.run('gcprepotools', command, reqfile)
269+
270+
271+
def session_reqrollup(session):
272+
"""Rolls up all requirements files into requirements-dev.txt.
273+
274+
This does not test for uniqueness. pip itself will validate that.
275+
"""
276+
requirements = set()
277+
requirements_files = list(list_files('.', 'requirements*.txt'))
278+
requirements_files.append('./requirements-dev.in')
279+
280+
for filename in requirements_files:
281+
if filename == './requirements-dev.txt':
282+
continue
283+
284+
with open(filename, 'r') as f:
285+
lines = f.readlines()
286+
requirements.update(lines)
287+
288+
def mark_if_necessary(requirement):
289+
"""Adds environment markers to Python 2.7-only libraries."""
290+
for library in PY27_ONLY_LIBRARIES:
291+
if requirement.startswith(library):
292+
return '{}; python_version == \'2.7\'\n'.format(
293+
requirement.strip())
294+
return requirement
295+
296+
requirements = [
297+
mark_if_necessary(requirement) for requirement in requirements]
298+
299+
with open('requirements-dev.txt', 'w') as f:
300+
f.write('# This file is generated by nox -s reqrollup. Do not edit.\n')
301+
for requirement in sorted(requirements, key=lambda s: s.lower()):
302+
if not requirement.startswith('#'):
303+
f.write(requirement)

‎requirements-dev.in

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# These requirements are compiled by nox -s reqrollup to requirements-dev.txt.
2+
beautifulsoup4==4.5.1
3+
coverage==4.2
4+
flaky==3.3.0
5+
funcsigs==1.0.2
6+
mock==2.0.0
7+
mysql-python==1.2.5
8+
PyCrypto==2.6.1
9+
pytest-cov==2.3.0
10+
pytest==2.9.2
11+
PyYAML==3.11
12+
responses==0.5.1
13+
WebTest==2.0.23

‎requirements-dev.txt

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
# This file is generated by nox -s reqrollup. Do not edit.
2+
beautifulsoup4==4.5.1
3+
coverage==4.2
4+
cryptography==1.4
5+
Django==1.10
6+
flaky==3.3.0
7+
flask-cors==2.1.2
8+
Flask-Sockets==0.2.1
9+
Flask-SQLAlchemy==2.1
10+
Flask==0.11.1
11+
fluent-logger==0.4.3
12+
funcsigs==1.0.2
13+
gcloud==0.17.0
14+
gcloud[grpc]==0.17.0
15+
google-api-python-client==1.5.1
16+
grpc-google-cloud-speech-v1beta1==1.0.1
17+
grpcio==1.0.0rc1
18+
gunicorn==19.6.0
19+
httplib2==0.9.2
20+
kinto==3.3.2
21+
mailjet-rest==v1.2.2
22+
mock==2.0.0
23+
mysql-python==1.2.5; python_version == '2.7'
24+
mysqlclient==1.3.7
25+
oauth2client==3.0.0
26+
Pillow==3.3.0
27+
pyasn1-modules==0.0.8
28+
pyasn1==0.1.9
29+
PyAudio==0.2.9
30+
PyCrypto==2.6.1
31+
pymemcache==1.3.6
32+
PyMySQL==0.7.6
33+
pytest-cov==2.3.0
34+
pytest==2.9.2
35+
pyyaml==3.11
36+
PyYAML==3.11
37+
redis==2.10.5
38+
requests-toolbelt==0.7.0
39+
requests==2.10.0
40+
requests[security]==2.10.0
41+
responses==0.5.1
42+
rsa==3.4.2
43+
sendgrid==3.1.10
44+
simplejson==3.8.2
45+
six==1.10.0
46+
twilio==6.3.dev0
47+
uritemplate==0.6
48+
WebTest==2.0.23
49+
wheel==0.29.0

‎requirements-python2.7-dev.txt

Lines changed: 0 additions & 31 deletions
This file was deleted.

‎requirements-python3.4-dev.txt

Lines changed: 0 additions & 29 deletions
This file was deleted.

0 commit comments

Comments
 (0)
0