forked from googleapis/google-cloud-python
-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathscript_utils.py
More file actions
345 lines (271 loc) · 11.4 KB
/
script_utils.py
File metadata and controls
345 lines (271 loc) · 11.4 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
# Copyright 2016 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Common helpers for testing scripts."""
from __future__ import print_function
import ast
import os
import subprocess
PROJECT_ROOT = os.path.abspath(
os.path.join(os.path.dirname(__file__), '..'))
LOCAL_REMOTE_ENV = 'GOOGLE_CLOUD_TESTING_REMOTE'
LOCAL_BRANCH_ENV = 'GOOGLE_CLOUD_TESTING_BRANCH'
IN_TRAVIS_ENV = 'TRAVIS'
TRAVIS_PR_ENV = 'TRAVIS_PULL_REQUEST'
TRAVIS_BRANCH_ENV = 'TRAVIS_BRANCH'
INST_REQS_KWARG = 'install_requires'
REQ_VAR = 'REQUIREMENTS'
PACKAGE_PREFIX = 'google-cloud-'
def in_travis():
"""Detect if we are running in Travis.
.. _Travis env docs: https://docs.travis-ci.com/user/\
environment-variables\
#Default-Environment-Variables
See `Travis env docs`_.
:rtype: bool
:returns: Flag indicating if we are running on Travis.
"""
return os.getenv(IN_TRAVIS_ENV) == 'true'
def in_travis_pr():
"""Detect if we are running in a pull request on Travis.
.. _Travis env docs: https://docs.travis-ci.com/user/\
environment-variables\
#Default-Environment-Variables
See `Travis env docs`_.
.. note::
This assumes we already know we are running in Travis.
:rtype: bool
:returns: Flag indicating if we are in a pull request on Travis.
"""
# NOTE: We're a little extra cautious and make sure that the
# PR environment variable is an integer.
try:
int(os.getenv(TRAVIS_PR_ENV, ''))
return True
except ValueError:
return False
def travis_branch():
"""Get the current branch of the PR.
.. _Travis env docs: https://docs.travis-ci.com/user/\
environment-variables\
#Default-Environment-Variables
See `Travis env docs`_.
.. note::
This assumes we already know we are running in Travis
during a PR.
:rtype: str
:returns: The name of the branch the current pull request is
changed against.
:raises: :class:`~exceptions.OSError` if the ``TRAVIS_BRANCH_ENV``
environment variable isn't set during a pull request
build.
"""
try:
return os.environ[TRAVIS_BRANCH_ENV]
except KeyError:
msg = ('Pull request build does not have an '
'associated branch set (via %s)') % (TRAVIS_BRANCH_ENV,)
raise OSError(msg)
def ch
8000
eck_output(*args):
"""Run a command on the operation system.
:type args: tuple
:param args: Arguments to pass to ``subprocess.check_output``.
:rtype: str
:returns: The raw STDOUT from the command (converted from bytes
if necessary).
"""
cmd_output = subprocess.check_output(args)
# On Python 3, this returns bytes (from STDOUT), so we
# convert to a string.
cmd_output = cmd_output.decode('utf-8')
# Also strip the output since it usually has a trailing newline.
return cmd_output.strip()
def rootname(filename):
"""Get the root directory that a file is contained in.
:type filename: str
:param filename: The path / name of a file.
:rtype: str
:returns: The root directory containing the file.
"""
if os.path.sep not in filename:
return ''
else:
file_root, _ = filename.split(os.path.sep, 1)
return file_root
def get_changed_packages(blob_name1, blob_name2, package_list):
"""Get a list of packages which have changed between two changesets.
:type blob_name1: str
:param blob_name1: The name of a commit hash or branch name or other
``git`` artifact.
:type blob_name2: str
:param blob_name2: The name of a commit hash or branch name or other
``git`` artifact.
:type package_list: list
:param package_list: The list of **all** valid packages with unit tests.
:rtype: list
:returns: A list of all package directories that have changed
between ``blob_name1`` and ``blob_name2``. Starts
with a list of valid packages (``package_list``)
and filters out the unchanged directories.
"""
changed_files = check_output(
'git', 'diff', '--name-only', blob_name1, blob_name2)
changed_files = changed_files.split('\n')
result = set()
for filename in changed_files:
file_root = rootname(filename)
if file_root in package_list:
result.add(file_root)
return sorted(result)
def local_diff_branch():
"""Get a remote branch to diff against in a local checkout.
Checks if the the local remote and local branch environment
variables specify a remote branch.
:rtype: str
:returns: The diffbase `{remote}/{branch}` if the environment
variables are defined. If not, returns ``None``.
"""
# Only allow specified remote and branch in local dev.
remote = os.getenv(LOCAL_REMOTE_ENV)
branch = os.getenv(LOCAL_BRANCH_ENV)
if remote is not None and branch is not None:
return '%s/%s' % (remote, branch)
def get_affected_files(allow_limited=True):
"""Gets a list of files in the repository.
By default, returns all files via ``git ls-files``. However, in some cases
uses a specific commit or branch (a so-called diff base) to compare
against for changed files. (This requires ``allow_limited=True``.)
To speed up linting on Travis pull requests against master, we manually
set the diff base to the branch the pull request is against. We don't do
this on "push" builds since "master" will be the currently checked out
code. One could potentially use ${TRAVIS_COMMIT_RANGE} to find a diff base
but this value is not dependable.
To allow faster local ``tox`` runs, the local remote and local branch
environment variables can be set to specify a remote branch to diff
against.
:type allow_limited: bool
:param allow_limited: Boolean indicating if a reduced set of files can
be used.
:rtype: pair
:returns: Tuple of the diff base using the list of filenames to be
linted.
"""
diff_base = None
if in_travis():
# In the case of a pull request into a branch, we want to
# diff against HEAD in that branch.
if in_travis_pr():
diff_base = travis_branch()
else:
diff_base = local_diff_branch()
if diff_base is not None and allow_limited:
result = subprocess.check_output(['git', 'diff', '--name-only',
diff_base])
print('Using files changed relative to %s:' % (diff_base,))
print('-' * 60)
print(result.rstrip('\n')) # Don't print trailing newlines.
print('-' * 60)
else:
print('Diff base not specified, listing all files in repository
A22D
.')
result = subprocess.check_output(['git', 'ls-files'])
return result.rstrip('\n').split('\n'), diff_base
def get_required_packages(file_contents):
"""Get required packages from a ``setup.py`` file.
Makes the following assumptions:
* ``install_requires=REQUIREMENTS`` occurs in the call to
``setup()`` in the ``file_contents``.
* The text ``install_requires`` occurs nowhere else in the file.
* The text ``REQUIREMENTS`` only appears when being passed to
``setup()`` (as above) and when being defined.
* The ``REQUIREMENTS`` variable is a list and the text from the
``setup.py`` file containing that list can be parsed using
``ast.literal_eval()``.
:type file_contents: str
:param file_contents: The contents of a ``setup.py`` file.
:rtype: list
:returns: The list of required packages.
:raises: :class:`~exceptions.ValueError` if the file is in an
unexpected format.
"""
# Make sure the only ``install_requires`` happens in the
# call to setup()
if file_contents.count(INST_REQS_KWARG) != 1:
raise ValueError('Expected only one use of keyword',
INST_REQS_KWARG, file_contents)
# Make sure the only usage of ``install_requires`` is to set
# install_requires=REQUIREMENTS.
keyword_stmt = INST_REQS_KWARG + '=' + REQ_VAR
if file_contents.count(keyword_stmt) != 1:
raise ValueError('Expected keyword to be set with variable',
INST_REQS_KWARG, REQ_VAR, file_contents)
# Split file on ``REQUIREMENTS`` variable while asserting that
# it only appear twice.
_, reqs_section, _ = file_contents.split(REQ_VAR)
# Find ``REQUIREMENTS`` list variable defined in ``reqs_section``.
reqs_begin = reqs_section.index('[')
reqs_end = reqs_section.index(']') + 1
# Convert the text to an actual list, but make sure no
# locals or globals can be used.
reqs_list_text = reqs_section[reqs_begin:reqs_end]
# We use literal_eval() because it limits to evaluating
# strings that only consist of a few Python literals: strings,
# numbers, tuples, lists, dicts, booleans, and None.
requirements = ast.literal_eval(reqs_list_text)
# Take the list of requirements and strip off the package name
# from each requirement.
result = []
for required in requirements:
parts = required.split()
result.append(parts[0])
return result
def get_dependency_graph(package_list):
"""Get a directed graph of package dependencies.
:type package_list: list
:param package_list: The list of **all** valid packages.
:rtype: dict
:returns: A dictionary where keys are packages and values are
the set of packages that depend on the key.
"""
result = {package: set() for package in package_list}
for package in package_list:
setup_file = os.path.join(PROJECT_ROOT, package,
'setup.py')
with open(setup_file, 'r') as file_obj:
file_contents = file_obj.read()
requirements = get_required_packages(file_contents)
for requirement in requirements:
if not requirement.startswith(PACKAGE_PREFIX):
continue
_, req_package = requirement.split(PACKAGE_PREFIX)
req_package = req_package.replace('-', '_')
result[req_package].add(package)
return result
def follow_dependencies(subset, package_list):
"""Get a directed graph of package dependencies.
:type subset: list
:param subset: List of a subset of package names.
:type package_list: list
:param package_list: The list of **all** valid packages.
:rtype: list
:returns: An expanded list of packages containing everything
in ``subset`` and any packages that depend on those.
"""
dependency_graph = get_dependency_graph(package_list)
curr_pkgs = None
updated_pkgs = set(subset)
while curr_pkgs != updated_pkgs:
curr_pkgs = updated_pkgs
updated_pkgs = set(curr_pkgs)
for package in curr_pkgs:
updated_pkgs.update(dependency_graph[package])
return sorted(curr_pkgs)