From ad0f3403d3860c6d3b0b28d9309c01b6221a1e4f Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Tue, 16 Feb 2021 23:47:33 +0000 Subject: [PATCH 1/7] feat: add lower bound checker Add a lower bound checker to be invoked on the command line by client libraries. Example Usage: lower-bound-checker check --package-name google-cloud-bigquery --constraints-file testing/constraints-3.6.txt lower-bound-checker update --package-name google-cloud-bigquery --constraints-file testing/constraints-3.6.txt --- noxfile.py | 70 ++++- setup.py | 13 +- test_utils/lower_bound_checker/__init__.py | 0 .../lower_bound_checker.py | 259 ++++++++++++++++++ testing/constraints-3.6.txt | 5 + testing/constraints-3.7.txt | 0 testing/constraints-3.8.txt | 0 testing/constraints-3.9.txt | 0 testing/constraints-3.9.tzt | 0 tests/unit/resources/bad_package/setup.py | 40 +++ tests/unit/resources/good_package/setup.py | 46 ++++ tests/unit/test_lower_bound_checker.py | 251 +++++++++++++++++ 12 files changed, 676 insertions(+), 8 deletions(-) create mode 100644 test_utils/lower_bound_checker/__init__.py create mode 100644 test_utils/lower_bound_checker/lower_bound_checker.py create mode 100644 testing/constraints-3.6.txt create mode 100644 testing/constraints-3.7.txt create mode 100644 testing/constraints-3.8.txt create mode 100644 testing/constraints-3.9.txt create mode 100644 testing/constraints-3.9.tzt create mode 100644 tests/unit/resources/bad_package/setup.py create mode 100644 tests/unit/resources/good_package/setup.py create mode 100644 tests/unit/test_lower_bound_checker.py diff --git a/noxfile.py b/noxfile.py index f3c38c2..dcf876d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -14,17 +14,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Generated by synthtool. DO NOT EDIT! from __future__ import absolute_import import os +import pathlib import shutil import nox +# 'update_lower_bounds' is excluded +nox.options.sessions = [ + "lint", + "blacken", + "lint_setup_py", + "unit", + "check_lower_bounds" +] + BLACK_VERSION = "black==19.3b0" BLACK_PATHS = ["test_utils", "setup.py"] +CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() + @nox.session(python="3.7") def lint(session): @@ -35,9 +46,7 @@ def lint(session): """ session.install("flake8", BLACK_VERSION) session.run( - "black", - "--check", - *BLACK_PATHS, + "black", "--check", *BLACK_PATHS, ) session.run("flake8", *BLACK_PATHS) @@ -54,8 +63,7 @@ def blacken(session): """ session.install(BLACK_VERSION) session.run( - "black", - *BLACK_PATHS, + "black", *BLACK_PATHS, ) @@ -63,4 +71,52 @@ def blacken(session): def lint_setup_py(session): """Verify that setup.py is valid (including RST check).""" session.install("docutils", "pygments") - session.run("python", "setup.py", "check", "--restructuredtext", "--strict") \ No newline at end of file + session.run("python", "setup.py", "check", "--restructuredtext", "--strict") + + +@nox.session(python=["3.6", "3.7", "3.8", "3.9"]) +def unit(session): + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + + # Install two fake packages for the lower-bound-checker tests + session.install("-e", "tests/unit/resources/good_package", "tests/unit/resources/bad_package") + + session.install("pytest") + session.install("-e", ".", "-c", constraints_path) + + # Run py.test against the unit tests. + session.run( + "py.test", + "--quiet", + os.path.join("tests", "unit"), + *session.posargs, + ) + +@nox.session(python="3.8") +def check_lower_bounds(session): + """Check lower bounds in setup.py are reflected in constraints file""" + session.install(".") + session.run( + "lower-bound-checker", + "check", + "--package-name", + "google-cloud-testutils", + "--constraints-file", + "testing/constraints-3.6.txt", + ) + + +@nox.session(python="3.8") +def update_lower_bounds(session): + """Update lower bounds in constraints.txt to match setup.py""" + session.install(".") + session.run( + "lower-bound-checker", + "update", + "--package-name", + "google-cloud-testutils", + "--constraints-file", + "testing/constraints-3.6.txt", + ) \ No newline at end of file diff --git a/setup.py b/setup.py index 5894713..087bdbc 100644 --- a/setup.py +++ b/setup.py @@ -24,6 +24,10 @@ with io.open(readme_filename, encoding="utf-8") as readme_file: readme = readme_file.read() +scripts = ( + ["lower-bound-checker=test_utils.lower_bound_checker.lower_bound_checker:main"], +) + setuptools.setup( name="google-cloud-testutils", version=version, @@ -33,9 +37,16 @@ license="Apache 2.0", url="https://github.com/googleapis/python-test-utils", packages=setuptools.PEP420PackageFinder.find(), + entry_points={"console_scripts": scripts}, platforms="Posix; MacOS X; Windows", include_package_data=True, - install_requires=("google-auth >= 0.4.0", "six"), + install_requires=( + "google-auth >= 0.4.0", + "six>=1.4.0", + "click>=7.0.0", + "packaging>=19.0", + "colorlog>=3.0.0", + ), python_requires=">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*", classifiers=[ "Development Status :: 4 - Beta", diff --git a/test_utils/lower_bound_checker/__init__.py b/test_utils/lower_bound_checker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test_utils/lower_bound_checker/lower_bound_checker.py b/test_utils/lower_bound_checker/lower_bound_checker.py new file mode 100644 index 0000000..853007a --- /dev/null +++ b/test_utils/lower_bound_checker/lower_bound_checker.py @@ -0,0 +1,259 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +from typing import List, Tuple, Set + +import click +from packaging.requirements import Requirement +from packaging.version import Version +import pkg_resources + + +def _get_package_requirements(package_name: str) -> List[Requirement]: + """ + Get a list of all requirements and extras declared by this package. + The package must already be installed in the environment. + + Args: + package_name (str): The name of the package. + + Returns: + List[pkg_resources.Requirement]: A list of package requirements and extras. + """ + dist = pkg_resources.get_distribution(package_name) + requirements = [Requirement(str(r)) for r in dist.requires(extras=dist.extras)] + + return requirements + + +def _parse_requirements_file(requirements_file: str) -> List[Requirement]: + """ + Get a list of requirements found in a requirements file. + + Args: + requirements_file (str): Path to a requirements file. + + Returns: + List[Requirement]: A list of requirements. + """ + requirements = [] + + with Path(requirements_file).open() as f: + for line in f: + line = line.strip() + if line and not line.startswith("#"): + requirements.append(Requirement(line)) + + return requirements + + +def _get_pinned_versions( + ctx: click.Context, requirements: List[Requirement] +) -> Set[Tuple[str, Version]]: + """Turn a list of requirements into a set of (package name, Version) tuples. + + The requirements are all expected to pin explicitly to one version. + Other formats will result in an error. + + {("requests", Version("1.25.0"), ("google-auth", Version("1.0.0")} + + Args: + ctx (click.Context): The current click context. + requirements (List[Requirement]): A list of requirements. + + Returns: + Set[Tuple[str, Version]]: Tuples of the package name and Version. + """ + constraints = set() + + invalid_requirements = [] + + for constraint in requirements: + spec_set = list(constraint.specifier) + if len(spec_set) != 1: + invalid_requirements.append(constraint.name) + else: + if spec_set[0].operator != "==": + invalid_requirements.append(constraint.name) + else: + constraints.add((constraint.name, Version(spec_set[0].version))) + + if invalid_requirements: + ctx.fail( + f"These requirements are not pinned to one version: {invalid_requirements}" + ) + + return constraints + + +class IndeterminableLowerBound(Exception): + pass + + +def _lower_bound(requirement: Requirement) -> str: + """ + Given a requirement, determine the lowest version that fulfills the requirement. + The lower bound can be determined for a requirement only if it is one of these + formats: + + foo==1.2.0 + foo>=1.2.0 + foo>=1.2.0, <2.0.0dev + foo<2.0.0dev, >=1.2.0 + + Args: + requirement (Requirement): A requirement to parse + + Returns: + str: The lower bound for the requirement. + """ + spec_set = list(requirement.specifier) + + # sort by operator: <, then >= + spec_set.sort(key=lambda x: x.operator) + + if len(spec_set) == 1: + # foo==1.2.0 + if spec_set[0].operator == "==": + return spec_set[0].version + # foo>=1.2.0 + elif spec_set[0].operator == ">=": + return spec_set[0].version + # foo<2.0.0, >=1.2.0 or foo>=1.2.0, <2.0.0 + elif len(spec_set) == 2: + if spec_set[0].operator == "<" and spec_set[1].operator == ">=": + return spec_set[1].version + + raise IndeterminableLowerBound( + f"Lower bound could not be determined for {requirement.name}" + ) + + +def _get_package_lower_bounds( + ctx: click.Context, requirements: List[Requirement] +) -> Set[Tuple[str, Version]]: + """Get a set of tuples ('package_name', Version('1.0.0')) from a + list of Requirements. + + Args: + ctx (click.Context): The current click context. + requirements (List[Requirement]): A list of requirements. + + Returns: + Set[Tuple[str, Version]]: A set of (package_name, lower_bound) + tuples. + """ + bad_package_lower_bounds = [] + package_lower_bounds = set() + + for req in requirements: + try: + version = _lower_bound(req) + package_lower_bounds.add((req.name, Version(version))) + except IndeterminableLowerBound: + bad_package_lower_bounds.append(req.name) + + if bad_package_lower_bounds: + ctx.fail( + f"setup.py is missing explicit lower bounds for the following packages: {str(bad_package_lower_bounds)}" + ) + else: + return package_lower_bounds + + +@click.group() +def main(): + pass + + +@main.command() +@click.option("--package-name", required=True, help="Name of the package.") +@click.option("--constraints-file", required=True, help="Path to constraints file.") +@click.pass_context +def update(ctx: click.Context, package_name: str, constraints_file: str) -> None: + """Create a constraints file with lower bounds for the specified package. + + If the constraints file already exists the contents will be overwritten. + """ + requirements = _get_package_requirements(package_name) + requirements.sort(key=lambda x: x.name) + + package_lower_bounds = list(_get_package_lower_bounds(ctx, requirements)) + package_lower_bounds.sort(key=lambda x: x[0]) + + constraints = [f"{name}=={version}" for name, version in package_lower_bounds] + Path(constraints_file).write_text("\n".join(constraints)) + + +@main.command() +@click.option("--package-name", required=True, help="Name of the package.") +@click.option("--constraints-file", required=True, help="Path to constraints file.") +@click.pass_context +def check(ctx: click.Context, package_name: str, constraints_file: str): + """Given a package name and a constraints_file, check that the constraints_file + explicitly pins to the lower bound specified in the setup.py for each requirement. + + The lower bound can be determined for a requirement only if it is one of these + formats: + + foo==1.2.0 + foo>=1.2.0 + foo>=1.2.0, <2.0.0dev + foo<2.0.0dev, >=1.2.0 + + The constraints file should pin every requirement to a single version. + + foo==1.2.0 + """ + + package_requirements = _get_package_requirements(package_name) + constraints = _parse_requirements_file(constraints_file) + + package_lower_bounds = _get_package_lower_bounds(ctx, package_requirements) + constraints_file_versions = _get_pinned_versions(ctx, constraints) + + # Look for dependencies in setup.py that are missing from constraints.txt + package_names = {x[0] for x in package_lower_bounds} + constraint_names = {x[0] for x in constraints_file_versions} + missing_from_constraints = package_names - constraint_names + + if missing_from_constraints: + ctx.fail( + ( + f"The following packages are declared as a requirement or extra" + f"in setup.py but were not found in {constraints_file}: {str(missing_from_constraints)}" + ) + ) + + # We use .issuperset() instead of == because there may be additional entries + # in constraints.txt (e.g., test only requirements) + if not constraints_file_versions.issuperset(package_lower_bounds): + first_line = f"The following packages have different versions {package_name}'s setup.py and {constraints_file}" + error_msg = [first_line, "-" * (7 + len(first_line))] + + difference = package_lower_bounds - constraints_file_versions + constraints_dict = dict(constraints_file_versions) + + for req, setup_py_version in difference: + error_msg.append( + f"'{req}' lower bound is {setup_py_version} in setup.py but constraints file has {constraints_dict[req]}" + ) + ctx.fail("\n".join(error_msg)) + + click.secho("All good!", fg="green") + + +if __name__ == "__main__": + main() diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt new file mode 100644 index 0000000..2a99377 --- /dev/null +++ b/testing/constraints-3.6.txt @@ -0,0 +1,5 @@ +click==7.0.0 +google-auth==0.4.0 +packaging==19.0 +six==1.4.0 +colorlog==3.0.0 \ No newline at end of file diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt new file mode 100644 index 0000000..e69de29 diff --git a/testing/constraints-3.8.txt b/testing/constraints-3.8.txt new file mode 100644 index 0000000..e69de29 diff --git a/testing/constraints-3.9.txt b/testing/constraints-3.9.txt new file mode 100644 index 0000000..e69de29 diff --git a/testing/constraints-3.9.tzt b/testing/constraints-3.9.tzt new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/resources/bad_package/setup.py b/tests/unit/resources/bad_package/setup.py new file mode 100644 index 0000000..de8645f --- /dev/null +++ b/tests/unit/resources/bad_package/setup.py @@ -0,0 +1,40 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import setuptools + + +requirements = [ + "requests", # no lower bound + "packaging>=14.0, !=15.0, <20.0.0", # too complex for tool + "six<2.0.0", # no lower bound + "click==7.0.0", +] + +setuptools.setup( + name="invalid-package", + version="0.0.1", + author="Example Author", + author_email="author@example.com", + description="A small example package", + long_description_content_type="text/markdown", + url="https://github.com/pypa/sampleproject", + classifiers=[ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", + ], + install_requires=requirements, + packages=setuptools.find_packages(), + python_requires=">=3.6", +) diff --git a/tests/unit/resources/good_package/setup.py b/tests/unit/resources/good_package/setup.py new file mode 100644 index 0000000..27fc837 --- /dev/null +++ b/tests/unit/resources/good_package/setup.py @@ -0,0 +1,46 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import setuptools + + +# This package has four requirements. +# Each uses a different kind of pin accepted by the function that +# extracts lower bounds. +requirements = [ + "requests>=1.0.0", + "packaging>=14.0, <20.0.0", + "six<2.0.0, >=1.0.0", + "click==7.0.0", +] + +extras = {"grpc": "grpcio>=1.0.0"} + +setuptools.setup( + name="valid-package", + version="0.0.1", + author="Example Author", + author_email="author@example.com", + description="A small example package", + long_description_content_type="text/markdown", + url="https://github.com/pypa/sampleproject", + classifiers=[ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", + ], + install_requires=requirements, + extras_require=extras, + packages=setuptools.find_packages(), + python_requires=">=3.6", +) diff --git a/tests/unit/test_lower_bound_checker.py b/tests/unit/test_lower_bound_checker.py new file mode 100644 index 0000000..4d6bb97 --- /dev/null +++ b/tests/unit/test_lower_bound_checker.py @@ -0,0 +1,251 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from contextlib import contextmanager +from pathlib import Path +import re +import tempfile +from typing import List + +from click.testing import CliRunner +import pytest + +from test_utils.lower_bound_checker import lower_bound_checker + +RUNNER = CliRunner() + +PACKGE_LIST_REGEX = re.compile("Error.*[\[\{](.+)[\]\}]") +DIFFERENT_VERSIONS_LIST_REGEX = re.compile("'(.*?)' lower bound is") + +# These packages are installed into the environment by the nox session +# See 'resources/' for the setup.py files +GOOD_PACKAGE = "valid-package" +BAD_PACKAGE = "invalid-package" + + +def parse_error_msg(msg: str) -> List[str]: + """Get package names from the error message. + + Example: + Error: setup.py is missing explicit lower bounds for the following packages: ["requests", "grpcio"] + """ + match = PACKGE_LIST_REGEX.search(msg) + + reqs = [] + + if match: + reqs = match.groups(1)[0].split(",") + reqs = [r.strip().replace("'", "").replace('"', "") for r in reqs] + + return reqs + +def parse_diff_versions_error_msg(msg: str) -> List[str]: + """Get package names from the error message listing different versions + + Example: + 'requests' lower bound is 1.2.0 in setup.py but constraints file has 1.3.0 + 'grpcio' lower bound is 1.0.0 in setup.py but constraints file has 1.10.0 + """ + pattern = re.compile(DIFFERENT_VERSIONS_LIST_REGEX) + pkg_names = pattern.findall(msg) + + return pkg_names + +@contextmanager +def constraints_file(requirements: List[str]): + """Write the list of requirements into a temporary file""" + + tmpdir = tempfile.TemporaryDirectory() + constraints_path = Path(tmpdir.name) / "constraints.txt" + + constraints_path.write_text("\n".join(requirements)) + yield constraints_path + + tmpdir.cleanup() + + +def test_update_constraints(): + with tempfile.TemporaryDirectory() as tmpdir: + constraints_path = Path(tmpdir) / "constraints.txt" + + result = RUNNER.invoke( + lower_bound_checker.update, ["--package-name", GOOD_PACKAGE, "--constraints-file", str(constraints_path)] + ) + + assert result.exit_code == 0 + assert constraints_path.exists() + + output = constraints_path.read_text().split("\n") + + assert output == ["click==7.0.0", "grpcio==1.0.0", "packaging==14.0", "requests==1.0.0", "six==1.0.0",] + + + +def test_update_constraints_overwrites_existing_file(): + constraints = [ + "requests==1.0.0", + "packaging==13.0", + "six==1.6.0", + "click==5.0.0", + ] + with constraints_file(constraints) as c: + result = RUNNER.invoke( + lower_bound_checker.update, ["--package-name", GOOD_PACKAGE, "--constraints-file", c] + ) + + assert result.exit_code == 0 + + output = c.read_text().split("\n") + assert output == ["click==7.0.0", "grpcio==1.0.0", "packaging==14.0", "requests==1.0.0", "six==1.0.0", + ] + +def test_update_constraints_with_setup_py_missing_lower_bounds(): + constraints = [ + "requests==1.0.0", + "packaging==14.0", + "six==1.0.0", + "click==7.0.0", + ] + with constraints_file(constraints) as c: + result = RUNNER.invoke( + lower_bound_checker.update, ["--package-name", BAD_PACKAGE, "--constraints-file", c] + ) + + assert result.exit_code == 2 + assert "setup.py is missing explicit lower bounds" in result.output + + invalid_pkg_list = parse_error_msg(result.output) + assert set(invalid_pkg_list) == {"requests", "packaging", "six"} + + + +def test_check(): + constraints = [ + "requests==1.0.0", + "packaging==14.0", + "six==1.0.0", + "click==7.0.0", + "grpcio==1.0.0" + ] + with constraints_file(constraints) as c: + result = RUNNER.invoke( + lower_bound_checker.check, ["--package-name", GOOD_PACKAGE, "--constraints-file", c] + ) + + assert result.exit_code == 0 + + +def test_update_constraints_with_extra_constraints(): + constraints = [ + "requests==1.0.0", + "packaging==14.0", + "six==1.0.0", + "click==7.0.0", + "grpcio==1.0.0", + "pytest==6.0.0", # additional requirement + ] + with constraints_file(constraints) as c: + result = RUNNER.invoke( + lower_bound_checker.check, ["--package-name", GOOD_PACKAGE, "--constraints-file", c] + ) + + assert result.exit_code == 0 + + +def test_check_with_missing_constraints_file(): + result = RUNNER.invoke( + lower_bound_checker.check, + [ + "--package-name", + GOOD_PACKAGE, + "--constraints-file", + "missing_constraints.txt", + ], + ) + + assert result.exit_code == 1 + assert isinstance(result.exception, FileNotFoundError) + + +def test_check_with_constraints_file_invalid_pins(): + constraints = [ + "requests==1.0.0", + "packaging==14.0", + "six==1.0.0, <2.0.0dev", # should be == + "click>=7.0.0", # should be == + ] + with constraints_file(constraints) as c: + result = RUNNER.invoke( + lower_bound_checker.check, ["--package-name", GOOD_PACKAGE, "--constraints-file", c] + ) + + assert result.exit_code == 2 + + invalid_pkg_list = parse_error_msg(result.output) + + assert set(invalid_pkg_list) == {"six", "click"} + + +def test_check_with_constraints_file_missing_packages(): + constraints = [ + "requests==1.0.0", + "packaging==14.0", + # missing 'six' and 'click' and extra 'grpcio' + ] + with constraints_file(constraints) as c: + result = RUNNER.invoke( + lower_bound_checker.check, ["--package-name", GOOD_PACKAGE, "--constraints-file", c] + ) + + assert result.exit_code == 2 + + invalid_pkg_list = parse_error_msg(result.output) + assert set(invalid_pkg_list) == {"six", "click", "grpcio"} + + +def test_check_with_constraints_file_different_versions(): + constraints = [ + "requests==1.2.0", # setup.py has 1.0.0 + "packaging==14.1", # setup.py has 14.0 + "six==1.4.0", # setup.py has 1.0.0 + "click==7.0.0", + "grpcio==1.0.0" + ] + with constraints_file(constraints) as c: + result = RUNNER.invoke( + lower_bound_checker.check, ["--package-name", GOOD_PACKAGE, "--constraints-file", c] + ) + + assert result.exit_code == 2 + + invalid_pkg_list = parse_diff_versions_error_msg(result.output) + assert set(invalid_pkg_list) == {"requests", "packaging", "six"} + + +def test_check_with_setup_py_missing_lower_bounds(): + constraints = [ + "requests==1.0.0", + "packaging==14.0", + "six==1.0.0", + "click==7.0.0", + ] + with constraints_file(constraints) as c: + result = RUNNER.invoke( + lower_bound_checker.check, ["--package-name", BAD_PACKAGE, "--constraints-file", c] + ) + + assert result.exit_code == 2 + + invalid_pkg_list = parse_error_msg(result.output) + assert set(invalid_pkg_list) == {"requests", "packaging", "six"} From 0bd10d2b3e6ef0791cf8ca244a93690358285e25 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Wed, 17 Feb 2021 00:01:18 +0000 Subject: [PATCH 2/7] fix: update deps, fix docs --- setup.py | 1 - test_utils/lower_bound_checker/lower_bound_checker.py | 2 ++ testing/constraints-3.9.tzt | 0 3 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 testing/constraints-3.9.tzt diff --git a/setup.py b/setup.py index 087bdbc..1060a2f 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,6 @@ "six>=1.4.0", "click>=7.0.0", "packaging>=19.0", - "colorlog>=3.0.0", ), python_requires=">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*", classifiers=[ diff --git a/test_utils/lower_bound_checker/lower_bound_checker.py b/test_utils/lower_bound_checker/lower_bound_checker.py index 853007a..7adf9c8 100644 --- a/test_utils/lower_bound_checker/lower_bound_checker.py +++ b/test_utils/lower_bound_checker/lower_bound_checker.py @@ -216,6 +216,8 @@ def check(ctx: click.Context, package_name: str, constraints_file: str): The constraints file should pin every requirement to a single version. foo==1.2.0 + + The package must already be installed in the environment. """ package_requirements = _get_package_requirements(package_name) diff --git a/testing/constraints-3.9.tzt b/testing/constraints-3.9.tzt deleted file mode 100644 index e69de29..0000000 From d9df36214444bbf4ceffbbc4365a26f80bdffc70 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Wed, 17 Feb 2021 00:15:09 +0000 Subject: [PATCH 3/7] docs: tweak more comments --- .../lower_bound_checker.py | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/test_utils/lower_bound_checker/lower_bound_checker.py b/test_utils/lower_bound_checker/lower_bound_checker.py index 7adf9c8..e4db46a 100644 --- a/test_utils/lower_bound_checker/lower_bound_checker.py +++ b/test_utils/lower_bound_checker/lower_bound_checker.py @@ -183,7 +183,7 @@ def main(): @click.option("--constraints-file", required=True, help="Path to constraints file.") @click.pass_context def update(ctx: click.Context, package_name: str, constraints_file: str) -> None: - """Create a constraints file with lower bounds for the specified package. + """Create a constraints file with lower bounds for packaage-name. If the constraints file already exists the contents will be overwritten. """ @@ -202,22 +202,26 @@ def update(ctx: click.Context, package_name: str, constraints_file: str) -> None @click.option("--constraints-file", required=True, help="Path to constraints file.") @click.pass_context def check(ctx: click.Context, package_name: str, constraints_file: str): - """Given a package name and a constraints_file, check that the constraints_file - explicitly pins to the lower bound specified in the setup.py for each requirement. + """Check that the constraints-file pins to the lower bound specified in the package-name's + setup.py for each requirement. - The lower bound can be determined for a requirement only if it is one of these - formats: + Requirements: - foo==1.2.0 - foo>=1.2.0 - foo>=1.2.0, <2.0.0dev - foo<2.0.0dev, >=1.2.0 + 1. The setup.py pins every requirement in one of the following formats: - The constraints file should pin every requirement to a single version. + * foo==1.2.0 - foo==1.2.0 + * foo>=1.2.0 + + * foo>=1.2.0, <2.0.0dev + + * foo<2.0.0dev, >=1.2.0 + + 2. The constraints file pins every requirement to a single version: + + * foo==1.2.0 - The package must already be installed in the environment. + 3. The package is already be installed in the environment. """ package_requirements = _get_package_requirements(package_name) From 8ed4eb9aa41f48d827c8dffadee55c484a09f459 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Wed, 17 Feb 2021 00:15:53 +0000 Subject: [PATCH 4/7] docs: more fixes --- test_utils/lower_bound_checker/lower_bound_checker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test_utils/lower_bound_checker/lower_bound_checker.py b/test_utils/lower_bound_checker/lower_bound_checker.py index e4db46a..0cf724d 100644 --- a/test_utils/lower_bound_checker/lower_bound_checker.py +++ b/test_utils/lower_bound_checker/lower_bound_checker.py @@ -202,7 +202,7 @@ def update(ctx: click.Context, package_name: str, constraints_file: str) -> None @click.option("--constraints-file", required=True, help="Path to constraints file.") @click.pass_context def check(ctx: click.Context, package_name: str, constraints_file: str): - """Check that the constraints-file pins to the lower bound specified in the package-name's + """Check that the constraints-file pins to the lower bound specified in package-name's setup.py for each requirement. Requirements: @@ -221,7 +221,7 @@ def check(ctx: click.Context, package_name: str, constraints_file: str): * foo==1.2.0 - 3. The package is already be installed in the environment. + 3. packge-name is already be installed in the environment. """ package_requirements = _get_package_requirements(package_name) From 693adc9dc5f4f28d3829302c7f593b15ff546174 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Wed, 17 Feb 2021 00:17:31 +0000 Subject: [PATCH 5/7] docs: fix typo --- test_utils/lower_bound_checker/lower_bound_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_utils/lower_bound_checker/lower_bound_checker.py b/test_utils/lower_bound_checker/lower_bound_checker.py index 0cf724d..0284b68 100644 --- a/test_utils/lower_bound_checker/lower_bound_checker.py +++ b/test_utils/lower_bound_checker/lower_bound_checker.py @@ -221,7 +221,7 @@ def check(ctx: click.Context, package_name: str, constraints_file: str): * foo==1.2.0 - 3. packge-name is already be installed in the environment. + 3. package-name is already installed in the environment. """ package_requirements = _get_package_requirements(package_name) From 15dfef1d6ff304cbbb652b297f738411f05cbfe0 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Wed, 17 Feb 2021 00:26:24 +0000 Subject: [PATCH 6/7] fix: fix deps --- setup.py | 2 +- .../lower_bound_checker/lower_bound_checker.py | 14 +++++++------- testing/constraints-3.6.txt | 2 +- tests/unit/test_lower_bound_checker.py | 4 ++-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/setup.py b/setup.py index 1060a2f..3593037 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ include_package_data=True, install_requires=( "google-auth >= 0.4.0", - "six>=1.4.0", + "six>=1.9.0", "click>=7.0.0", "packaging>=19.0", ), diff --git a/test_utils/lower_bound_checker/lower_bound_checker.py b/test_utils/lower_bound_checker/lower_bound_checker.py index 0284b68..5c24a9d 100644 --- a/test_utils/lower_bound_checker/lower_bound_checker.py +++ b/test_utils/lower_bound_checker/lower_bound_checker.py @@ -68,11 +68,11 @@ def _get_pinned_versions( Other formats will result in an error. {("requests", Version("1.25.0"), ("google-auth", Version("1.0.0")} - + Args: ctx (click.Context): The current click context. requirements (List[Requirement]): A list of requirements. - + Returns: Set[Tuple[str, Version]]: Tuples of the package name and Version. """ @@ -115,7 +115,7 @@ def _lower_bound(requirement: Requirement) -> str: Args: requirement (Requirement): A requirement to parse - + Returns: str: The lower bound for the requirement. """ @@ -146,11 +146,11 @@ def _get_package_lower_bounds( ) -> Set[Tuple[str, Version]]: """Get a set of tuples ('package_name', Version('1.0.0')) from a list of Requirements. - + Args: ctx (click.Context): The current click context. requirements (List[Requirement]): A list of requirements. - + Returns: Set[Tuple[str, Version]]: A set of (package_name, lower_bound) tuples. @@ -205,7 +205,7 @@ def check(ctx: click.Context, package_name: str, constraints_file: str): """Check that the constraints-file pins to the lower bound specified in package-name's setup.py for each requirement. - Requirements: + Requirements: 1. The setup.py pins every requirement in one of the following formats: @@ -220,7 +220,7 @@ def check(ctx: click.Context, package_name: str, constraints_file: str): 2. The constraints file pins every requirement to a single version: * foo==1.2.0 - + 3. package-name is already installed in the environment. """ diff --git a/testing/constraints-3.6.txt b/testing/constraints-3.6.txt index 2a99377..87fc11f 100644 --- a/testing/constraints-3.6.txt +++ b/testing/constraints-3.6.txt @@ -1,5 +1,5 @@ click==7.0.0 google-auth==0.4.0 packaging==19.0 -six==1.4.0 +six==1.9.0 colorlog==3.0.0 \ No newline at end of file diff --git a/tests/unit/test_lower_bound_checker.py b/tests/unit/test_lower_bound_checker.py index 4d6bb97..e177960 100644 --- a/tests/unit/test_lower_bound_checker.py +++ b/tests/unit/test_lower_bound_checker.py @@ -25,7 +25,7 @@ RUNNER = CliRunner() -PACKGE_LIST_REGEX = re.compile("Error.*[\[\{](.+)[\]\}]") +PACKAGE_LIST_REGEX = re.compile(r"Error.*[\[\{](.+)[\]\}]") DIFFERENT_VERSIONS_LIST_REGEX = re.compile("'(.*?)' lower bound is") # These packages are installed into the environment by the nox session @@ -40,7 +40,7 @@ def parse_error_msg(msg: str) -> List[str]: Example: Error: setup.py is missing explicit lower bounds for the following packages: ["requests", "grpcio"] """ - match = PACKGE_LIST_REGEX.search(msg) + match = PACKAGE_LIST_REGEX.search(msg) reqs = [] From 7099ff7ffe7ebcb3e81a5acd7e6e470085fd901a Mon Sep 17 00:00:00 2001 From: Bu Sun Kim <8822365+busunkim96@users.noreply.github.com> Date: Fri, 19 Feb 2021 18:22:46 -0700 Subject: [PATCH 7/7] fix: fix typo Co-authored-by: Anthonios Partheniou --- test_utils/lower_bound_checker/lower_bound_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_utils/lower_bound_checker/lower_bound_checker.py b/test_utils/lower_bound_checker/lower_bound_checker.py index 5c24a9d..7a8e65b 100644 --- a/test_utils/lower_bound_checker/lower_bound_checker.py +++ b/test_utils/lower_bound_checker/lower_bound_checker.py @@ -183,7 +183,7 @@ def main(): @click.option("--constraints-file", required=True, help="Path to constraints file.") @click.pass_context def update(ctx: click.Context, package_name: str, constraints_file: str) -> None: - """Create a constraints file with lower bounds for packaage-name. + """Create a constraints file with lower bounds for package-name. If the constraints file already exists the contents will be overwritten. """