diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 44600a2..c4c27ba 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -8,8 +8,6 @@ jobs: fail-fast: false matrix: python: - - v: "3.6" - tox_env: "py36" - v: "3.7" tox_env: "py37" - v: "3.7" @@ -21,17 +19,17 @@ jobs: tox_env: "py39" - v: "3.10" tox_env: "py310" - - v: "3.9" - tox_env: "linting" + - v: "3.11" + tox_env: "py311" os: [ubuntu-latest, windows-latest] steps: - name: Set Git to use LF run: | git config --global core.autocrlf false git config --global core.eol lf - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python.v }} - name: Install tox diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index cd429eb..49f5d0c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,17 +1,17 @@ --- repos: - repo: https://github.com/psf/black - rev: 20.8b1 + rev: 22.3.0 hooks: - id: black args: [--safe, --quiet, --target-version, py36] - repo: https://github.com/asottile/blacken-docs - rev: v1.10.0 + rev: v1.12.1 hooks: - id: blacken-docs additional_dependencies: [black==20.8b1] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v4.2.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer @@ -20,8 +20,8 @@ repos: - id: check-yaml - id: debug-statements language_version: python3 - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 + - repo: https://github.com/PyCQA/flake8 + rev: 4.0.1 hooks: - id: flake8 language_version: python3 @@ -30,25 +30,25 @@ repos: rev: v0.3.3 hooks: - id: pep257 - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 + - repo: https://github.com/PyCQA/flake8 + rev: 4.0.1 hooks: - id: flake8 language_version: python3 - repo: https://github.com/asottile/reorder_python_imports - rev: v2.4.0 + rev: v3.1.0 hooks: - id: reorder-python-imports - repo: https://github.com/asottile/pyupgrade - rev: v2.10.0 + rev: v2.32.1 hooks: - id: pyupgrade args: [--keep-percent-format, --py36-plus] - repo: https://github.com/pre-commit/pygrep-hooks - rev: v1.8.0 + rev: v1.9.0 hooks: - id: rst-backticks - repo: https://github.com/adrienverge/yamllint.git - rev: v1.26.0 + rev: v1.26.3 hooks: - id: yamllint diff --git a/README.rst b/README.rst index 70fbd79..ddf025c 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ pytest-timeout ============== -|python| |version| |anaconda| |ci| +|python| |version| |anaconda| |ci| |pre-commit| .. |version| image:: https://img.shields.io/pypi/v/pytest-timeout.svg :target: https://pypi.python.org/pypi/pytest-timeout @@ -16,13 +16,18 @@ pytest-timeout .. |python| image:: https://img.shields.io/pypi/pyversions/pytest-timeout.svg :target: https://pypi.python.org/pypi/pytest-timeout/ -**This is not the timeout you are looking for!** +.. |pre-commit| image:: https://results.pre-commit.ci/badge/github/pytest-dev/pytest-timeout/master.svg + :target: https://results.pre-commit.ci/latest/github/pytest-dev/pytest-timeout/master + .. warning:: Please read this README carefully and only use this plugin if you - understand the consequences. Remember your test suite needs to be - **fast**, timeouts are a last resort not an expected failure mode. + understand the consequences. This plugin is designed to catch + excessively long test durations like deadlocked or hanging tests, + it is not designed for precise timings or performance regressions. + Remember your test suite should aim to be **fast**, with timeouts + being a last resort, not an expected failure mode. This plugin will time each test and terminate it when it takes too long. Termination may or may not be graceful, please see below, but @@ -236,16 +241,20 @@ check to see if the module it belongs to is present in a set of known debugging frameworks modules OR if pytest itself drops you into a pdb session using ``--pdb`` or similar. +This functionality can be disabled with the ``--disable-debugger-detection`` flag +or the corresponding ``timeout_disable_debugger_detection`` ini setting / environment +variable. + -Extending pytest-timeout with plugings -====================================== +Extending pytest-timeout with plugins +===================================== ``pytest-timeout`` provides two hooks that can be used for extending the tool. These -hooks are used for for setting the timeout timer and cancelling it it the timeout is not +hooks are used for setting the timeout timer and cancelling it if the timeout is not reached. For example, ``pytest-asyncio`` can provide asyncio-specific code that generates better -traceback and points on timed out ``await`` instead of the running loop ieration. +traceback and points on timed out ``await`` instead of the running loop iteration. See `pytest hooks documentation `_ for more info @@ -313,6 +322,7 @@ function: import pytest import pytest_timeout + def on_timeout(): if pytest_timeout.is_debugging(): return @@ -322,6 +332,12 @@ function: Changelog ========= +2.2.0 +----- + +- Add ``--timeout-disable-debugger-detection`` flag, thanks + Michael Peters + 2.1.0 ----- @@ -348,7 +364,7 @@ Changelog thread to avoid crash. - Fix pycharm debugger detection so timeouts are not triggered during debugger usage. -- Dropped support for Python 2, minimum pytest version upported is 5.0.0. +- Dropped support for Python 2, minimum pytest version supported is 5.0.0. 1.4.2 ----- @@ -383,7 +399,7 @@ Changelog 1.3.2 ----- -- This changelog was ommitted for the 1.3.2 release and was added +- This changelog was omitted for the 1.3.2 release and was added afterwards. Apologies for the confusion. - Fix pytest 3.7.3 compatibility. The capture API had changed slightly and this needed fixing. Thanks Bruno Oliveira for the diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..fed528d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" diff --git a/pytest_timeout.py b/pytest_timeout.py index da02c0d..674aab6 100644 --- a/pytest_timeout.py +++ b/pytest_timeout.py @@ -40,11 +40,17 @@ function body, ignoring the time it takes when evaluating any fixtures used in the test. """.strip() +DISABLE_DEBUGGER_DETECTION_DESC = """ +When specified, disables debugger detection. breakpoint(), pdb.set_trace(), etc. +will be interrupted by the timeout. +""".strip() # bdb covers pdb, ipdb, and possibly others # pydevd covers PyCharm, VSCode, and possibly others KNOWN_DEBUGGING_MODULES = {"pydevd", "bdb", "pydevd_frame_evaluator"} -Settings = namedtuple("Settings", ["timeout", "method", "func_only"]) +Settings = namedtuple( + "Settings", ["timeout", "method", "func_only", "disable_debugger_detection"] +) @pytest.hookimpl @@ -68,9 +74,21 @@ def pytest_addoption(parser): choices=["signal", "thread"], help=METHOD_DESC, ) + group.addoption( + "--timeout-disable-debugger-detection", + dest="timeout_disable_debugger_detection", + action="store_true", + help=DISABLE_DEBUGGER_DETECTION_DESC, + ) parser.addini("timeout", TIMEOUT_DESC) parser.addini("timeout_method", METHOD_DESC) - parser.addini("timeout_func_only", FUNC_ONLY_DESC, type="bool") + parser.addini("timeout_func_only", FUNC_ONLY_DESC, type="bool", default=False) + parser.addini( + "timeout_disable_debugger_detection", + DISABLE_DEBUGGER_DETECTION_DESC, + type="bool", + default=False, + ) class TimeoutHooks: @@ -107,19 +125,24 @@ def pytest_configure(config): """Register the marker so it shows up in --markers output.""" config.addinivalue_line( "markers", - "timeout(timeout, method=None, func_only=False): Set a timeout, timeout " + "timeout(timeout, method=None, func_only=False, " + "disable_debugger_detection=False): Set a timeout, timeout " "method and func_only evaluation on just one test item. The first " "argument, *timeout*, is the timeout in seconds while the keyword, " - "*method*, takes the same values as the --timeout_method option. The " + "*method*, takes the same values as the --timeout-method option. The " "*func_only* keyword, when set to True, defers the timeout evaluation " "to only the test function body, ignoring the time it takes when " - "evaluating any fixtures used in the test.", + "evaluating any fixtures used in the test. The " + "*disable_debugger_detection* keyword, when set to True, disables " + "debugger detection, allowing breakpoint(), pdb.set_trace(), etc. " + "to be interrupted", ) settings = get_env_settings(config) config._env_timeout = settings.timeout config._env_timeout_method = settings.method config._env_timeout_func_only = settings.func_only + config._env_timeout_disable_debugger_detection = settings.disable_debugger_detection @pytest.hookimpl(hookwrapper=True) @@ -238,7 +261,7 @@ def pytest_timeout_set_timer(item, settings): def handler(signum, frame): __tracebackhide__ = True - timeout_sigalrm(item, settings.timeout) + timeout_sigalrm(item, settings) def cancel(): signal.setitimer(signal.ITIMER_REAL, 0) @@ -248,9 +271,7 @@ def cancel(): signal.signal(signal.SIGALRM, handler) signal.setitimer(signal.ITIMER_REAL, settings.timeout) elif timeout_method == "thread": - timer = threading.Timer( - settings.timeout, timeout_timer, (item, settings.timeout) - ) + timer = threading.Timer(settings.timeout, timeout_timer, (item, settings)) timer.name = "%s %s" % (__name__, item.nodeid) def cancel(): @@ -299,17 +320,21 @@ def get_env_settings(config): method = DEFAULT_METHOD func_only = config.getini("timeout_func_only") - if func_only == []: - # No value set - func_only = None - if func_only is not None: - func_only = _validate_func_only(func_only, "config file") - return Settings(timeout, method, func_only or False) + + disable_debugger_detection = config.getvalue("timeout_disable_debugger_detection") + if disable_debugger_detection is None: + ini = config.getini("timeout_disable_debugger_detection") + if ini: + disable_debugger_detection = _validate_disable_debugger_detection( + ini, "config file" + ) + + return Settings(timeout, method, func_only, disable_debugger_detection) def _get_item_settings(item, marker=None): """Return (timeout, method) for an item.""" - timeout = method = func_only = None + timeout = method = func_only = disable_debugger_detection = None if not marker: marker = item.get_closest_marker("timeout") if marker is not None: @@ -317,15 +342,18 @@ def _get_item_settings(item, marker=None): timeout = _validate_timeout(settings.timeout, "marker") method = _validate_method(settings.method, "marker") func_only = _validate_func_only(settings.func_only, "marker") + disable_debugger_detection = _validate_disable_debugger_detection( + settings.disable_debugger_detection, "marker" + ) if timeout is None: timeout = item.config._env_timeout if method is None: method = item.config._env_timeout_method if func_only is None: func_only = item.config._env_timeout_func_only - if func_only is None: - func_only = False - return Settings(timeout, method, func_only) + if disable_debugger_detection is None: + disable_debugger_detection = item.config._env_timeout_disable_debugger_detection + return Settings(timeout, method, func_only, disable_debugger_detection) def _parse_marker(marker): @@ -362,7 +390,7 @@ def _parse_marker(marker): method = None if func_only is NOTSET: func_only = None - return Settings(timeout, method, func_only) + return Settings(timeout, method, func_only, None) def _validate_timeout(timeout, where): @@ -384,20 +412,31 @@ def _validate_method(method, where): def _validate_func_only(func_only, where): if func_only is None: - return False + return None if not isinstance(func_only, bool): raise ValueError("Invalid func_only value %s from %s" % (func_only, where)) return func_only -def timeout_sigalrm(item, timeout): +def _validate_disable_debugger_detection(disable_debugger_detection, where): + if disable_debugger_detection is None: + return None + if not isinstance(disable_debugger_detection, bool): + raise ValueError( + "Invalid disable_debugger_detection value %s from %s" + % (disable_debugger_detection, where) + ) + return disable_debugger_detection + + +def timeout_sigalrm(item, settings): """Dump stack of threads and raise an exception. This will output the stacks of any threads other then the current to stderr and then raise an AssertionError, thus terminating the test. """ - if is_debugging(): + if not settings.disable_debugger_detection and is_debugging(): return __tracebackhide__ = True nthreads = len(threading.enumerate()) @@ -406,16 +445,16 @@ def timeout_sigalrm(item, timeout): dump_stacks() if nthreads > 1: write_title("Timeout", sep="+") - pytest.fail("Timeout >%ss" % timeout) + pytest.fail("Timeout >%ss" % settings.timeout) -def timeout_timer(item, timeout): +def timeout_timer(item, settings): """Dump stack of threads and call os._exit(). This disables the capturemanager and dumps stdout and stderr. Then the stacks are dumped and os._exit(1) is called. """ - if is_debugging(): + if not settings.disable_debugger_detection and is_debugging(): return try: capman = item.config.pluginmanager.getplugin("capturemanager") @@ -482,7 +521,7 @@ def write_title(title, stream=None, sep="~"): def write(text, stream=None): """Write text to stream. - Pretty stupid really, only here for symetry with .write_title(). + Pretty stupid really, only here for symmetry with .write_title(). """ if stream is None: stream = sys.stderr diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..be6b63f --- /dev/null +++ b/setup.cfg @@ -0,0 +1,39 @@ +[metadata] +name = pytest-timeout +description = pytest plugin to abort hanging tests +long_description = file: README.rst +version = 2.2.0 +author = Floris Bruynooghe +author_email = flub@devork.be +url = https://github.com/pytest-dev/pytest-timeout +license = MIT +classifiers = + Development Status :: 5 - Production/Stable + Environment :: Console + Environment :: Plugins + Intended Audience :: Developers + License :: DFSG approved + License :: OSI Approved :: MIT License + Operating System :: OS Independent + Programming Language :: Python + Programming Language :: Python :: Implementation :: PyPy + Programming Language :: Python :: Implementation :: CPython + Programming Language :: Python :: 3 + Programming Language :: Python :: 3 :: Only + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Topic :: Software Development :: Testing + Framework :: Pytest + +[options] +py_modules = pytest_timeout +install_requires = + pytest>=5.0.0 +python_requires = >=3.7 + +[options.entry_points] +pytest11 = + timeout = pytest_timeout diff --git a/setup.py b/setup.py index 212c0c1..4289c21 100644 --- a/setup.py +++ b/setup.py @@ -1,42 +1,5 @@ """Setuptools install script for pytest-timeout.""" from setuptools import setup -with open("README.rst", encoding="utf-8") as f: - long_description = f.read() - - -setup( - name="pytest-timeout", - description="pytest plugin to abort hanging tests", - long_description=long_description, - version="2.1.0", - author="Floris Bruynooghe", - author_email="flub@devork.be", - url="https://github.com/pytest-dev/pytest-timeout", - license="MIT", - py_modules=["pytest_timeout"], - entry_points={"pytest11": ["timeout = pytest_timeout"]}, - install_requires=["pytest>=5.0.0"], - python_requires=">=3.6", - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Environment :: Plugins", - "Intended Audience :: Developers", - "License :: DFSG approved", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: Implementation :: PyPy", - "Programming Language :: Python :: Implementation :: CPython", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Topic :: Software Development :: Testing", - "Framework :: Pytest", - ], -) +if __name__ == "__main__": + setup() diff --git a/test_pytest_timeout.py b/test_pytest_timeout.py index 1ff2213..da735eb 100644 --- a/test_pytest_timeout.py +++ b/test_pytest_timeout.py @@ -355,7 +355,30 @@ def test_ini_timeout_func_only(testdir): @pytest.fixture def slow(): time.sleep(2) + def test_foo(slow): + pass + """ + ) + testdir.makeini( + """ + [pytest] + timeout = 1 + timeout_func_only = true + """ + ) + result = testdir.runpytest() + assert result.ret == 0 + + +def test_ini_timeout_func_only_marker_override(testdir): + testdir.makepyfile( + """ + import time, pytest + @pytest.fixture + def slow(): + time.sleep(2) + @pytest.mark.timeout(1.5) def test_foo(slow): pass """ @@ -463,6 +486,50 @@ def test_foo(): assert "fail" not in result +@pytest.mark.parametrize( + ["debugging_module", "debugging_set_trace"], + [ + ("pdb", "set_trace()"), + pytest.param( + "ipdb", + "set_trace()", + marks=pytest.mark.xfail( + reason="waiting on https://github.com/pytest-dev/pytest/pull/7207" + " to allow proper testing" + ), + ), + pytest.param( + "pydevd", + "settrace(port=4678)", + marks=pytest.mark.xfail(reason="in need of way to setup pydevd server"), + ), + ], +) +@have_spawn +def test_disable_debugger_detection_flag( + testdir, debugging_module, debugging_set_trace +): + p1 = testdir.makepyfile( + """ + import pytest, {debugging_module} + + @pytest.mark.timeout(1) + def test_foo(): + {debugging_module}.{debugging_set_trace} + """.format( + debugging_module=debugging_module, debugging_set_trace=debugging_set_trace + ) + ) + child = testdir.spawn_pytest(f"{p1} --timeout-disable-debugger-detection") + child.expect("test_foo") + time.sleep(1.2) + result = child.read().decode().lower() + if child.isalive(): + child.terminate(force=True) + assert "timeout >1.0s" in result + assert "fail" in result + + def test_is_debugging(monkeypatch): import pytest_timeout diff --git a/tox.ini b/tox.ini index bc5e988..966ea49 100644 --- a/tox.ini +++ b/tox.ini @@ -3,7 +3,7 @@ minversion = 2.8 addopts = -ra [tox] -envlist = py36,py37,py38,py39,py310,pypy3 +envlist = py37,py38,py39,py310,py311,pypy3 [testenv] deps = pytest